168 lines
6.8 KiB
Diff
168 lines
6.8 KiB
Diff
From b30d694a027eb771c02a3db0dee0ca03ccab7377 Mon Sep 17 00:00:00 2001
|
|
From: Stefan Eissing <stefan@eissing.org>
|
|
Date: Thu, 28 Mar 2024 11:08:15 +0100
|
|
Subject: [PATCH] content_encoding: brotli and others, pass through 0-length
|
|
writes
|
|
|
|
- curl's transfer handling may write 0-length chunks at the end of the
|
|
download with an EOS flag. (HTTP/2 does this commonly)
|
|
|
|
- content encoders need to pass-through such a write and not count this
|
|
as error in case they are finished decoding
|
|
|
|
Fixes #13209
|
|
Fixes #13212
|
|
Closes #13219
|
|
---
|
|
lib/content_encoding.c | 10 +++++-----
|
|
tests/http/test_02_download.py | 13 +++++++++++++
|
|
tests/http/testenv/env.py | 7 ++++++-
|
|
tests/http/testenv/httpd.py | 20 ++++++++++++++++++++
|
|
4 files changed, 44 insertions(+), 6 deletions(-)
|
|
|
|
diff --git a/lib/content_encoding.c b/lib/content_encoding.c
|
|
index c1abf24e8c027c..8e926dd2ecd5ad 100644
|
|
--- a/lib/content_encoding.c
|
|
+++ b/lib/content_encoding.c
|
|
@@ -300,7 +300,7 @@ static CURLcode deflate_do_write(struct Curl_easy *data,
|
|
struct zlib_writer *zp = (struct zlib_writer *) writer;
|
|
z_stream *z = &zp->z; /* zlib state structure */
|
|
|
|
- if(!(type & CLIENTWRITE_BODY))
|
|
+ if(!(type & CLIENTWRITE_BODY) || !nbytes)
|
|
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
|
|
|
|
/* Set the compressed input when this function is called */
|
|
@@ -457,7 +457,7 @@ static CURLcode gzip_do_write(struct Curl_easy *data,
|
|
struct zlib_writer *zp = (struct zlib_writer *) writer;
|
|
z_stream *z = &zp->z; /* zlib state structure */
|
|
|
|
- if(!(type & CLIENTWRITE_BODY))
|
|
+ if(!(type & CLIENTWRITE_BODY) || !nbytes)
|
|
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
|
|
|
|
if(zp->zlib_init == ZLIB_INIT_GZIP) {
|
|
@@ -669,7 +669,7 @@ static CURLcode brotli_do_write(struct Curl_easy *data,
|
|
CURLcode result = CURLE_OK;
|
|
BrotliDecoderResult r = BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT;
|
|
|
|
- if(!(type & CLIENTWRITE_BODY))
|
|
+ if(!(type & CLIENTWRITE_BODY) || !nbytes)
|
|
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
|
|
|
|
if(!bp->br)
|
|
@@ -762,7 +762,7 @@ static CURLcode zstd_do_write(struct Curl_easy *data,
|
|
ZSTD_outBuffer out;
|
|
size_t errorCode;
|
|
|
|
- if(!(type & CLIENTWRITE_BODY))
|
|
+ if(!(type & CLIENTWRITE_BODY) || !nbytes)
|
|
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
|
|
|
|
if(!zp->decomp) {
|
|
@@ -916,7 +916,7 @@ static CURLcode error_do_write(struct Curl_easy *data,
|
|
(void) buf;
|
|
(void) nbytes;
|
|
|
|
- if(!(type & CLIENTWRITE_BODY))
|
|
+ if(!(type & CLIENTWRITE_BODY) || !nbytes)
|
|
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
|
|
|
|
failf(data, "Unrecognized content encoding type. "
|
|
diff --git a/tests/http/test_02_download.py b/tests/http/test_02_download.py
|
|
index 4db9c9d36e9ed5..395fc862f2f839 100644
|
|
--- a/tests/http/test_02_download.py
|
|
+++ b/tests/http/test_02_download.py
|
|
@@ -394,6 +394,19 @@ def test_02_27_paused_no_cl(self, env: Env, httpd, nghttpx, repeat):
|
|
r = client.run(args=[url])
|
|
r.check_exit_code(0)
|
|
|
|
+ @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3'])
|
|
+ def test_02_28_get_compressed(self, env: Env, httpd, nghttpx, repeat, proto):
|
|
+ if proto == 'h3' and not env.have_h3():
|
|
+ pytest.skip("h3 not supported")
|
|
+ count = 1
|
|
+ urln = f'https://{env.authority_for(env.domain1brotli, proto)}/data-100k?[0-{count-1}]'
|
|
+ curl = CurlClient(env=env)
|
|
+ r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[
|
|
+ '--compressed'
|
|
+ ])
|
|
+ r.check_exit_code(code=0)
|
|
+ r.check_response(count=count, http_status=200)
|
|
+
|
|
def check_downloads(self, client, srcfile: str, count: int,
|
|
complete: bool = True):
|
|
for i in range(count):
|
|
diff --git a/tests/http/testenv/env.py b/tests/http/testenv/env.py
|
|
index a207059dcd57c5..13c5d6bd46ee57 100644
|
|
--- a/tests/http/testenv/env.py
|
|
+++ b/tests/http/testenv/env.py
|
|
@@ -129,10 +129,11 @@ def __init__(self):
|
|
self.htdocs_dir = os.path.join(self.gen_dir, 'htdocs')
|
|
self.tld = 'http.curl.se'
|
|
self.domain1 = f"one.{self.tld}"
|
|
+ self.domain1brotli = f"brotli.one.{self.tld}"
|
|
self.domain2 = f"two.{self.tld}"
|
|
self.proxy_domain = f"proxy.{self.tld}"
|
|
self.cert_specs = [
|
|
- CertificateSpec(domains=[self.domain1, 'localhost'], key_type='rsa2048'),
|
|
+ CertificateSpec(domains=[self.domain1, self.domain1brotli, 'localhost'], key_type='rsa2048'),
|
|
CertificateSpec(domains=[self.domain2], key_type='rsa2048'),
|
|
CertificateSpec(domains=[self.proxy_domain, '127.0.0.1'], key_type='rsa2048'),
|
|
CertificateSpec(name="clientsX", sub_specs=[
|
|
@@ -376,6 +377,10 @@ def htdocs_dir(self) -> str:
|
|
def domain1(self) -> str:
|
|
return self.CONFIG.domain1
|
|
|
|
+ @property
|
|
+ def domain1brotli(self) -> str:
|
|
+ return self.CONFIG.domain1brotli
|
|
+
|
|
@property
|
|
def domain2(self) -> str:
|
|
return self.CONFIG.domain2
|
|
diff --git a/tests/http/testenv/httpd.py b/tests/http/testenv/httpd.py
|
|
index c04c22699a62c4..b8615875a9a558 100644
|
|
--- a/tests/http/testenv/httpd.py
|
|
+++ b/tests/http/testenv/httpd.py
|
|
@@ -50,6 +50,7 @@ class Httpd:
|
|
'alias', 'env', 'filter', 'headers', 'mime', 'setenvif',
|
|
'socache_shmcb',
|
|
'rewrite', 'http2', 'ssl', 'proxy', 'proxy_http', 'proxy_connect',
|
|
+ 'brotli',
|
|
'mpm_event',
|
|
]
|
|
COMMON_MODULES_DIRS = [
|
|
@@ -203,6 +204,7 @@ def _mkpath(self, path):
|
|
|
|
def _write_config(self):
|
|
domain1 = self.env.domain1
|
|
+ domain1brotli = self.env.domain1brotli
|
|
creds1 = self.env.get_credentials(domain1)
|
|
domain2 = self.env.domain2
|
|
creds2 = self.env.get_credentials(domain2)
|
|
@@ -285,6 +287,24 @@ def _write_config(self):
|
|
f'</VirtualHost>',
|
|
f'',
|
|
])
|
|
+ # Alternate to domain1 with BROTLI compression
|
|
+ conf.extend([ # https host for domain1, h1 + h2
|
|
+ f'<VirtualHost *:{self.env.https_port}>',
|
|
+ f' ServerName {domain1brotli}',
|
|
+ f' Protocols h2 http/1.1',
|
|
+ f' SSLEngine on',
|
|
+ f' SSLCertificateFile {creds1.cert_file}',
|
|
+ f' SSLCertificateKeyFile {creds1.pkey_file}',
|
|
+ f' DocumentRoot "{self._docs_dir}"',
|
|
+ f' SetOutputFilter BROTLI_COMPRESS',
|
|
+ ])
|
|
+ conf.extend(self._curltest_conf(domain1))
|
|
+ if domain1 in self._extra_configs:
|
|
+ conf.extend(self._extra_configs[domain1])
|
|
+ conf.extend([
|
|
+ f'</VirtualHost>',
|
|
+ f'',
|
|
+ ])
|
|
conf.extend([ # https host for domain2, no h2
|
|
f'<VirtualHost *:{self.env.https_port}>',
|
|
f' ServerName {domain2}',
|