Subject
When using a partial read (.read(512)) followed by .read(), when decode_content=True, only partial content is returned.
Environment
This should be unrelated to environment.
OS Linux-6.8.0-62-generic-x86_64-with-glibc2.39
Python 3.13.0
OpenSSL 3.0.15 3 Sep 2024
urllib3 2.5.0
Steps to Reproduce
Relevant code:
with urllib3.PoolManager() as pool:
with pool.request(
"GET",
f"http://127.0.0.1:12345{path}",
redirect=True,
decode_content=True,
preload_content=False,
) as response:
prefix = response.read(512)
remainder = response.read()here prefix + remainder != server_data, if the data has a Content-Encoding
full reproduction script
# /// script
# requires-python = ">=3.13"
# dependencies = [
# "urllib3[zstd]==2.5.0",
# "zstd",
# ]
# ///
import gzip
import http.server
import string
import threading
import zstd
import urllib3
data = 100 * string.printable.encode()
class HTTPTestRequestHandler(http.server.BaseHTTPRequestHandler):
def do_GET(self):
if self.path == "/":
self.send_response(200)
self.send_header("Content-Length", str(len(data)))
self.end_headers()
self.wfile.write(data)
elif self.path == "/gzip":
cdata = gzip.compress(data)
self.send_response(200)
self.send_header("Content-Encoding", "gzip")
self.send_header("Content-Length", str(len(cdata)))
self.end_headers()
self.wfile.write(cdata)
elif self.path == "/zstd":
cdata = zstd.compress(data)
self.send_response(200)
self.send_header("Content-Encoding", "zstd")
self.send_header("Content-Length", str(len(cdata)))
self.end_headers()
self.wfile.write(cdata)
else:
self.send_response(404)
self.send_header("Content-Length", "0")
self.end_headers()
httpd = http.server.ThreadingHTTPServer(("127.0.0.1", 12345), HTTPTestRequestHandler)
thread = threading.Thread(target=httpd.serve_forever)
thread.daemon = True
thread.start()
def test_equal(path: str):
with urllib3.PoolManager() as pool:
with pool.request(
"GET",
f"http://127.0.0.1:12345{path}",
redirect=True,
decode_content=True,
preload_content=False,
) as response:
prefix = response.read(512)
remainder = response.read()
if prefix + remainder != data:
print(f"{path} not equal:")
print(" ", prefix[-20:] + remainder[:120])
print(" ", data[512 - 20 : 512 + 120])
def show_version():
import platform
import ssl
import urllib3
print("OS", platform.platform())
print("Python", platform.python_version())
print(ssl.OPENSSL_VERSION)
print("urllib3", urllib3.__version__)
show_version()
test_equal("/")
test_equal("/gzip")
test_equal("/zstd")Expected Behavior
The read() should respect the decoded data, and return the remainder + the newly read data.
urllib3 v1 behaves this way.
Actual Behavior
read() reads the remaining data ignoring the cached _decoded_buffer.
127.0.0.1 - - [01/Jul/2025 10:22:53] "GET / HTTP/1.1" 200 -
127.0.0.1 - - [01/Jul/2025 10:22:53] "GET /gzip HTTP/1.1" 200 -
/gzip not equal:
b'}~ \t\n\r\x0b\x0c0123456789ab'
b'}~ \t\n\r\x0b\x0c0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~ \t\n\r\x0b\x0c0123456789abcdefghijklmnopqrstuv'
127.0.0.1 - - [01/Jul/2025 10:22:53] "GET /zstd HTTP/1.1" 200 -
/zstd not equal:
b'}~ \t\n\r\x0b\x0c0123456789ab'
b'}~ \t\n\r\x0b\x0c0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~ \t\n\r\x0b\x0c0123456789abcdefghijklmnopqrstuv'
.read(512) then .read(99999999999999) works around this issue, since it does not hit the relevant branch
|
if amt is None: |
|
data = self._decode(data, decode_content, flush_decoder) |
|
if cache_content: |
|
self._body = data |
|
else: |
|
# do not waste memory on buffer when not decoding |
|
if not decode_content: |
|
if self._has_decoded_content: |
|
raise RuntimeError( |
|
"Calling read(decode_content=False) is not supported after " |
|
"read(decode_content=True) was called." |
|
) |
|
return data |
|
|
|
decoded_data = self._decode(data, decode_content, flush_decoder) |
Subject
When using a partial read (
.read(512)) followed by.read(), whendecode_content=True, only partial content is returned.Environment
This should be unrelated to environment.
Steps to Reproduce
Relevant code:
here
prefix + remainder != server_data, if the data has aContent-Encodingfull reproduction script
Expected Behavior
The
read()should respect the decoded data, and return the remainder + the newly read data.urllib3v1 behaves this way.Actual Behavior
read()reads the remaining data ignoring the cached_decoded_buffer..read(512)then.read(99999999999999)works around this issue, since it does not hit the relevant branchurllib3/src/urllib3/response.py
Lines 987 to 1001 in 98a60f9