From f6707042d8b18d2a3737380bf58ff020872fb07b Mon Sep 17 00:00:00 2001 From: Bruce Adams Date: Mon, 27 Nov 2023 17:27:43 -0500 Subject: [PATCH 1/3] Unit test for string containing multi-byte UTF-8 There are two tests here. One demonstrating existing, correct behavior for `data=bytes`, and another, failing, test for the case where `data=string` and the string contains multi-byte UTF-8. --- tests/test_requests.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_requests.py b/tests/test_requests.py index a71fe7d6b8..b6fb84d1bd 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -1808,6 +1808,23 @@ def test_autoset_header_values_are_native(self, httpbin): assert p.headers["Content-Length"] == length + def test_content_length_for_bytes_data(self, httpbin): + data = "This is a string containing multi-byte UTF-8 ☃️" + encoded_data = data.encode("utf-8") + length = str(len(encoded_data)) + req = requests.Request("POST", httpbin("post"), data=encoded_data) + p = req.prepare() + + assert p.headers["Content-Length"] == length + + def test_content_length_for_string_data_counts_bytes(self, httpbin): + data = "This is a string containing multi-byte UTF-8 ☃️" + length = str(len(data.encode("utf-8"))) + req = requests.Request("POST", httpbin("post"), data=data) + p = req.prepare() + + assert p.headers["Content-Length"] == length + def test_nonhttp_schemes_dont_check_URLs(self): test_urls = ( "data:image/gif;base64,R0lGODlhAQABAHAAACH5BAUAAAAALAAAAAABAAEAAAICRAEAOw==", From 82d1641582f6d65c486aa06728407afd542a2ade Mon Sep 17 00:00:00 2001 From: goelbenj Date: Tue, 28 Nov 2023 13:06:01 -0500 Subject: [PATCH 2/3] Fix handling of len of strings in super_len --- src/requests/utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/requests/utils.py b/src/requests/utils.py index c3b123ea4e..651a5cb6db 100644 --- a/src/requests/utils.py +++ b/src/requests/utils.py @@ -134,9 +134,14 @@ def super_len(o): total_length = None current_position = 0 - if hasattr(o, "__len__"): + if hasattr(o, "__len__") and not isinstance(o, str): total_length = len(o) + elif isinstance(o, str): + # str with unicode chars might have multi-byte UTF-8 + # representations, so they must be encoded + total_length = len(o.encode('utf-8')) + elif hasattr(o, "len"): total_length = o.len From f9f587f9183b2335602b1e71fa951a8d3bf6b42e Mon Sep 17 00:00:00 2001 From: goelbenj Date: Tue, 28 Nov 2023 13:40:34 -0500 Subject: [PATCH 3/3] Run pre-commit --- src/requests/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/requests/utils.py b/src/requests/utils.py index 651a5cb6db..34b44af987 100644 --- a/src/requests/utils.py +++ b/src/requests/utils.py @@ -140,7 +140,7 @@ def super_len(o): elif isinstance(o, str): # str with unicode chars might have multi-byte UTF-8 # representations, so they must be encoded - total_length = len(o.encode('utf-8')) + total_length = len(o.encode("utf-8")) elif hasattr(o, "len"): total_length = o.len