From ed81971e6b26c34445f06850192b34458b029337 Mon Sep 17 00:00:00 2001 From: RanKKI Date: Sun, 17 Nov 2024 10:01:52 +1100 Subject: [PATCH] gh-124452: Fix header mismatches when folding/unfolding with email message (#125919) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The header-folder of the new email API has a long standing known buglet where if the first token is longer than max_line_length, it puts that token on the next line. It turns out there is also a *parsing* bug when parsing such a header: the space prefixing that first, non-empty line gets preserved and tacked on to the start of the header value, which is not the expected behavior per the RFCs. The bug arises from the fact that the parser assumed that there would be at least one token on the line with the header, which is going to be true for probably every email producer other than the python email library with its folding buglet. Clearly, though, this is a case that needs to be handled correctly. The fix is simple: strip the blanks off the start of the whole value, not just the first physical line of the value. Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/email/_policybase.py | 4 +- Lib/email/policy.py | 4 +- Lib/test/test_email/test_message.py | 50 ++++++++++++++++++- ...-10-24-10-49-47.gh-issue-124452.eqTRgx.rst | 4 ++ 4 files changed, 56 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-10-24-10-49-47.gh-issue-124452.eqTRgx.rst diff --git a/Lib/email/_policybase.py b/Lib/email/_policybase.py index c7694a44e26..4b63b97217a 100644 --- a/Lib/email/_policybase.py +++ b/Lib/email/_policybase.py @@ -302,12 +302,12 @@ class Compat32(Policy): """+ The name is parsed as everything up to the ':' and returned unmodified. The value is determined by stripping leading whitespace off the - remainder of the first line, joining all subsequent lines together, and + remainder of the first line joined with all subsequent lines, and stripping any trailing carriage return or linefeed characters. """ name, value = sourcelines[0].split(':', 1) - value = value.lstrip(' \t') + ''.join(sourcelines[1:]) + value = ''.join((value, *sourcelines[1:])).lstrip(' \t\r\n') return (name, value.rstrip('\r\n')) def header_store_parse(self, name, value): diff --git a/Lib/email/policy.py b/Lib/email/policy.py index 46b7de5bb6d..6e109b65011 100644 --- a/Lib/email/policy.py +++ b/Lib/email/policy.py @@ -119,13 +119,13 @@ class EmailPolicy(Policy): """+ The name is parsed as everything up to the ':' and returned unmodified. The value is determined by stripping leading whitespace off the - remainder of the first line, joining all subsequent lines together, and + remainder of the first line joined with all subsequent lines, and stripping any trailing carriage return or linefeed characters. (This is the same as Compat32). """ name, value = sourcelines[0].split(':', 1) - value = value.lstrip(' \t') + ''.join(sourcelines[1:]) + value = ''.join((value, *sourcelines[1:])).lstrip(' \t\r\n') return (name, value.rstrip('\r\n')) def header_store_parse(self, name, value): diff --git a/Lib/test/test_email/test_message.py b/Lib/test/test_email/test_message.py index 034f7626c1f..96979db27f3 100644 --- a/Lib/test/test_email/test_message.py +++ b/Lib/test/test_email/test_message.py @@ -1,6 +1,6 @@ -import unittest import textwrap -from email import policy, message_from_string +import unittest +from email import message_from_bytes, message_from_string, policy from email.message import EmailMessage, MIMEPart from test.test_email import TestEmailBase, parameterize @@ -958,6 +958,52 @@ class TestEmailMessage(TestEmailMessageBase, TestEmailBase): b'123456789-123456789\n 123456789 Hello ' b'=?utf-8?q?W=C3=B6rld!?= 123456789 123456789\n\n') + def test_folding_with_short_nospace_1(self): + # bpo-36520 + # + # Fold a line that contains a long whitespace after + # the fold point. + + m = EmailMessage(policy.default) + m['Message-ID'] = '123456789' * 3 + parsed_msg = message_from_bytes(m.as_bytes(), policy=policy.default) + self.assertEqual(parsed_msg['Message-ID'], m['Message-ID']) + + def test_folding_with_long_nospace_default_policy_1(self): + # Fixed: https://github.com/python/cpython/issues/124452 + # + # When the value is too long, it should be converted back + # to its original form without any modifications. + + m = EmailMessage(policy.default) + message = '123456789' * 10 + m['Message-ID'] = message + self.assertEqual(m.as_bytes(), + f'Message-ID:\n {message}\n\n'.encode()) + parsed_msg = message_from_bytes(m.as_bytes(), policy=policy.default) + self.assertEqual(parsed_msg['Message-ID'], m['Message-ID']) + + def test_folding_with_long_nospace_compat32_policy_1(self): + m = EmailMessage(policy.compat32) + message = '123456789' * 10 + m['Message-ID'] = message + parsed_msg = message_from_bytes(m.as_bytes(), policy=policy.default) + self.assertEqual(parsed_msg['Message-ID'], m['Message-ID']) + + def test_folding_with_long_nospace_smtp_policy_1(self): + m = EmailMessage(policy.SMTP) + message = '123456789' * 10 + m['Message-ID'] = message + parsed_msg = message_from_bytes(m.as_bytes(), policy=policy.default) + self.assertEqual(parsed_msg['Message-ID'], m['Message-ID']) + + def test_folding_with_long_nospace_http_policy_1(self): + m = EmailMessage(policy.HTTP) + message = '123456789' * 10 + m['Message-ID'] = message + parsed_msg = message_from_bytes(m.as_bytes(), policy=policy.default) + self.assertEqual(parsed_msg['Message-ID'], m['Message-ID']) + def test_get_body_malformed(self): """test for bpo-42892""" msg = textwrap.dedent("""\ diff --git a/Misc/NEWS.d/next/Library/2024-10-24-10-49-47.gh-issue-124452.eqTRgx.rst b/Misc/NEWS.d/next/Library/2024-10-24-10-49-47.gh-issue-124452.eqTRgx.rst new file mode 100644 index 00000000000..b0d63794022 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-24-10-49-47.gh-issue-124452.eqTRgx.rst @@ -0,0 +1,4 @@ +Fix an issue in :meth:`email.policy.EmailPolicy.header_source_parse` and +:meth:`email.policy.Compat32.header_source_parse` that introduced spurious +leading whitespaces into header values when the header includes a newline +character after the header name delimiter (``:``) and before the value.