GeezerVison is showing github.com — gh-107862: Fix infinite loop in email header folding found by a Hypothesis round-trip test by pganssle · Pull Request #159771 · python/cpython
gh-107862: Fix infinite loop in email header folding found by a Hypothesis round-trip test
#159771
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement . We’ll occasionally send you account related emails.
Already on GitHub?
Sign in
to your account
Uh oh!
There was an error while loading. Please reload this page.
Show file tree
Hide file tree
Changes from all commits
Commits
File filter
Filter by extension
.py
(3)
.rst
(2)
All 2 file types selected
Viewed files
Conversations
Failed to load comments.
Loading
Uh oh!
There was an error while loading. Please reload this page.
Jump to
Jump to file
Failed to load files.
Loading
Uh oh!
There was an error while loading. Please reload this page.
Diff view
Hide whitespace
Diff view
Hide whitespace
There are no files selected for viewing
Original file line number
Diff line number
Diff line change
Expand Up
@@ -3125,12 +3125,15 @@ def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset,
to_encode_word = to_encode[:text_space]
encoded_word = _ew.encode(to_encode_word, charset=encode_as)
excess = len(encoded_word) - remaining_space
while excess > 0:
while excess > 0 and len(to_encode_word) > 1 :
# Since the chunk to encode is guaranteed to fit into less than 100 characters,
# shrinking it by one at a time shouldn't take long.
to_encode_word = to_encode_word[:-1]
encoded_word = _ew.encode(to_encode_word, charset=encode_as)
excess = len(encoded_word) - remaining_space
# A single character may not fit even on a fresh line (a four byte
# UTF-8 sequence needs eight base64 characters). Emit it anyway,
# going over maxlen, rather than looping forever on an empty word.
lines[-1] += encoded_word
to_encode = to_encode[len(to_encode_word):]
Expand Down
Original file line number
Diff line number
Diff line change
@@ -1,3 +1,4 @@
from dataclasses import dataclass, replace
from enum import Enum
import functools
import unittest
Expand All
@@ -18,21 +19,52 @@
from . import stateful, strategies
@dataclass(frozen=True)
class _Example:
args: tuple
kwargs: dict
# Set by example.xfail(): the exception types the example must raise.
raises: type[BaseException] | tuple[type[BaseException], ...] | None = None
reason: str = ""
def __repr__(self):
bits = [repr(a) for a in self.args]
bits += [f"{k}={v!r}" for k, v in self.kwargs.items()]
return f"@example({', '.join(bits)})"
def _run_example(f, self, ex):
if ex.raises is None:
f(self, *ex.args, **ex.kwargs)
return
try:
f(self, *ex.args, **ex.kwargs)
except Exception as exc:
if not isinstance(exc, ex.raises):
raise
return
reason = f" because {ex.reason}" if ex.reason else ""
raise AssertionError(
f"Expected an exception from {ex!r}{reason}, "
"but no exception was raised"
)
def given(*_args, **_kwargs):
def decorator(f):
if examples := getattr(f, "_examples", []):
@functools.wraps(f)
def test_function(self):
for example_args, example_kwargs in examples:
if len(example_args ) < 2:
subtest_args = example_args
for ex in examples:
if len(ex.args ) < 2:
subtest_args = ex.args
else:
# subTest takes up to one positional argument.
# When there are more, display them as a tuple
subtest_args = [example_args ]
with self.subTest(*subtest_args, **example_kwargs ):
f(self, *example_args, **example_kwargs )
subtest_args = [ex.args ]
with self.subTest(*subtest_args, **ex.kwargs ):
_run_example(f, self, ex )
else:
# If we have found no examples, we must skip the test. If @example
Expand All
@@ -49,16 +81,18 @@ def test_function(self):
return decorator
def example(*args, **kwargs):
if bool(args) == bool(kwargs):
raise ValueError("Must specify exactly one of *args or **kwargs")
class example:
def __init__(self, *args, **kwargs):
if bool(args) == bool(kwargs):
raise ValueError("Must specify exactly one of *args or **kwargs")
self._example = _Example(args, kwargs)
def decorator( f):
def __call__(self, f):
base_func = getattr(f, "__wrapped__", f)
if not hasattr(base_func, "_examples"):
base_func._examples = []
base_func._examples.append((args, kwargs) )
base_func._examples.append(self._example )
if getattr(f, "_given", False):
# If the given decorator is below all the example decorators,
Expand All
@@ -68,7 +102,17 @@ def decorator(f):
return f
return decorator
def xfail(self, condition=True, *, reason="", raises=BaseException):
# Like real Hypothesis, an expected failure is strict: the example
# must raise one of the given exceptions, or the test fails.
if condition:
self._example = replace(self._example, raises=raises,
reason=reason)
return self
def via(self, whence, /):
# A label for tooling; it does not change what runs.
return self
def assume(condition):
Expand Down
Original file line number
Diff line number
Diff line change
Expand Up
@@ -8,6 +8,7 @@
import email.generator
import email.message
from email import headerregistry
from test.support.hypothesis_helper import hypothesis
def make_defaults(base_defaults, differences):
defaults = base_defaults.copy()
Expand Down
Expand Up
@@ -295,6 +296,74 @@ def test_short_maxlen_error(self):
with self.assertRaises(email.errors.HeaderParseError):
policy.fold("Subject", subject)
def test_wide_char_too_long_for_line_does_not_cause_inf_loop(self):
# A four byte UTF-8 character needs 12 characters in a q-encoded
# word, so with the RFC 2047 chrome and the leading space it cannot
# fit on a 20 character line at all. Rather than looping forever
# trying to shrink the word, the line is allowed to run over.
policy = email.policy.default.clone(max_line_length=20)
actual = policy.fold('Subject', '\U0001f600')
self.assertEqual(actual, 'Subject:\n =?utf-8?q?=F0=9F=98=80?=\n')
# Pieces chosen to exercise folding: words, runs of whitespace, RFC 5322
# specials, characters needing 2, 3 and 4 byte UTF-8 encodings, and the
# delimiters of RFC 2047 encoded words. The pieces cannot combine into a
# well-formed encoded word (there is no lone '?'), since such text is
# written out verbatim and decoded on reparse rather than encoded; that
# is gh-160148, pinned by the expected-failure examples below.
_fold_pieces = hypothesis.strategies.sampled_from([
'a', 'word', 'longwordlongwordlongword', ' ', ' ', '\t', ',', ';',
':', '(', ')', '<', '>', '"', '@', '\\', '.', '-', '_', '\xa0',
'\xe9', '\u0444', '\u3042', '\U0001f600', '=?', '?=',
])
@hypothesis.given(
subject=hypothesis.strategies.lists(
_fold_pieces, max_size=12).map(''.join),
maxlen=(hypothesis.strategies.integers(1, 120)
| hypothesis.strategies.sampled_from([0, None])),
)
# Looped forever before the single character case was handled.
@hypothesis.example(subject='\U0001f600', maxlen=20)
@hypothesis.example(subject='\U0001f600a', maxlen=16)
@hypothesis.example(subject='\U0001f600\U0001f600', maxlen=20)
# bpo-33529 and bpo-36564, the earlier infinite loops.
@hypothesis.example(subject='\u0105' * 12, maxlen=20)
@hypothesis.example(subject='Melt away the pounds! 1234567890', maxlen=9)
# Encoded words that must be recombined on reparse (test_generator).
@hypothesis.example(subject='aaaaaaaaa \u0444\u0444\u0444\u0444\u0444\u0444\u0444\u0444 ', maxlen=30)
@hypothesis.example(subject='aaa aaaaaaa aaa \u0444\u0444\u0444 \u0444\u0444\u0444\u0444 ', maxlen=30)
# No wrapping at all, and nothing but whitespace.
@hypothesis.example(subject='word \U0001f600 word', maxlen=None)
@hypothesis.example(subject='word \U0001f600 word', maxlen=0)
@hypothesis.example(subject=' ', maxlen=78)
@hypothesis.example(subject='', maxlen=78)
# gh-160148: text that is itself a well-formed encoded word.
@hypothesis.example(subject='=?utf-8?q?x?=', maxlen=78).xfail(
reason="gh-160148", raises=AssertionError)
@hypothesis.example(subject='=?utf-8?q?x?=)', maxlen=78).xfail(
reason="gh-160148", raises=AssertionError)
@hypothesis.example(subject='=?utf-8?q?x?=\xe9', maxlen=78).xfail(
reason="gh-160148", raises=AssertionError)
def test_fold_unstructured_round_trip(self, subject, maxlen):
policy = email.policy.default.clone(max_line_length=maxlen)
try:
folded = policy.fold('Subject', subject)
except email.errors.HeaderParseError:
# The utf-8 RFC 2047 chrome is 12 characters, so no encoded
# word fits on a line of 13 characters or fewer; see
# test_short_maxlen_error. Any other width must fold.
self.assertIsNotNone(maxlen)
self.assertLessEqual(maxlen, 13)
return
reparsed = email.message_from_string(
folded, policy=email.policy.default)['Subject']
# Leading whitespace is dropped by the parser; everything else,
# including trailing whitespace, survives.
self.assertEqual(reparsed.strip(' \t'), subject.strip(' \t'))
if subject == subject.lstrip(' \t'):
self.assertEqual(reparsed, subject)
def test_verify_generated_headers(self):
# Turning protection off allows header injection
policy = email.policy.default.clone(verify_generated_headers=False)
Expand Down
Original file line number
Diff line number
Diff line change
@@ -0,0 +1,3 @@
Fix an infinite loop in :mod:`email` header folding when a single character's
:rfc:`2047` encoded word is wider than the policy's ``max_line_length``; the
line now exceeds the limit instead.
Original file line number
Diff line number
Diff line change
@@ -0,0 +1,3 @@
Add ``example.xfail()`` and ``example.via()`` to the Hypothesis stubs used
when Hypothesis is not installed, so that property tests can mark an explicit
example as an expected failure in both modes.
Loading
Oops, something went wrong.
Uh oh!
There was an error while loading. Please reload this page.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.