From d84f9ce5cdee311594302033ac0616430955d2c9 Mon Sep 17 00:00:00 2001 From: Rahul Ranjan Date: Sun, 8 Nov 2026 22:00:14 +0000 Subject: [PATCH 1/3] fix(parser): honor a legacy CHARSET property parameter when parsing bytes from_ical(bytes) decoded non-UTF-8 input with errors=replace before the property parameters were ever looked at, so a value that explicitly declared its encoding with a legacy CHARSET parameter (the marker pre-RFC 5545 producers left behind) still came back as U+FFFD soup. Decode each content line with the charset its CHARSET parameter declares; lines without the parameter keep the previous best-effort UTF-8 decoding, and an unknown charset falls back the same way. Fixes #2078 AI: GLM-5.3 (z-ai, via OpenRouter). How used: AI coding assistant. I supplied the repro from issue #2078 and the fix design (decode per content line, restricted to the parameter section before the first colon); the model drafted the patch, the regression tests and this message from my instructions. I reviewed the change and verified it by running the test suite. Co-authored-by: rahul05ranjan <11793455+rahul05ranjan@users.noreply.github.com> --- news/2078.bugfix | 1 + src/icalendar/parser/content_line.py | 37 +++++++++++++++++++++-- src/icalendar/tests/test_encoding.py | 45 ++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 news/2078.bugfix diff --git a/news/2078.bugfix b/news/2078.bugfix new file mode 100644 index 000000000..c45ccfceb --- /dev/null +++ b/news/2078.bugfix @@ -0,0 +1 @@ +Honor a legacy ``CHARSET`` property parameter when parsing bytes: a value that declares its encoding (e.g. ``SUMMARY;CHARSET=ISO-8859-1:...``) now decodes with that charset instead of coming back as U+FFFD replacement characters. I used AI to help with this change. @rahul05ranjan diff --git a/src/icalendar/parser/content_line.py b/src/icalendar/parser/content_line.py index 00a7d5313..754f31968 100644 --- a/src/icalendar/parser/content_line.py +++ b/src/icalendar/parser/content_line.py @@ -18,6 +18,11 @@ # to the first line break of a run, making a failed attempt O(1) instead of a # full rescan, while matching exactly the same strings. UFOLD = re.compile(r"(?:(? None: return "".join(out).strip() +def _decode_with_charset(value: bytes) -> str: + """Decode content lines, honoring a legacy ``CHARSET`` parameter. + + When the input is not valid UTF-8 as a whole, decode each content + line with the charset its ``CHARSET`` parameter declares, so a value + that names its encoding does not come back as U+FFFD. Lines without + a ``CHARSET`` parameter (or with an unknown one) keep the previous + best-effort UTF-8 decoding. + """ + lines = [] + for line in NEWLINE_BYTES.split(value): + # the parameters end at the first colon: a parameter VALUE quoting + # the text "CHARSET=..." must not select an encoding + head, _sep, _tail = line.partition(b":") + match = CHARSET_PARAM_BYTES.search(head) + if match: + charset = match.group(1).decode("ascii", "replace").strip().strip('"') + try: + lines.append(line.decode(charset)) + continue + except (LookupError, UnicodeDecodeError): + pass + lines.append(to_unicode(line)) + return "\r\n".join(lines) + + class Contentline(str): r"""A content line is a string that can be folded and parsed into parts. @@ -287,7 +318,9 @@ def value_separator_index(self) -> int: @classmethod def from_ical(cls, ical, strict=False): """Unfold the content lines in an iCalendar into long content lines.""" - ical = to_unicode(ical) + ical = ( + _decode_with_charset(ical) if isinstance(ical, bytes) else to_unicode(ical) + ) # a fold is carriage return followed by either a space or a tab return cls(UFOLD.sub("", ical), strict=strict) @@ -311,7 +344,7 @@ def to_ical(self): @classmethod def from_ical(cls, st): """Parses a string into content lines.""" - st = to_unicode(st) + st = _decode_with_charset(st) if isinstance(st, bytes) else to_unicode(st) try: # a fold is carriage return followed by either a space or a tab unfolded = UFOLD.sub("", st) diff --git a/src/icalendar/tests/test_encoding.py b/src/icalendar/tests/test_encoding.py index 4cc1557b0..993bc002f 100644 --- a/src/icalendar/tests/test_encoding.py +++ b/src/icalendar/tests/test_encoding.py @@ -2,6 +2,8 @@ import pytest +from icalendar import Calendar + @pytest.mark.parametrize( ("field", "expected_value"), @@ -96,3 +98,46 @@ def test_apple_xlocation(calendars): """ for event in calendars.x_location.walk("vevent"): assert len(event.errors) == 0, "Got too many errors" + + +def test_charset_parameter_decodes_legacy_non_utf8_value_issue_2078(): + """A legacy CHARSET parameter declares the encoding of the property + value: latin-1 bytes decode with the declared charset instead of + coming back as U+FFFD replacement characters (issue #2078). + """ + ics = ( + b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//t//EN\r\n" + b"BEGIN:VEVENT\r\nUID:1\r\nDTSTART:20261108T190000Z\r\n" + b"SUMMARY;CHARSET=ISO-8859-1:Caf\xe9 Meeting\r\n" + b"END:VEVENT\r\nEND:VCALENDAR\r\n" + ) + event = Calendar.from_ical(ics).walk("VEVENT")[0] + assert str(event["SUMMARY"]) == "Caf\u00e9 Meeting" + + +def test_unknown_charset_falls_back_gracefully_issue_2078(): + """An unknown CHARSET must not raise; the value degrades exactly as + it did before the CHARSET parameter was honored (issue #2078).""" + ics = ( + b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//t//EN\r\n" + b"BEGIN:VEVENT\r\nUID:1\r\n" + b"SUMMARY;CHARSET=BOGUS-XYZ:Caf\xe9 Meeting\r\n" + b"END:VEVENT\r\nEND:VCALENDAR\r\n" + ) + event = Calendar.from_ical(ics).walk("VEVENT")[0] + assert "Meeting" in str(event["SUMMARY"]) + + +def test_charset_text_inside_quoted_param_value_is_not_a_directive_issue_2078(): + """UTF-8 text with a quoted parameter value mentioning CHARSET= must + still decode as UTF-8, not as the encoding named in the quotes.""" + summary = "Caf\u00e9 Meeting" + ics = ( + b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//t//EN\r\n" + b"BEGIN:VEVENT\r\nUID:1\r\n" + b'SUMMARY;ALTREP="cid:x;CHARSET=ISO-8859-1":' + + summary.encode("utf-8") + + b"\r\nEND:VEVENT\r\nEND:VCALENDAR\r\n" + ) + event = Calendar.from_ical(ics).walk("VEVENT")[0] + assert str(event["SUMMARY"]) == summary From f8467d3cb3a1b9ce3ae308c576637c61412f3d87 Mon Sep 17 00:00:00 2001 From: Rahul Ranjan Date: Sun, 15 Nov 2026 19:30:52 +0000 Subject: [PATCH 2/3] test(parser): clarify RFC citations and exercise both CHARSET regex variants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback on #2080 (thank you @angatha): - RFC 5545 §3.1.4, not the §3.1 grammar, is the section on charset: it states "There is not a property parameter to declare the charset used in a property value. The default charset for an iCalendar stream is UTF-8". The quoted-parameter-value test now cites what actually supports it: §3.1's `quoted-string = DQUOTE *QSAFE-CHAR DQUOTE` (QSAFE-CHAR = any character except CONTROL and DQUOTE), which lets ";" and "=" appear inside a quoted parameter value, and §3.1.4, which is why an unquoted legacy CHARSET= is only ever tolerated (issue #2078), never conforming. - The CHARSET parameter regex now uses a possessive quantifier where the running Python supports it, following the existing version-guard pattern from the XML property (prop/xml.py): Python 3.11+ gets `([^;:\r\n]++)`, Python 3.10 keeps the plain greedy form (the character class excludes the delimiters, so both match the same strings). The three issue #2078 tests are parametrized over both regexes so each runs everywhere it compiles. Co-authored-by: rahul05ranjan <11793455+rahul05ranjan@users.noreply.github.com> --- src/icalendar/parser/content_line.py | 21 ++++++++++++-- src/icalendar/tests/test_encoding.py | 42 +++++++++++++++++++++++++--- 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/src/icalendar/parser/content_line.py b/src/icalendar/parser/content_line.py index b8942ba80..6e0388b54 100644 --- a/src/icalendar/parser/content_line.py +++ b/src/icalendar/parser/content_line.py @@ -1,6 +1,7 @@ """parsing and generation of content lines""" import re +import sys from icalendar.parser.parameter import Parameters, closes_quoted_section from icalendar.parser.property import unescape_backslash, unescape_list_or_string @@ -19,10 +20,26 @@ # full rescan, while matching exactly the same strings. UFOLD = re.compile(r"(?:(?= (3, 11): + # the possessive quantifier never gives matched characters back, + # keeping the match linear + CHARSET_PARAM_BYTES = re.compile( + rb";[ \t]*CHARSET[ \t]*=[ \t]*([^;:\r\n]++)", re.IGNORECASE + ) +else: + CHARSET_PARAM_BYTES = _CHARSET_PARAM_BYTES_PY310 NEWLINE = re.compile(r"\r?\n") OWS = " \t" diff --git a/src/icalendar/tests/test_encoding.py b/src/icalendar/tests/test_encoding.py index 993bc002f..a61a1e6b0 100644 --- a/src/icalendar/tests/test_encoding.py +++ b/src/icalendar/tests/test_encoding.py @@ -1,8 +1,10 @@ import datetime +import sys import pytest from icalendar import Calendar +from icalendar.parser import content_line @pytest.mark.parametrize( @@ -100,7 +102,9 @@ def test_apple_xlocation(calendars): assert len(event.errors) == 0, "Got too many errors" -def test_charset_parameter_decodes_legacy_non_utf8_value_issue_2078(): +def test_charset_parameter_decodes_legacy_non_utf8_value_issue_2078( + charset_param_regex, +): """A legacy CHARSET parameter declares the encoding of the property value: latin-1 bytes decode with the declared charset instead of coming back as U+FFFD replacement characters (issue #2078). @@ -115,7 +119,7 @@ def test_charset_parameter_decodes_legacy_non_utf8_value_issue_2078(): assert str(event["SUMMARY"]) == "Caf\u00e9 Meeting" -def test_unknown_charset_falls_back_gracefully_issue_2078(): +def test_unknown_charset_falls_back_gracefully_issue_2078(charset_param_regex): """An unknown CHARSET must not raise; the value degrades exactly as it did before the CHARSET parameter was honored (issue #2078).""" ics = ( @@ -128,9 +132,39 @@ def test_unknown_charset_falls_back_gracefully_issue_2078(): assert "Meeting" in str(event["SUMMARY"]) -def test_charset_text_inside_quoted_param_value_is_not_a_directive_issue_2078(): +@pytest.fixture(params=["py311", "py310"]) +def charset_param_regex(request, monkeypatch): + """Run with the CHARSET regex of Python 3.11+ and of Python 3.10. + + Python 3.10 has no possessive quantifiers, so it uses a simpler regex. + Both are tested where they can be compiled. + """ + if request.param == "py310": + monkeypatch.setattr( + content_line, + "CHARSET_PARAM_BYTES", + content_line._CHARSET_PARAM_BYTES_PY310, + ) + elif sys.version_info < (3, 11): + pytest.skip("Possessive quantifiers need Python 3.11 or later.") + + +def test_charset_text_inside_quoted_param_value_is_not_a_directive_issue_2078( + charset_param_regex, +): """UTF-8 text with a quoted parameter value mentioning CHARSET= must - still decode as UTF-8, not as the encoding named in the quotes.""" + still decode as UTF-8, not as the encoding named in the quotes. + + RFC 5545 §3.1.4 states that there is no property parameter to declare + the charset of a value (iCalendar text is UTF-8), so CHARSET is not a + conforming parameter at all — an unquoted top-level ``CHARSET=`` is only + tolerated as a legacy marker of pre-RFC 5545 producers (issue #2078). + Inside a quoted parameter value it is plain data: RFC 5545 §3.1 defines + ``quoted-string = DQUOTE *QSAFE-CHAR DQUOTE`` with QSAFE-CHAR being any + character except CONTROL and DQUOTE, so ";" and "=" (and thus the text + ``;CHARSET=ISO-8859-1``) may occur inside the quotes without forming a + second parameter. + """ summary = "Caf\u00e9 Meeting" ics = ( b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//t//EN\r\n" From 6d0241e5bd6eb66cd25dd2e7dc8da4c7b31e24c5 Mon Sep 17 00:00:00 2001 From: rahul05ranjan <11793455+rahul05ranjan@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:27:37 +0000 Subject: [PATCH 3/3] fix(parser): keep valid UTF-8 input on the untouched UTF-8 path The _decode_with_charset docstring promised that valid UTF-8 input is never rerouted through the CHARSET parameter, but the code checked no such gate: a file that was re-encoded to UTF-8 while keeping a vestigial legacy CHARSET=ISO-8859-1 parameter decoded its UTF-8 bytes as latin-1 and turned non-ASCII text into mojibake, where the pre-CHARSET code parsed it correctly. Valid UTF-8 input (a conforming RFC 5545 stream) is now decoded as before, so a vestigial CHARSET cannot override the charset the stream actually uses. Only input that is not valid UTF-8 as a whole gets the per-line legacy CHARSET decoding. The quoted-parameter test gains a latin-1 DESCRIPTION line so it still exercises the CHARSET scan under that gate, and a new regression test pins the vestigial-parameter case. Co-authored-by: rahul05ranjan <11793455+rahul05ranjan@users.noreply.github.com> --- src/icalendar/parser/content_line.py | 17 ++++++++++---- src/icalendar/tests/test_encoding.py | 35 ++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/src/icalendar/parser/content_line.py b/src/icalendar/parser/content_line.py index 6e0388b54..7f5472455 100644 --- a/src/icalendar/parser/content_line.py +++ b/src/icalendar/parser/content_line.py @@ -146,12 +146,19 @@ def flush_pending() -> None: def _decode_with_charset(value: bytes) -> str: """Decode content lines, honoring a legacy ``CHARSET`` parameter. - When the input is not valid UTF-8 as a whole, decode each content - line with the charset its ``CHARSET`` parameter declares, so a value - that names its encoding does not come back as U+FFFD. Lines without - a ``CHARSET`` parameter (or with an unknown one) keep the previous - best-effort UTF-8 decoding. + Input that is valid UTF-8 — a conforming :rfc:`5545` stream — is + decoded exactly as before this parameter was honored, so a vestigial + ``CHARSET`` parameter in a file that actually uses UTF-8 cannot + override it. Only when the input is not valid UTF-8 as a whole is + each content line decoded with the charset its ``CHARSET`` parameter + declares, so a value that names its encoding does not come back as + U+FFFD. Lines without a ``CHARSET`` parameter (or with an unknown + one) keep the previous best-effort UTF-8 decoding. """ + try: + return value.decode("utf-8-sig") + except UnicodeDecodeError: + pass lines = [] for line in NEWLINE_BYTES.split(value): # the parameters end at the first colon: a parameter VALUE quoting diff --git a/src/icalendar/tests/test_encoding.py b/src/icalendar/tests/test_encoding.py index a61a1e6b0..3a89ca31f 100644 --- a/src/icalendar/tests/test_encoding.py +++ b/src/icalendar/tests/test_encoding.py @@ -164,13 +164,48 @@ def test_charset_text_inside_quoted_param_value_is_not_a_directive_issue_2078( character except CONTROL and DQUOTE, so ";" and "=" (and thus the text ``;CHARSET=ISO-8859-1``) may occur inside the quotes without forming a second parameter. + + The ``DESCRIPTION`` line makes the whole file invalid UTF-8, so the + per-line CHARSET scan runs: the ``SUMMARY`` line is valid UTF-8 but + must still not be decoded with the charset named inside its quoted + parameter value, while the real directive on ``DESCRIPTION`` is + honored. """ summary = "Caf\u00e9 Meeting" + description = "Caf\u00e9 R\u00e9sum\u00e9" # latin-1 bytes below ics = ( b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//t//EN\r\n" b"BEGIN:VEVENT\r\nUID:1\r\n" b'SUMMARY;ALTREP="cid:x;CHARSET=ISO-8859-1":' + summary.encode("utf-8") + + b"\r\nDESCRIPTION;CHARSET=ISO-8859-1:" + + description.encode("iso-8859-1") + + b"\r\nEND:VEVENT\r\nEND:VCALENDAR\r\n" + ) + event = Calendar.from_ical(ics).walk("VEVENT")[0] + assert str(event["SUMMARY"]) == summary + assert str(event["DESCRIPTION"]) == description + + +def test_vestigial_charset_in_valid_utf8_does_not_override_issue_2078( + charset_param_regex, +): + """A file that is valid UTF-8 parses as UTF-8 even if it still + carries a vestigial legacy ``CHARSET`` parameter. + + Re-encoded migrations of legacy files can keep the old ``CHARSET`` + parameter while the bytes are already UTF-8. RFC 5545 §3.1.4 makes + UTF-8 the charset of a conforming stream, so valid UTF-8 input must + parse exactly as it did before this parameter was honored: the + declared charset must not turn the UTF-8 text into mojibake + (issue #2078). + """ + summary = "Caf\u00e9 Meeting" + ics = ( + b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//t//EN\r\n" + b"BEGIN:VEVENT\r\nUID:1\r\n" + b"SUMMARY;CHARSET=ISO-8859-1:" + + summary.encode("utf-8") + b"\r\nEND:VEVENT\r\nEND:VCALENDAR\r\n" ) event = Calendar.from_ical(ics).walk("VEVENT")[0]