diff --git a/news/2078.bugfix b/news/2078.bugfix new file mode 100644 index 000000000..c45ccfceb --- /dev/null +++ b/news/2078.bugfix @@ -0,0 +1 @@ +Honor a legacy ``CHARSET`` property parameter when parsing bytes: a value that declares its encoding (e.g. ``SUMMARY;CHARSET=ISO-8859-1:...``) now decodes with that charset instead of coming back as U+FFFD replacement characters. I used AI to help with this change. @rahul05ranjan diff --git a/src/icalendar/parser/content_line.py b/src/icalendar/parser/content_line.py index 9047546b0..7f5472455 100644 --- a/src/icalendar/parser/content_line.py +++ b/src/icalendar/parser/content_line.py @@ -1,6 +1,7 @@ """parsing and generation of content lines""" import re +import sys from icalendar.parser.parameter import Parameters, closes_quoted_section from icalendar.parser.property import unescape_backslash, unescape_list_or_string @@ -18,6 +19,27 @@ # to the first line break of a run, making a failed attempt O(1) instead of a # full rescan, while matching exactly the same strings. UFOLD = re.compile(r"(?:(?= (3, 11): + # the possessive quantifier never gives matched characters back, + # keeping the match linear + CHARSET_PARAM_BYTES = re.compile( + rb";[ \t]*CHARSET[ \t]*=[ \t]*([^;:\r\n]++)", re.IGNORECASE + ) +else: + CHARSET_PARAM_BYTES = _CHARSET_PARAM_BYTES_PY310 NEWLINE = re.compile(r"\r?\n") OWS = " \t" @@ -121,6 +143,39 @@ def flush_pending() -> None: return "".join(out).strip() +def _decode_with_charset(value: bytes) -> str: + """Decode content lines, honoring a legacy ``CHARSET`` parameter. + + Input that is valid UTF-8 — a conforming :rfc:`5545` stream — is + decoded exactly as before this parameter was honored, so a vestigial + ``CHARSET`` parameter in a file that actually uses UTF-8 cannot + override it. Only when the input is not valid UTF-8 as a whole is + each content line decoded with the charset its ``CHARSET`` parameter + declares, so a value that names its encoding does not come back as + U+FFFD. Lines without a ``CHARSET`` parameter (or with an unknown + one) keep the previous best-effort UTF-8 decoding. + """ + try: + return value.decode("utf-8-sig") + except UnicodeDecodeError: + pass + lines = [] + for line in NEWLINE_BYTES.split(value): + # the parameters end at the first colon: a parameter VALUE quoting + # the text "CHARSET=..." must not select an encoding + head, _sep, _tail = line.partition(b":") + match = CHARSET_PARAM_BYTES.search(head) + if match: + charset = match.group(1).decode("ascii", "replace").strip().strip('"') + try: + lines.append(line.decode(charset)) + continue + except (LookupError, UnicodeDecodeError): + pass + lines.append(to_unicode(line)) + return "\r\n".join(lines) + + class Contentline(str): r"""A content line is a string that can be folded and parsed into parts. @@ -318,7 +373,9 @@ def value_separator_index(self) -> int: @classmethod def from_ical(cls, ical, strict=False): """Unfold the content lines in an iCalendar into long content lines.""" - ical = to_unicode(ical) + ical = ( + _decode_with_charset(ical) if isinstance(ical, bytes) else to_unicode(ical) + ) # a fold is carriage return followed by either a space or a tab return cls(UFOLD.sub("", ical), strict=strict) @@ -342,7 +399,7 @@ def to_ical(self): @classmethod def from_ical(cls, st): """Parses a string into content lines.""" - st = to_unicode(st) + st = _decode_with_charset(st) if isinstance(st, bytes) else to_unicode(st) try: # a fold is carriage return followed by either a space or a tab unfolded = UFOLD.sub("", st) diff --git a/src/icalendar/tests/test_encoding.py b/src/icalendar/tests/test_encoding.py index 4cc1557b0..3a89ca31f 100644 --- a/src/icalendar/tests/test_encoding.py +++ b/src/icalendar/tests/test_encoding.py @@ -1,7 +1,11 @@ import datetime +import sys import pytest +from icalendar import Calendar +from icalendar.parser import content_line + @pytest.mark.parametrize( ("field", "expected_value"), @@ -96,3 +100,113 @@ def test_apple_xlocation(calendars): """ for event in calendars.x_location.walk("vevent"): assert len(event.errors) == 0, "Got too many errors" + + +def test_charset_parameter_decodes_legacy_non_utf8_value_issue_2078( + charset_param_regex, +): + """A legacy CHARSET parameter declares the encoding of the property + value: latin-1 bytes decode with the declared charset instead of + coming back as U+FFFD replacement characters (issue #2078). + """ + ics = ( + b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//t//EN\r\n" + b"BEGIN:VEVENT\r\nUID:1\r\nDTSTART:20261108T190000Z\r\n" + b"SUMMARY;CHARSET=ISO-8859-1:Caf\xe9 Meeting\r\n" + b"END:VEVENT\r\nEND:VCALENDAR\r\n" + ) + event = Calendar.from_ical(ics).walk("VEVENT")[0] + assert str(event["SUMMARY"]) == "Caf\u00e9 Meeting" + + +def test_unknown_charset_falls_back_gracefully_issue_2078(charset_param_regex): + """An unknown CHARSET must not raise; the value degrades exactly as + it did before the CHARSET parameter was honored (issue #2078).""" + ics = ( + b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//t//EN\r\n" + b"BEGIN:VEVENT\r\nUID:1\r\n" + b"SUMMARY;CHARSET=BOGUS-XYZ:Caf\xe9 Meeting\r\n" + b"END:VEVENT\r\nEND:VCALENDAR\r\n" + ) + event = Calendar.from_ical(ics).walk("VEVENT")[0] + assert "Meeting" in str(event["SUMMARY"]) + + +@pytest.fixture(params=["py311", "py310"]) +def charset_param_regex(request, monkeypatch): + """Run with the CHARSET regex of Python 3.11+ and of Python 3.10. + + Python 3.10 has no possessive quantifiers, so it uses a simpler regex. + Both are tested where they can be compiled. + """ + if request.param == "py310": + monkeypatch.setattr( + content_line, + "CHARSET_PARAM_BYTES", + content_line._CHARSET_PARAM_BYTES_PY310, + ) + elif sys.version_info < (3, 11): + pytest.skip("Possessive quantifiers need Python 3.11 or later.") + + +def test_charset_text_inside_quoted_param_value_is_not_a_directive_issue_2078( + charset_param_regex, +): + """UTF-8 text with a quoted parameter value mentioning CHARSET= must + still decode as UTF-8, not as the encoding named in the quotes. + + RFC 5545 §3.1.4 states that there is no property parameter to declare + the charset of a value (iCalendar text is UTF-8), so CHARSET is not a + conforming parameter at all — an unquoted top-level ``CHARSET=`` is only + tolerated as a legacy marker of pre-RFC 5545 producers (issue #2078). + Inside a quoted parameter value it is plain data: RFC 5545 §3.1 defines + ``quoted-string = DQUOTE *QSAFE-CHAR DQUOTE`` with QSAFE-CHAR being any + character except CONTROL and DQUOTE, so ";" and "=" (and thus the text + ``;CHARSET=ISO-8859-1``) may occur inside the quotes without forming a + second parameter. + + The ``DESCRIPTION`` line makes the whole file invalid UTF-8, so the + per-line CHARSET scan runs: the ``SUMMARY`` line is valid UTF-8 but + must still not be decoded with the charset named inside its quoted + parameter value, while the real directive on ``DESCRIPTION`` is + honored. + """ + summary = "Caf\u00e9 Meeting" + description = "Caf\u00e9 R\u00e9sum\u00e9" # latin-1 bytes below + ics = ( + b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//t//EN\r\n" + b"BEGIN:VEVENT\r\nUID:1\r\n" + b'SUMMARY;ALTREP="cid:x;CHARSET=ISO-8859-1":' + + summary.encode("utf-8") + + b"\r\nDESCRIPTION;CHARSET=ISO-8859-1:" + + description.encode("iso-8859-1") + + b"\r\nEND:VEVENT\r\nEND:VCALENDAR\r\n" + ) + event = Calendar.from_ical(ics).walk("VEVENT")[0] + assert str(event["SUMMARY"]) == summary + assert str(event["DESCRIPTION"]) == description + + +def test_vestigial_charset_in_valid_utf8_does_not_override_issue_2078( + charset_param_regex, +): + """A file that is valid UTF-8 parses as UTF-8 even if it still + carries a vestigial legacy ``CHARSET`` parameter. + + Re-encoded migrations of legacy files can keep the old ``CHARSET`` + parameter while the bytes are already UTF-8. RFC 5545 §3.1.4 makes + UTF-8 the charset of a conforming stream, so valid UTF-8 input must + parse exactly as it did before this parameter was honored: the + declared charset must not turn the UTF-8 text into mojibake + (issue #2078). + """ + summary = "Caf\u00e9 Meeting" + ics = ( + b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//t//EN\r\n" + b"BEGIN:VEVENT\r\nUID:1\r\n" + b"SUMMARY;CHARSET=ISO-8859-1:" + + summary.encode("utf-8") + + b"\r\nEND:VEVENT\r\nEND:VCALENDAR\r\n" + ) + event = Calendar.from_ical(ics).walk("VEVENT")[0] + assert str(event["SUMMARY"]) == summary