Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions news/2078.bugfix
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Honor a legacy ``CHARSET`` property parameter when parsing bytes: a value that declares its encoding (e.g. ``SUMMARY;CHARSET=ISO-8859-1:...``) now decodes with that charset instead of coming back as U+FFFD replacement characters. I used AI to help with this change. @rahul05ranjan
61 changes: 59 additions & 2 deletions src/icalendar/parser/content_line.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""parsing and generation of content lines"""

import re
import sys

from icalendar.parser.parameter import Parameters, closes_quoted_section
from icalendar.parser.property import unescape_backslash, unescape_list_or_string
Expand All @@ -18,6 +19,27 @@
# to the first line break of a run, making a failed attempt O(1) instead of a
# full rescan, while matching exactly the same strings.
UFOLD = re.compile(r"(?:(?<!\n)\r\n|(?<![\r\n])\n)(?:\r?\n)*[ \t]")
NEWLINE_BYTES = re.compile(rb"\r?\n")
# a legacy CHARSET parameter declares the encoding of one property value.
# RFC 5545 §3.1.4 defines no property parameter to declare a charset
# ("There is not a property parameter to declare the charset used in a
# property value. The default charset for an iCalendar stream is UTF-8"),
# so this exists only to tolerate files from pre-RFC 5545 producers
# that kept the vCard-style CHARSET parameter (issue #2078).
# The character class excludes the delimiters, so both regexes below match
# the same strings; Python 3.10 has no possessive quantifiers, so it uses
# the simpler regex.
_CHARSET_PARAM_BYTES_PY310 = re.compile(
rb";[ \t]*CHARSET[ \t]*=[ \t]*([^;:\r\n]+)", re.IGNORECASE

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you please use the possessive quantifier.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Your are right, but one can detect the python version used and use possessive quantifiers when available.

)
if sys.version_info >= (3, 11):
# the possessive quantifier never gives matched characters back,
# keeping the match linear
CHARSET_PARAM_BYTES = re.compile(
rb";[ \t]*CHARSET[ \t]*=[ \t]*([^;:\r\n]++)", re.IGNORECASE
)
else:
CHARSET_PARAM_BYTES = _CHARSET_PARAM_BYTES_PY310
NEWLINE = re.compile(r"\r?\n")

OWS = " \t"
Expand Down Expand Up @@ -121,6 +143,39 @@ def flush_pending() -> None:
return "".join(out).strip()


def _decode_with_charset(value: bytes) -> str:
"""Decode content lines, honoring a legacy ``CHARSET`` parameter.

Input that is valid UTF-8 — a conforming :rfc:`5545` stream — is
decoded exactly as before this parameter was honored, so a vestigial
``CHARSET`` parameter in a file that actually uses UTF-8 cannot
override it. Only when the input is not valid UTF-8 as a whole is
each content line decoded with the charset its ``CHARSET`` parameter
declares, so a value that names its encoding does not come back as
U+FFFD. Lines without a ``CHARSET`` parameter (or with an unknown
one) keep the previous best-effort UTF-8 decoding.
"""
try:
return value.decode("utf-8-sig")
except UnicodeDecodeError:
pass
lines = []
for line in NEWLINE_BYTES.split(value):
# the parameters end at the first colon: a parameter VALUE quoting
# the text "CHARSET=..." must not select an encoding
head, _sep, _tail = line.partition(b":")
match = CHARSET_PARAM_BYTES.search(head)
if match:
charset = match.group(1).decode("ascii", "replace").strip().strip('"')
try:
lines.append(line.decode(charset))
continue
except (LookupError, UnicodeDecodeError):
pass
lines.append(to_unicode(line))
return "\r\n".join(lines)


class Contentline(str):
r"""A content line is a string that can be folded and parsed into parts.

Expand Down Expand Up @@ -318,7 +373,9 @@ def value_separator_index(self) -> int:
@classmethod
def from_ical(cls, ical, strict=False):
"""Unfold the content lines in an iCalendar into long content lines."""
ical = to_unicode(ical)
ical = (
_decode_with_charset(ical) if isinstance(ical, bytes) else to_unicode(ical)
)
# a fold is carriage return followed by either a space or a tab
return cls(UFOLD.sub("", ical), strict=strict)

Expand All @@ -342,7 +399,7 @@ def to_ical(self):
@classmethod
def from_ical(cls, st):
"""Parses a string into content lines."""
st = to_unicode(st)
st = _decode_with_charset(st) if isinstance(st, bytes) else to_unicode(st)
try:
# a fold is carriage return followed by either a space or a tab
unfolded = UFOLD.sub("", st)
Expand Down
114 changes: 114 additions & 0 deletions src/icalendar/tests/test_encoding.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
import datetime
import sys

import pytest

from icalendar import Calendar
from icalendar.parser import content_line


@pytest.mark.parametrize(
("field", "expected_value"),
Expand Down Expand Up @@ -96,3 +100,113 @@ def test_apple_xlocation(calendars):
"""
for event in calendars.x_location.walk("vevent"):
assert len(event.errors) == 0, "Got too many errors"


def test_charset_parameter_decodes_legacy_non_utf8_value_issue_2078(
charset_param_regex,
):
"""A legacy CHARSET parameter declares the encoding of the property
value: latin-1 bytes decode with the declared charset instead of
coming back as U+FFFD replacement characters (issue #2078).
"""
ics = (
b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//t//EN\r\n"
b"BEGIN:VEVENT\r\nUID:1\r\nDTSTART:20261108T190000Z\r\n"
b"SUMMARY;CHARSET=ISO-8859-1:Caf\xe9 Meeting\r\n"
b"END:VEVENT\r\nEND:VCALENDAR\r\n"
)
event = Calendar.from_ical(ics).walk("VEVENT")[0]
assert str(event["SUMMARY"]) == "Caf\u00e9 Meeting"


def test_unknown_charset_falls_back_gracefully_issue_2078(charset_param_regex):
"""An unknown CHARSET must not raise; the value degrades exactly as
it did before the CHARSET parameter was honored (issue #2078)."""
ics = (
b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//t//EN\r\n"
b"BEGIN:VEVENT\r\nUID:1\r\n"
b"SUMMARY;CHARSET=BOGUS-XYZ:Caf\xe9 Meeting\r\n"
b"END:VEVENT\r\nEND:VCALENDAR\r\n"
)
event = Calendar.from_ical(ics).walk("VEVENT")[0]
assert "Meeting" in str(event["SUMMARY"])


@pytest.fixture(params=["py311", "py310"])
def charset_param_regex(request, monkeypatch):
"""Run with the CHARSET regex of Python 3.11+ and of Python 3.10.

Python 3.10 has no possessive quantifiers, so it uses a simpler regex.
Both are tested where they can be compiled.
"""
if request.param == "py310":
monkeypatch.setattr(
content_line,
"CHARSET_PARAM_BYTES",
content_line._CHARSET_PARAM_BYTES_PY310,
)
elif sys.version_info < (3, 11):
pytest.skip("Possessive quantifiers need Python 3.11 or later.")


def test_charset_text_inside_quoted_param_value_is_not_a_directive_issue_2078(
charset_param_regex,
):
"""UTF-8 text with a quoted parameter value mentioning CHARSET= must
still decode as UTF-8, not as the encoding named in the quotes.

RFC 5545 §3.1.4 states that there is no property parameter to declare
the charset of a value (iCalendar text is UTF-8), so CHARSET is not a
conforming parameter at all — an unquoted top-level ``CHARSET=`` is only
tolerated as a legacy marker of pre-RFC 5545 producers (issue #2078).
Inside a quoted parameter value it is plain data: RFC 5545 §3.1 defines
``quoted-string = DQUOTE *QSAFE-CHAR DQUOTE`` with QSAFE-CHAR being any
character except CONTROL and DQUOTE, so ";" and "=" (and thus the text
``;CHARSET=ISO-8859-1``) may occur inside the quotes without forming a
second parameter.

The ``DESCRIPTION`` line makes the whole file invalid UTF-8, so the
per-line CHARSET scan runs: the ``SUMMARY`` line is valid UTF-8 but
must still not be decoded with the charset named inside its quoted
parameter value, while the real directive on ``DESCRIPTION`` is
honored.
"""
summary = "Caf\u00e9 Meeting"
description = "Caf\u00e9 R\u00e9sum\u00e9" # latin-1 bytes below
ics = (
b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//t//EN\r\n"
b"BEGIN:VEVENT\r\nUID:1\r\n"
b'SUMMARY;ALTREP="cid:x;CHARSET=ISO-8859-1":'
+ summary.encode("utf-8")
+ b"\r\nDESCRIPTION;CHARSET=ISO-8859-1:"
+ description.encode("iso-8859-1")
+ b"\r\nEND:VEVENT\r\nEND:VCALENDAR\r\n"
)
event = Calendar.from_ical(ics).walk("VEVENT")[0]
assert str(event["SUMMARY"]) == summary
assert str(event["DESCRIPTION"]) == description


def test_vestigial_charset_in_valid_utf8_does_not_override_issue_2078(
charset_param_regex,
):
"""A file that is valid UTF-8 parses as UTF-8 even if it still
carries a vestigial legacy ``CHARSET`` parameter.

Re-encoded migrations of legacy files can keep the old ``CHARSET``
parameter while the bytes are already UTF-8. RFC 5545 §3.1.4 makes
UTF-8 the charset of a conforming stream, so valid UTF-8 input must
parse exactly as it did before this parameter was honored: the
declared charset must not turn the UTF-8 text into mojibake
(issue #2078).
"""
summary = "Caf\u00e9 Meeting"
ics = (
b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//t//EN\r\n"
b"BEGIN:VEVENT\r\nUID:1\r\n"
b"SUMMARY;CHARSET=ISO-8859-1:"
+ summary.encode("utf-8")
+ b"\r\nEND:VEVENT\r\nEND:VCALENDAR\r\n"
)
event = Calendar.from_ical(ics).walk("VEVENT")[0]
assert str(event["SUMMARY"]) == summary
Loading