diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml new file mode 100644 index 00000000..20e6ae0c --- /dev/null +++ b/.github/workflows/audit.yml @@ -0,0 +1,43 @@ +--- +name: audit + +# Dependency vulnerability audit (pip-audit, see the `audit` env in tox.ini). +# +# This is deliberately *not* wired into the `tests` workflow: a new advisory +# can be published against an unchanged dependency tree, so the audit is +# time-triggered rather than change-triggered. The pull_request trigger is +# narrowed to the files that can change the dependency tree. +on: + pull_request: + paths: + - pyproject.toml + - tox.ini + - .github/workflows/audit.yml + workflow_dispatch: + schedule: + # Mondays 04:17 UTC, well clear of the nightly link check (22:03). + - cron: "17 4 * * 1" + +concurrency: + group: audit-${{ github.ref }} + cancel-in-progress: false + +jobs: + pip-audit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + # hatch-vcs derives the version from git tags; without them the + # project metadata pip-audit reads cannot be built. + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + - uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: pip|${{ hashFiles('pyproject.toml') }}|${{ hashFiles('tox.ini') }} + - run: pip install tox + - name: Audit dependencies for known vulnerabilities + run: tox -e audit diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml index 061798b9..c7ebd51d 100644 --- a/.github/workflows/linkcheck.yml +++ b/.github/workflows/linkcheck.yml @@ -7,6 +7,10 @@ on: schedule: - cron: "03 22 * * *" +concurrency: + group: linkcheck-${{ github.ref }} + cancel-in-progress: false + jobs: linkcheck: runs-on: ubuntu-latest @@ -14,6 +18,12 @@ jobs: issues: write steps: - uses: actions/checkout@v5 + - name: Restore lychee cache + uses: actions/cache@v4 + with: + path: .lycheecache + key: cache-lychee-${{ github.run_id }} + restore-keys: cache-lychee- - name: Check links with Lychee id: lychee uses: lycheeverse/lychee-action@v2 @@ -21,15 +31,33 @@ jobs: fail: false args: >- --root-dir "$(pwd)" - --timeout 20 - --max-retries 3 + --timeout 30 + --max-retries 6 + --retry-wait-time 2 --cache --max-cache-age 14d . - - name: Create Issue From File - if: steps.lychee.outputs.exit_code != 0 - uses: peter-evans/create-issue-from-file@v5 - with: - title: Link Checker Report - content-filepath: ./lychee/out.md - labels: report, automated issue + - name: Create or update Link Checker issue + if: steps.lychee.outputs.exit_code != 0 && github.ref == 'refs/heads/master' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Keep the oldest open report issue as canonical, fold duplicates in. + ISSUES=$(gh issue list --label "report" --state open --json number --jq '.[].number' | sort -n) + CANON=$(printf '%s\n' "$ISSUES" | head -1) + for n in $(printf '%s\n' "$ISSUES" | tail -n +2); do + gh issue close "$n" --comment "Duplicate of #${CANON} - auto-closed by the link checker." + done + if [ -n "$CANON" ]; then + gh issue edit "$CANON" --body-file ./lychee/out.md + else + gh issue create --title "Link Checker Report" --body-file ./lychee/out.md --label "report" --label "automated issue" + fi + - name: Close Link Checker issue if all links are healthy + if: steps.lychee.outputs.exit_code == 0 && github.ref == 'refs/heads/master' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + for n in $(gh issue list --label "report" --state open --json number --jq '.[].number'); do + gh issue close "$n" --comment "All links are now healthy." + done diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 778054ad..ca967543 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -97,7 +97,7 @@ jobs: - uses: actions/cache@v4 with: path: ~/.cache/pip - key: pip|${{ hashFiles('setup.py') }}|${{ hashFiles('tox.ini') }} + key: pip|${{ hashFiles('pyproject.toml') }}|${{ hashFiles('tox.ini') }} - run: pip install tox - name: Configure Baikal with pre-seeded database run: | @@ -326,7 +326,7 @@ jobs: - uses: actions/cache@v4 with: path: ~/.cache/pip - key: pip|${{ hashFiles('setup.py') }}|${{ hashFiles('tox.ini') }} + key: pip|${{ hashFiles('pyproject.toml') }}|${{ hashFiles('tox.ini') }} - run: pip install tox - run: tox -e docs style: @@ -339,7 +339,7 @@ jobs: - uses: actions/cache@v4 with: path: ~/.cache/pip - key: pip|${{ hashFiles('setup.py') }}|${{ hashFiles('tox.ini') }} + key: pip|${{ hashFiles('pyproject.toml') }}|${{ hashFiles('tox.ini') }} - uses: actions/cache@v4 with: path: ~/.cache/pre-commit @@ -356,7 +356,7 @@ jobs: - uses: actions/cache@v4 with: path: ~/.cache/pip - key: pip|${{ hashFiles('setup.py') }}|${{ hashFiles('tox.ini') }} + key: pip|${{ hashFiles('pyproject.toml') }}|${{ hashFiles('tox.ini') }} - run: pip install tox - run: tox -e deptry # The three async-* jobs below exist to test the async backend *selection* logic, diff --git a/.gitignore b/.gitignore index bbbc3491..8f2aceff 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,7 @@ tests/docker-test-servers/*/baikal-backup/ !tests/docker-test-servers/baikal/Specific/ # Local test server configuration (may contain credentials) tests/caldav_test_servers.yaml +# Lychee link checker cache +.lycheecache +# Scratch files from AI sessions (review notes, draft commit messages) +docs/design/tmp-* diff --git a/.lycheeignore b/.lycheeignore index 2a9c7c03..53f7e163 100644 --- a/.lycheeignore +++ b/.lycheeignore @@ -2,6 +2,7 @@ https?://your\.server\.example\.com/.* https?://.*\.example\.com(:\d+)?(/.*)?$ https?://domain/.* +https?://evil.attacker.com/caldav/ # Localhost URLs for test servers (not accessible in CI) http://localhost:\d+/.* @@ -17,6 +18,7 @@ https://caldav\.gmx\.net/.* https://caldav\.icloud\.com/.* https://p\d+-caldav\.icloud\.com/.* https://posteo\.de:\d+/.* +https://sync\.infomaniak\.com/.* https://purelymail\.com/.* https://webmail\.all-inkl\.com/.* https://www\.google\.com/calendar/dav/.* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e25a8425..a2489f42 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,21 +1,21 @@ --- repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.9.4 + rev: v0.15.20 hooks: - - id: ruff + - id: ruff-check args: [--fix] - id: ruff-format - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - - id: check-byte-order-marker + - id: fix-byte-order-marker - id: trailing-whitespace - id: end-of-file-fixer - repo: https://github.com/pycalendar/ai-prompt-auto-commit - rev: v0.0.5 + rev: v0.0.8 hooks: - id: unstage-ai-prompts - id: append-ai-prompts @@ -26,13 +26,13 @@ repos: stages: [manual] - repo: https://github.com/compilerla/conventional-pre-commit - rev: v3.4.0 + rev: v4.4.0 hooks: - id: conventional-pre-commit stages: [commit-msg] - repo: https://github.com/lycheeverse/lychee - rev: lychee-v0.24.1 + rev: lychee-v0.24.2 hooks: - id: lychee args: ["--no-progress", "--timeout", "10", "--exclude-path", ".lycheeignore", "--max-cache-age=30d", "--cache"] diff --git a/CHANGELOG.md b/CHANGELOG.md index 486aa3ed..301dcfaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,78 @@ Changelogs prior to v3.0 is pruned, but was available in the v3.1 release This project should adhere to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), though for pre-releases PEP 440 takes precedence. +## [Unreleased] + +### Added + +* New `write-delay` server-peculiarity in `compatibility_hints.py`: for servers that process writes asynchronously (a PUT/DELETE/MKCALENDAR/PROPPATCH/... returns before the change is queryable, so an immediate read-back 404s or returns stale data), a client must wait a bit after every write. This is the general, write-side counterpart of `search-cache` (which only delays searches). Configured as `{'behaviour': 'delay', 'delay': }`; the integration test suites (sync and async) honour it by sleeping after every write request. The Infomaniak profile now uses `write-delay` (10s) instead of its former `search-cache` delay, since the asynchronicity there is server-wide rather than search-specific. +* New compatibility flag `save-load.event.recurrences.exception.reschedule`: whether the server accepts re-anchoring a whole recurring event (moving the master `DTSTART`) while detached exceptions (`RECURRENCE-ID`) are attached. OX App Suite rejects this with `409 Conflict` even with a matching `If-Match` etag, although rescheduling an exception-free recurring event works there. `testEditSingleRecurrence` now gates its final `save(all_recurrences=True)` dtstart/dtend step on this flag. + +### Fixed + +* `compatibility_hints.py`: `testCheckCompatibility` had a blind spot — a sub-feature the server-tester explicitly probed whose *observed* status happened to equal the type default (e.g. a server-feature observed as `full`) was dropped from the compacted observed dict, and if its *declared* status was only inherited from a parent (not an explicit key) it was absent from the compacted expected dict too. Being in neither dict, it was never compared, so a real conflict went unreported. Concretely, Infomaniak declares `search.comp-type` `unsupported` while genuinely supporting the optional-comp-type behaviour (`search.comp-type.optional`), and the mismatch slipped through. The comparison is now a unit-tested `FeatureSet.compare()` method that also iterates the probed feature set, and the Infomaniak profile declares `search.comp-type.optional: full` explicitly. +* `jmap/client.py` and `jmap/async_client.py` `update_event()`: to honour RFC 8620 PatchObject merge semantics, the update null-injects every optional property absent from the new iCalendar so removed properties are actually cleared server-side. Some servers (observed with Stalwart) reject a property they do not support — e.g. `recurrenceRules`/`excludedRecurrenceRules` — as `invalidProperties` even when it is being set to `null`, which made every `update_event()` against such a server fail. Nulling an absent property is harmless cleanup, so the update now drops the server-rejected null-cleanup keys and retries (looping, since some servers report only one offending property per response) until the update succeeds. A rejection of a property the client actually assigned a value still surfaces as `JMAPMethodError`. +* `async_davclient.py` `_async_request()`: the issue-#158 connection-abort workaround sent a probe GET to detect the auth challenge; if the probe returned anything other than 401+WWW-Authenticate (e.g. a 200 HTML login page), the code fell through to `response = DAVResponse(probe_r, self)` — returning the probe response as if it were the original request's response, and silently swallowing the real connection error. Now the original exception is re-raised when the probe does not yield a challenge. +* `collection.py` `freebusy_request()`: for async clients, `add_attendee()` was called on each attendee before the `is_async_client` check dispatched to `_async_freebusy_request()`. For a `Principal` attendee on an async client, `get_vcal_address()` returns a coroutine; `add_attendee` then tried to set `.params` on the coroutine → AttributeError. `_async_save_with_invites` already awaited `get_vcal_address()` correctly. Fixed by passing attendees to `_async_freebusy_request()` and performing the await there. +* `search.py`: the documented `operator='=='` exact-match guarantee was never enforced — `post_filter` was not set to `True` for `==` searches, so the server's substring semantics leaked through. `icalendar_searcher.check_component()` already handles `==` as exact-match; the fix adds `==` to the `post_filter=True` trigger conditions. +* `async_davclient.py` `aio.get_calendars(calendar_name=...)`: name-based lookup iterated `self.get_calendars()` synchronously through the non-async `Principal.calendar(name=...)` path, which returns a coroutine for async clients; the loop body was iterating over the coroutine object, never the calendars, so name-based lookup returned nothing. Fixed by calling `await principal.get_calendars()` and filtering by display-name in the async path. +* `async_davclient.py` `get_calendars()`: lacked the GMX principal-URL fallback present in the sync client — when `calendar-home-set` was missing the async path returned `[]` immediately instead of falling back to the principal URL as calendar home. Parity restored. +* `collection.py` `Principal.calendar(cal_id=...)`: for async clients a bare (non-URL) `cal_id`/`name` raised `TypeError: argument of type 'coroutine' is not a container or iterable`, because the synchronous `calendar_home_set` property evaluated `"@" in ` without awaiting the async `get_property`. It now returns a coroutine that resolves the calendar home set (PROPFIND) before constructing the `Calendar`; a full-URL `cal_id`/`cal_url` needs no home set and still returns synchronously. The async integration tests' pre-test calendar cleanup relied on this call and swallowed the error in a bare `except`, so leftover calendars were never removed and a later MKCALENDAR `405 "resource already exists"` followed (seen against Infomaniak). Cleanup is now centralised in the `adelete_calendar_if_present()` test helper with a narrow `except`. +* `search.py`: the `undef` operator branch used `property.upper()` without the `category→CATEGORIES` alias mapping that the regular filter branch applies, so `add_property_filter('category', '', operator='undef')` queried the nonexistent `CATEGORY` property. `is-not-defined` on a nonexistent property matches every object, so the filter silently returned all events regardless of whether they had categories. +* `davclient.py` and `async_davclient.py`: rate-limit retry raised `TypeError: unsupported operand type(s) for +=: 'NoneType' and 'float'` on the second 429 response when the server provides no usable `Retry-After` value (`compute_sleep_seconds` returns `None`). The `sleep_seconds += rate_limit_time_slept / 2` line executed before the `sleep_seconds is None` guard. Now re-raise `RateLimitError` first, then update the sleep estimate. +* `davclient.py` `DAVClient.__init__()`: (a) `DAVClient(url='https://user@host/', password='secret')` crashed with `TypeError` — `unquote(self.url.password)` was called unconditionally when the URL had a username, but `self.url.password` is `None` when the URL contains no password. (b) URL-embedded credentials silently overrode explicit `username`/`password` kwargs; the async client already gave explicit kwargs higher precedence. Now: explicit kwargs win; URL credentials are only used as fallback when kwargs are absent. +* `config.py` `resolve_features()`: returning a named profile (`features='xandikos'`) returned the module-level dict object directly without copying it. Any code that then mutated the returned dict (e.g. `testing.py` patching `auto-connect.url.domain`) permanently corrupted the module-level dict for the whole process lifetime, so a second `DAVClient(features='xandikos')` would see the mutated domain. Similarly `testing.py` `XandikosServer`/`RadicaleServer` used a shallow `.copy()` — nested dict mutation still reached the module level. All three now use `copy.deepcopy()`. +* `config.py` `get_connection_params()`: explicit keyword arguments (e.g. `get_davclient(password='secret')`) were only respected when `url` or `features` was also present. When an env-var or config-file source was found instead, explicit params were silently dropped. Now the explicit params are merged (overlaid) on top of whatever lower-priority source wins. +* `calendarobjectresource.py` `_complete_recurring_safe()`: completing a recurring task passed the caller-supplied `completion_timestamp` to `_next()` correctly but then called `completed.complete()` without it, so the completed copy always recorded the current wall-clock time as `COMPLETED` regardless of the timestamp the caller specified. The async twin already passed `completion_timestamp` through; sync is now consistent. +* `calendarobjectresource.py` `_get_duration()`: `isinstance(i["DTSTART"], datetime)` tested the `vDDDTypes` wrapper object (which is never a `datetime`), so the date-vs-datetime branch always took the "is a date" path. A VTODO with a timed DTSTART and no DUE/DURATION got `duration = timedelta(days=1)` instead of `timedelta(0)`, shifting the next due date by one day when completing a recurring task. Fixed: test `isinstance(i["DTSTART"].dt, datetime)`. +* `jmap/client.py` and `jmap/async_client.py` `create_task()`: a JMAP server response that returned an empty `created` dict (with neither a `created` entry nor a `notCreated` entry for `"new-0"`) raised a bare `KeyError` instead of the documented `JMAPMethodError`. The `create_event()` method already had the required guard; `create_task()` was missing it in both sync and async clients. +* `lib/vcal.py` `fix()`: truncated iCalendar data (no `END:` line) triggered a bare `assert` which gave no useful message and was silently skipped under `python -O`. Now logs a warning and returns the data unchanged instead. +* `lib/vcal.py` `create_ical()`: when both `alarm_*` props and `ical_fragment` were supplied, the fragment was injected before the first `END:V` line — which is `END:VALARM`, placing e.g. an `RRULE` *inside* the alarm component. The regex now targets `END:V(EVENT|TODO|JOURNAL)` specifically. +* `lib/vcal.py` `fix()`: the backslash-unescape step used `('\"')` as a regex group, which matches only the literal two-character sequence `'"`. A backslash before a lone `'` or lone `"` was silently left in place. Fixed by using the character class `['\"]`. +* `lib/vcal.py` `fix()`: the trailing-whitespace fixup (`re.sub(" *$", "", fixed)`) lacked `re.MULTILINE`, so it only stripped trailing spaces at the very end of the document and never per-line. iCloud X-APPLE-STRUCTURED-LOCATION fold lines with trailing spaces were left intact, which can distort base64-encoded property values. Fixed by adding `re.MULTILINE`. +* `lib/error.py` `PYTHON_CALDAV_COMMDUMP`: when this debug env-var is set, a `logging.warning()` is now emitted at import time to remind the operator that request/response bodies and headers (including credentials and calendar PII) are being written to uniquely-named files under `/tmp` that accumulate indefinitely. +* `jmap/objects/calendar.py` `JMAPCalendar.search()`: `datetime` arguments for `start`/`end` were formatted with `datetime.isoformat()`, which produces `+HH:MM` offsets for aware non-UTC datetimes and no timezone indicator for naive datetimes. JMAP requires UTCDate format (`YYYY-MM-DDTHH:MM:SSZ`). Fixed by converting to UTC and using `strftime`. +* `jmap/client.py` and `jmap/async_client.py` `get_objects_by_sync_token()`: the `newState` from `CalendarEvent/changes` was discarded into `_`, so callers could not chain sync calls without a separate `get_sync_token()` round-trip — a race window where intervening changes would be silently missed. The method now returns a 4-tuple `(added, modified, deleted, new_sync_token)` instead of a 3-tuple. +* `compatibility_hints.py` `FeatureSet.copyFeatureSet()`: merging a plain-string feature value over an existing string-valued entry in the feature set raised a bare `AssertionError` — the `'support' not in server_node` guard blocked the update branch and fell through to `else: raise AssertionError`. Plain strings are the dominant style in the hint dicts, so any two-layer server config expressing the same feature crashed. Fixed by removing the `not in server_node` condition. +* `compatibility_hints.py` `FeatureSet.copyFeatureSet()`: an unknown feature name in a config file produced only a `UserWarning` but still stored the bad key in `_server_features`; a later `collapse()`/`is_supported()` call then raised a message-less `AssertionError` far from the original config. Fixed by `continue`-ing after the warning so unknown keys are never stored. +* `async_davclient.py`: HTML-on-401 diagnostic hint checked `self.headers` (the client's own request headers) for `Content-Type: text/html` instead of `r.headers`, so the intended "server returned an HTML login page, consider setting auth_type" message could never fire. +* `base_client.py` `get_calendars(calendar_urls=...)`: a calendar explicitly requested by URL was silently omitted from the result when its `displayname` property is the empty string `""`, because the check `if _try(calendar.get_display_name, ...)` was a truthiness test. The async counterpart already used `is not None`; sync is now consistent. +* `config.py` `expand_config_section()`: requesting a section name that is absent from the config raised `KeyError` instead of returning `[]`, causing plain `caldav.get_calendars()` to crash with `KeyError: 'default'` on configs with no `default` section. +* `config.py` `expand_config_section()`: `disable: true` was silently ignored for sections fetched by explicit name or via a `contains` list — the check used the string literal `"section"` as the config key instead of the `section` variable. Only the glob `"*"` path honoured `disable`. +* `lib/auth.py` `extract_auth_types()`: a `WWW-Authenticate` header ending with a trailing comma (seen in the wild) raised `IndexError` in the set comprehension because `h.split()` on an empty segment fails. Added an `if h.strip()` guard. +* `calendarobjectresource.py` `change_attendee_status()`: calling the method on an event with no `ATTENDEE` properties at all raised a bare `KeyError('ATTENDEE')` instead of the expected `NotFoundError`; the `try/except NotFoundError` wrapper in the `Principal` dispatch path could not catch it, so the "Principal is not invited" message was unreachable. Also, the genuine not-found error message contained a literal `%s` that was never substituted with the attendee address. +* `calendarobjectresource.py` `add_attendee()`: passing an attendee address with an uppercase or mixed-case URI scheme (`"MAILTO:user@example.com"`) raised `UnboundLocalError` — the scheme check used `str.startswith("mailto:")` which is case-sensitive, so the address fell through all branches without assigning `attendee_obj`. RFC 3986 §3.1 specifies URI schemes are case-insensitive. +* `response.py`: XML parser lacked `resolve_entities=False` and `no_network=True`, leaving entity expansion and DTD network fetches enabled by default. A malicious or MITM server could inject arbitrary text into parsed property values via inline DOCTYPE entity definitions. Fixed by adding `resolve_entities=False, no_network=True` to `etree.XMLParser`. +* `discovery.py` `discover_service()`: `require_tls=True` was not enforced on the well-known URI redirect target — a same-domain `Location: http://...` passed the domain-validation check and was returned as `ServiceInfo(tls=False)`, allowing a misconfigured or MITM server to silently downgrade the connection to plaintext. Fixed by checking `well_known_info.tls` against `require_tls` before returning the result. +* `datastate.py` `RawDataState.get_component_type()`: tested for the string `"BEGIN:FREEBUSY"` but real iCalendar data uses `"BEGIN:VFREEBUSY"`, so any `FreeBusy` object holding raw data returned `component_type=None` — making `is_loaded()` and `has_component()` return `False`, `save()` silently no-op at its early return, and `load(only_if_unloaded=True)` reload spuriously on every call. The same typo appeared in the base-class `get_uid()` and `get_component_type()` fallback parsers which looked for `comp.name == "FREEBUSY"` instead of `"VFREEBUSY"`. +* `calendarobjectresource.py` `_set_data()`: the raw-string branch cleared the legacy `_data`/`_vobject_instance`/`_icalendar_instance` attributes but never reset `self._state`. Once `_state` was populated by an earlier call to `_ensure_state()` (triggered by e.g. `.id` or `is_loaded()`), all subsequent reads via `get_data()`, `get_icalendar_instance()`, and `.id` served the pre-reload content even after `load()` fetched new data from the server. +* `search.py`: the `search.combined-is-logical-and: unsupported` workaround (triggered on e.g. Nextcloud) stripped all property filters from the server query to send only the time range, but passed `post_filter=None` (the ambient value) to `filter()` instead of `True`. `_filter_search_results` short-circuits when `post_filter` is falsy, so a search with both a time range and a property filter (e.g. `SUMMARY contains "foo"`) returned every object in the time range — the property filter was silently dropped. The sibling workarounds in the same function already used `post_filter=True`; this one now does too. +* `URL.canonical()`: two related bugs — (a) the canonical form was built from `self.url_parsed` (which still contains `user:pass@` in the netloc) rather than the auth-stripped URL, so `canonical()` leaked credentials into the returned URL and `__eq__`/`__hash__` comparisons between an authenticated client URL and a server-returned href (no credentials) were False; (b) when a URL had no auth part, `unauth()` returned `self` and `canonical()` then overwrote `url_raw`/`url_parsed` in place — a bare `==` or `hash()` call silently mutated the URL object, potentially re-encoding special characters (e.g. `+` → `%2B`) and causing subsequent requests to target the wrong resource. Fixed by using the auth-stripped URL's parsed form for `arr` and always returning a fresh `URL` object. +* `_post_put`: a 302 response to `PUT` always raised `IndexError` instead of following the redirect — iterating the headers dict yields key strings, not tuples, so `x[0]` was the first character of each header name, never `"location"`. Fixed by using `r.headers.get("location")`. +* `vcal.fix()`: the `COMPLETED` date-to-datetime regex consumed the trailing newline, merging the following iCal property into the `COMPLETED` value on every inbound object from a server that stores `COMPLETED` as a plain date (e.g. SOGo). Fixed by using a lookahead `(?=\s)` instead of consuming `\s`. + +* Time-range searches without a component type (`search(start=..., end=...)` with no `event`/`todo`/`journal`/`comp_class`) crashed against SabreDAV-based servers (Baikal, Nextcloud, ...) with `ReportError`: *"You cannot add time-range filters on the VCALENDAR component"*. A `CALDAV:time-range` is only valid inside a `VEVENT`/`VTODO`/`VJOURNAL`/`VFREEBUSY`/`VALARM` comp-filter (RFC4791 section 9.7), never directly under `VCALENDAR`. The library now splits such a search into one query per component type, and additionally recovers from the server rejection at runtime if it occurs anyway. See https://github.com/python-caldav/caldav/issues/681 +* Property-filter searches without a component type (e.g. `search(category=...)` or other attribute filters with no `event`/`todo`/`journal`/`comp_class`) silently returned nothing on most servers (Xandikos, SabreDAV, ...): the prop-filter landed under the `VCALENDAR` comp-filter, which has no component properties like `CATEGORIES` to match. The library now splits such a search into one query per component type as well (`search.text.comp-type-optional`). See https://github.com/python-caldav/caldav/issues/681 +* `search()`'s generator driver now feeds exceptions raised while executing a request back into the search logic, so the server-compatibility fallbacks and per-object load error handling actually take effect (previously dead code). Applies to both the sync and async code paths. +* `compatibility_hints`: OX was pinned to `create-calendar.set-displayname: unsupported` (a value masked by a checker bug that verified the feature by display-name lookup, which a leftover/colliding calendar would shadow); OX stores the display name as a property separate from the calendar URL and honours it at creation time, so the expectation is corrected to `full`. +* `compatibility_hints`: Stalwart's `search.recurrences.expanded.exception` was inheriting the default `full`, but Stalwart's server-side `CALDAV:expand` only suppresses the exception-overridden occurrence when `SEQUENCE` is absent. With `SEQUENCE` present (as real-world clients always emit) it returns both the original occurrence and the override, so the expectation is corrected to `fragile`. +* Config file sections with `features` but no `caldav_url` were rejected, even though the URL can be derived from the `auto-connect.url` compatibility hints. Explicitly passed parameters already worked this way; now `get_davclient(config_section=...)` and friends behave consistently. +* `jmap/convert/jscal_to_ical.py`: a `recurrenceOverrides` entry that does not include a `"start"` key (the common case — title-only change, description update, etc.) produced a child `VEVENT` with `DTSTART` copied from the master event's start time rather than from the override key. This effectively relocated every non-rescheduled override to the master's first occurrence, breaking all override display. Default is now the override key itself. +* `jmap/convert/jscal_to_ical.py`: `EXDATE` and `RECURRENCE-ID` values were always emitted as floating (timezone-less) `DATE-TIME` regardless of the event's `timeZone` or `showWithoutTime` flag. Per RFC 5545 §3.8.5.1 the value type must match `DTSTART`; a floating `EXDATE` on a `TZID`-anchored event does not match any instance, so excluded occurrences reappear. Override keys are now parsed with the event timezone applied (`TZID`-anchored events) or as `date` objects (all-day events). +* `jmap/convert/_utils.py` `_format_local_dt()`: UTC datetimes produced a `Z`-suffixed string. RFC 8984 §1.4 defines `LocalDateTime` (the type required for `recurrenceOverrides` keys and `recurrenceRules.until`) as a bare `YYYY-MM-DDThh:mm:ss` without any suffix; `Z`-suffixed override keys cannot match `LocalDateTime` occurrence keys, causing mismatches on strict servers. The function now always returns a timezone-stripped local representation. +* `jmap/convert/ical_to_jscal.py` and `jmap/convert/jscal_to_ical.py`: the `STATUS` property was silently dropped in both conversion directions. `STATUS:CANCELLED` round-tripped as `status: confirmed` (JSCalendar default), so cancelled meetings appeared active. Mappings `CONFIRMED ↔ confirmed`, `TENTATIVE ↔ tentative`, `CANCELLED ↔ cancelled` are now implemented. +* `jmap/client.py` and `jmap/async_client.py` `update_event()`: RFC 8620 §3.3 specifies that absent keys in a PatchObject preserve the server value. `update_event` sent the full converted JSCalendar dict as the patch; properties the caller removed (e.g. LOCATION, VALARM) were absent from the patch and therefore silently persisted on the server. `update_event` now explicitly sets all optional top-level JSCalendar properties to `null` when they are absent from the conversion result, ensuring the server removes them. + +### Added + +* `caldav.config.extract_conn_params_from_section` is now public API (renamed from `_extract_conn_params_from_section`), so that downstream tools like plann can map plann-style config sections (`caldav_url`, `caldav_user`, `features`, etc.) to `DAVClient` parameters without duplicating the logic. +* New compatibility feature `create-calendar.stable-url` (default `full`): whether a calendar, once created, remains addressable at the URL derived from the requested `cal_id`. Some servers assign a different *canonical* URL: Zimbra relocates the collection to a display-name-derived path when a display name is set (a collection alias lingers at the `cal_id` and answers `PROPFIND`/`REPORT`, but a `GET` on a child object under it 404s, so the `cal_id` is not a usable address); OX always exposes an opaque `cal://0/NNN` (base64-segment) canonical URL. Both are marked `create-calendar.stable-url: unsupported`. For such servers `Calendar._create()` now discovers and adopts the canonical URL after creation (re-pointing `self.url`) instead of dropping the display name, so the calendar keeps its name *and* every later URL-based operation resolves — identical handling for Zimbra and OX. +* New `compatibility_workarounds` parameter on `Calendar.search()` / `CalDAVSearcher.search()` / `async_search()`. When `False`, all server-compatibility workarounds are disabled and the query is sent verbatim (a single REPORT, no comp-type splitting, no filter rewriting, no fallback retries). Mainly for the server-compatibility checker, to observe raw server behaviour. + +### Changed + +* `compatibility_hints`: eight directly-probed feature *nodes* that also have refinement sub-features now carry their own explicit `default` (`get-current-user-principal`, `create-calendar.set-displayname`, `delete-calendar`, `save-load.todo.recurrences`, `search.text.category`, `search.recurrences.includes-implicit.todo`, `scheduling`, `sync-token`). This marks them as *independent* features: `is_supported()` and `collapse()` no longer derive/fold them away from their children, so e.g. `sync-token` stays `full` even when `sync-token.delete` is `unsupported`. Each such node has a corresponding check in the server-tester (`search.comp-type` gained one); `principal-search` deliberately keeps no default since it is a genuine OR-grouping of its sub-searches. + ## [3.2.1] - 2026-05-28 The changeset in 3.2.1 is predominently added async integration tests. Those tests should now be replicating all the logic in the good old sync integration tests under `test_caldav.py`. Some few more bugs were found while adding those tests. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6110246c..3e33f20a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,7 +20,7 @@ The types used should (as for now) be one of: The `compatibility_hints.py` has been moved from the test directory to the codebase not so very long ago. Some special rules here: * Adjusting the feature set for some calendar server? Check if there exists some workarounds etc in the code for said feature, if so, then it should be considered a fix or a feature. Perhaps even a breaking change. Otherwise, use `test: ...`. (because it is relevant for the compatibility test, if nothing else). -* Adding a new feature hint? Ensure it's covered by the caldav-server-tester. Since we have a compatibility test, it will be relevant for the test - so use `test: (...)`. It should be covered by the caldav-serveer-tester, so refer to some issue or pull request for the caldav-server-tester in the commit message. +* Adding a new feature hint? Ensure it's covered by the caldav-server-tester. Since we have a compatibility test, it will be relevant for the test - so use `test: (...)`. It should be covered by the caldav-server-tester. * Changing some descriptions? That goes as `docs: ...` even if it's actually changing a variable in the code. This is not set in stone. If you feel strongly for using something else, use something else in the commit message and update this file in the same commit. diff --git a/SECURITY.md b/SECURITY.md index 0a1a63aa..1da5501e 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,48 +1,84 @@ # Security policy -Issues should be fixed ASAP, and information on any security issue should be published as soon as it's fixed. Use the GitHub issue tracker or check up [CONTACT](CONTACT.md) or even the [CODE OF CONDUCT](CODE_OF_CONDUCT) file to get in touch with the maintainer. +Issues should be fixed ASAP, and information on any security issue should be published as soon as it's fixed. Serious issues should be reported privately and kept under wraps until a fix is released — use [GitHub's private vulnerability reporting](https://github.com/python-caldav/caldav/security/advisories/new) (the "Report a vulnerability" button on the Security tab), or get in touch with the maintainer via [CONTACT](CONTACT.md) or the [CODE OF CONDUCT](CODE_OF_CONDUCT) file. Use the public GitHub issue tracker only for non-sensitive issues. There are no "LTS"-releases of the CalDAV package, but the maintainer will always consider backporting security fixes if it's deemed relevant. The maintainer is doing most of the maintenance on hobby-basis and may have other things in life preventing him from dealing with issues on the go, so no guarantees are given. -All contributions are carefully reviewed by the maintainer, and all releases are carefully tested and tagged with a PGP-signed commit. +All contributions are carefully reviewed by Tobias Brox, and AI-tools are used for code reviews prior to each release. All releases are carefully tested and tagged with a PGP-signed commit. # Known security issues and risks ## RFC6764 -I do see a major security flaw with the RFC6764 discovery. If the DNS is not to be trusted, someone can highjack the connection by spoofing the service records, and also spoofing the TLS setting, encouraging the client to connect over plain-text HTTP without certificate validation. Utilizing this it may be possible to steal the credentials. This flaw can be mitigated by using DNSSEC, but DNSSEC is not widely used, and fixing support for DNSSEC validation in the CalDAV library was found to be non-trivial (perhaps I'll look into it again some time after 3.0 has been released). This has been mitigated by adding a require_tls` connection parameter that is True by default, plus by ensuring one isn't routed to a different domain. +**Summary**: auto-discovery of the CalDAV-URL seems to be insecure by design, anyone controlling your local resolver (or upstream resolvers) may try to fish out username and password. -## DDoS/OOM risk +**Mitigation**: Leave `require_tls`, `ssl_verify_cert` to the default - or better still: use a URL rather than a domain when configuring the library. + +RFC6764 discovery depends on correct DNS-lookups, but DNS is not to be trusted. The proper solution is DNSSEC, unfortunately DNSSEC is not widely used, and fixing support for DNSSEC validation in the CalDAV library was found to be non-trivial (some work has been done, but it was found to be too difficult - the pull request is stalled as for now). + +Connections can be hijacked if someone spoofs the service records. This has been partly mitigated by adding a `require_tls` connection parameter that is `True` by default, cert validation, and ensuring one isn't routed to a different domain. + +Auto-discovery and HTTP redirects also mean the host you end up talking to may not be the one you configured. If you run the library in a context where it can reach internal/private network resources (server-side request forgery, SSRF), be aware that a malicious DNS resolver or a malicious/compromised server can steer requests towards other hosts. + +## DDoS/OOM risk - recurring events/tasks search + +**Summary:** If you allow untrusted parties to specify search-terms towards a calendar containing recurring events/tasks, bad things may happen. The package offers both client-side and server-side expansion of recurring events and tasks. It currently does not offer expansion for open-ended date searches - but with a large enough timespan and a frequent enough RRULE, there may be millions of recurrences returned. Those recurrences are returned as a generator, so things will not break down immediately. However, there is no guaranteed sort order of the recurrences ... and once you add sorting parameters to the search, bad things may happen. +## XML parsing + +**Summary:** XML responses are parsed defensively by default; only relax this against servers you trust. + +The library parses XML responses from the server using lxml. By default the parser is configured to resist common XML attacks: external entity resolution is disabled (`resolve_entities=False`) and network access during parsing is blocked (`no_network=True`), guarding against XXE (XML External Entity) attacks, and lxml's built-in limits protect against oversized "billion laughs"-style entity-expansion payloads. + +The `huge_tree` connection option (default off) disables lxml's built-in parser limits so that very large calendar objects can be handled. With `huge_tree` enabled, a malicious or compromised server can exhaust available memory with a crafted XML payload — only enable it against servers you trust. See the [lxml XMLParser documentation](https://lxml.de/api/lxml.etree.XMLParser-class.html). + ## Bugs causing weird things happening -Weird things may happen due to bugs both on in the CalDAV package, on your side and on the server side. Here are some weird experiences with Zimbra: +**Summary:** Always expect the unexpected -* I have experiences that cancelling participation in an event caused the event to be cancelled for all participants (even if the person deciding to not go to the event was not an organizer and should have no permissions to edit the event). Clearly a server-side issue. -* I once tried to restore from backup and push ten years of ical code to the calendar server. The calendar server responded by re-inviting people to the meetings we had ten years ago. I'm inclined to call that also a server side bug. -* Many other things may happen. +Weird things may happen due to bugs both on in the CalDAV package, on your side and on the server side. Some anecdotes from using Zimbra: -## Malicious usage +* I once tried to restore from backup and push ten years of ical code to the calendar server. The calendar server responded by re-inviting people to the meetings we had ten years ago. I'm inclined to call that a server side bug - but it also highlights the risk of using the CalDAV library for doing operations that ordinary calendaring clients aren't doing. +* It's been observed that cancelling participation in an event caused the event to be cancelled for all participants (even if the person deciding to not go to the event was not an organizer and should have no permissions to edit the event). Clearly a server-side issue. -Beware of risks and exposure when creating applications: +## Other things to consider -* Your code may handle username and password, be careful not to expose such credentials. Even the URL to the calendar server and/or calendar may be something people want to keep private. +**Summary:** Beware of risks and exposure when creating applications: + +* Your code may handle username and password, be careful not to expose such credentials. Even the URL to the calendar server and/or calendar may be something people want to keep private. The library includes code for reading this data from a standard config file - please use it rather than reinventing the wheel or hard-coding credentials directly into your code. * Consider that calendar events and such is personal data, which deserves protection. In the EU with the GDPR, such protection is even mandated by law. * If you allow arbitrary people to create calendar content to be saved to a server, there may be some risks involved: * Depending on the server implementation, it may be possible to use the caldav library for sending spam emails. * Be aware of DoS-attacks: By storing too much / too big / specially crafted icalendar data, the server and/or client may crash or consume all available resources. - * If allowing anonymous parties to save and retrieve data from your server, you may end up with responsibility for spreading illicit information. This may include things like child porn. Political or religious propaganda may be legitimate and legal in some countries, but may involve death penalty in other countries. Your calendar server may also be used for coordinating criminal activity. -* If you allow arbitrary people to fetch calendar content from the server, there may also be some risks involved - in particular, a DoS-attack by requesting a large time span of expanded events. + * If allowing anonymous parties to save and retrieve data from your server, you may end up with responsibility for spreading illicit information. This may include things like child sexual abuse material. Political or religious propaganda may be legitimate and legal in some countries, but may involve death penalty in other countries. Your calendar server may also be used for coordinating criminal activity. +* If you allow arbitrary people to fetch calendar content from the server, there may also be some risks involved - see the separate section on DoS-attack by requesting a large time span of expanded events. + +## Supply attack risk -## Malicious code +**Summary:** Stick to released versions and check the PGP signature in the release-tag -All code contributions are carefully reviewed by Tobias Brox. Version tags are signed with PGP. Of course there is always a risk that someone takes over my PGP key and github access (It's hard to be immune against a [5$ wrench attack](https://xkcd.com/538/)). The original owner of the repository is still alive and may take over the project again should something happen to me. I would anyway encourage using AI to do risk assessments. +All code contributions are carefully reviewed by Tobias Brox. Version tags are signed with PGP. Of course there is always a risk that someone takes over my PGP key and GitHub access (It's hard to be immune against a [$5 wrench attack](https://xkcd.com/538/)). The original owner of the repository is still alive and may take over the project again should something happen to me. I would encourage using AI to do risk assessments. The library comes with a number of dependencies, one may need to evaluate the security of those too. The pyproject contains the current list. Some notes: * niquests is an optional dependency - you may replace it with requests if you don't trust niquests -* recurring-ical-events and icalendar both has the same maintainer (Nicco Kunzmann). He is considered trustworthy. -* Tobias now has a policy of moving code not related to CalDAV into separate packages. Packages under the `python-caldav` ownership on GitHub should be considered to be of the same quality and security level as the CalDAV library. -* No security review have been done of the other dependencies. +* recurring-ical-events and icalendar both have the same maintainer (Nicco Kunzmann). He is considered trustworthy. +* Tobias now has a policy of moving code not related to CalDAV into separate packages. Those packages are most of the time either published under the `python-caldav` or `pycalendar` ownership on GitHub, and should be considered to be of the same quality and security level as the CalDAV library. +* No independent security review has been done of the other dependencies - those are all considered to be mature and robust projects. + +## Communication dumper debug hook + +**Summary:** If someone has the ability to both alter the environment and full read access to /tmp (basically, someone has root access to the computer where the code is run), it will be possible to get access to all communication. Also, anyone using this debug hook must take responsibility of deleting the dumped files. + +**Mitigation:** If this worries you, set `caldav.lib.error.debug_dump_communication=False` after importing caldav. + +The following was written when `PYTHON_CALDAV_COMMDUMP` was introduced in v1.4.0: + +* An attacker that has access to alter the environment the application is running under may cause a DoS-attack, filling up available disk space with debug logging. +* An attacker that has access to alter the environment the application is running under, and access to read files under /tmp (files being 0600 and owned by the uid the application is running under), will be able to read the communication between the server and the client, communication that may be private and confidential. + +Thinking it through three times, I'm not too concerned — if someone has access to alter the environment the process is running under and access to read files run by the uid of the application, then this someone should already be trusted and will probably have the possibility to DoS the system or gather this communication through other means. + +As of v3.3 (to be released towards the end of 2026-06), a warning is logged at import time when this variable is set, reminding the operator that request/response bodies and headers (including credentials and calendar PII) are written to uniquely-named files under `/tmp` that accumulate indefinitely. diff --git a/caldav/async_davclient.py b/caldav/async_davclient.py index 2c4bd006..bd2f877e 100644 --- a/caldav/async_davclient.py +++ b/caldav/async_davclient.py @@ -163,6 +163,9 @@ def __init__( features: FeatureSet for server compatibility workarounds. enable_rfc6764: Enable RFC6764 DNS-based service discovery. require_tls: Require TLS for discovered services (security consideration). + Only gates the RFC6764 discovery path; it does NOT reject an + explicitly-passed http:// URL. Global enforcement is deferred to + 4.0 — see https://github.com/python-caldav/caldav/issues/687 rate_limit_handle: When True, automatically sleep and retry on 429/503 responses. When None (default), auto-detected from server features. When False, raise RateLimitError immediately. @@ -258,19 +261,9 @@ def __init__( } self.headers.update(headers) - rate_limit = self.features.is_supported("rate-limit", dict) - if rate_limit_handle is None: - if rate_limit and rate_limit.get("enable"): - rate_limit_handle = True - if "default_sleep" in rate_limit: - rate_limit_default_sleep = rate_limit["default_sleep"] - if "max_sleep" in rate_limit: - rate_limit_max_sleep = rate_limit["max_sleep"] - else: - rate_limit_handle = False - self.rate_limit_handle = rate_limit_handle - self.rate_limit_default_sleep = rate_limit_default_sleep - self.rate_limit_max_sleep = rate_limit_max_sleep + self._init_rate_limit_config( + rate_limit_handle, rate_limit_default_sleep, rate_limit_max_sleep + ) def _create_session(self) -> None: """Create or recreate the async HTTP client with current settings.""" @@ -365,20 +358,7 @@ async def request( try: return await self._async_request(url, method, body, headers) except error.RateLimitError as e: - if not self.rate_limit_handle: - raise - sleep_seconds = error.compute_sleep_seconds( - e.retry_after_seconds, - self.rate_limit_default_sleep, - self.rate_limit_max_sleep, - ) - if rate_limit_time_slept: - sleep_seconds += rate_limit_time_slept / 2 - if sleep_seconds is None or ( - self.rate_limit_max_sleep is not None - and rate_limit_time_slept > self.rate_limit_max_sleep - ): - raise + sleep_seconds = self._rate_limit_sleep_seconds(e, rate_limit_time_slept) await asyncio.sleep(sleep_seconds) return await self.request( url, method, body, headers, rate_limit_time_slept + sleep_seconds @@ -432,7 +412,7 @@ async def _async_request( log.debug(f"server responded with {r.status_code} {reason}") if ( r.status_code == 401 - and "text/html" in self.headers.get("Content-Type", "") + and "text/html" in r.headers.get("Content-Type", "") and not self.auth ): msg = ( @@ -484,7 +464,11 @@ async def _async_request( # Retry original request with auth request_kwargs["auth"] = self.auth r = await self.session.request(**request_kwargs) - response = DAVResponse(r, self) + response = DAVResponse(r, self) + else: + # Probe GET did not give us a 401+WWW-Authenticate challenge — + # auth negotiation failed; re-raise the original connection error + raise # Handle 429/503 rate-limit responses error.raise_if_rate_limited(r.status_code, str(url_obj), r.headers.get("Retry-After")) @@ -936,14 +920,6 @@ async def get_calendars(self, principal: Optional["Principal"] = None) -> list[" for cal in calendars: print(f"Calendar: {cal.get_display_name()}") """ - from caldav.collection import Calendar - from caldav.collection import ( - _extract_calendar_home_set_from_results as extract_home_set, - ) - from caldav.collection import ( - _extract_calendars_from_propfind_results as extract_calendars, - ) - if principal is None: principal = await self.get_principal() @@ -953,12 +929,7 @@ async def get_calendars(self, principal: Optional["Principal"] = None) -> list[" props=self.CALENDAR_HOME_SET_PROPS, depth=0, ) - calendar_home_url = extract_home_set(response.results) - if not calendar_home_url: - return [] - - # Make URL absolute if relative - calendar_home_url = self._make_absolute_url(calendar_home_url) + calendar_home_url = self._calendar_home_url(response, principal) # Fetch calendars via PROPFIND response = await self.propfind( @@ -967,14 +938,7 @@ async def get_calendars(self, principal: Optional["Principal"] = None) -> list[" depth=1, ) - # Process results using shared helper - calendar_infos = extract_calendars(response.results) - - # Convert CalendarInfo objects to Calendar objects - return [ - Calendar(client=self, url=info.url, name=info.name, id=info.cal_id) - for info in calendar_infos - ] + return self._build_calendars_from_propfind(response) async def search_calendar( self, @@ -1222,7 +1186,11 @@ async def get_calendars( for cal in calendars: print(await cal.get_display_name()) """ - from caldav.base_client import CalendarCollection, _normalize_to_list + from caldav.base_client import ( + CalendarCollection, + _normalize_to_list, + _warn_unreadable_display_name, + ) def _try(coro_result, errmsg): """Handle errors based on raise_errors flag.""" @@ -1267,13 +1235,31 @@ def _try(coro_result, errmsg): raise # Fetch specific calendars by name - for cal_name in calendar_names: + if calendar_names: try: - calendar = await principal.calendar(name=cal_name) - if calendar: - calendars.append(calendar) + all_cals_for_name = await principal.get_calendars() + for cal_name in calendar_names: + for cal in all_cals_for_name: + try: + display_name = await cal.get_display_name() + if display_name == cal_name: + calendars.append(cal) + break + except Exception as e: + # Skip calendars whose display name can't be read; warn + # only when the failure is unexpected (see helper). + # Continuing ensures one unreadable calendar doesn't abort + # the whole name lookup. + _warn_unreadable_display_name(client, cal, cal_name, e) + continue + else: + log.error(f"No calendar with name '{cal_name}' found") + if raise_errors: + raise error.NotFoundError(f"No calendar with name '{cal_name}' found") + except error.NotFoundError: + raise except Exception as e: - log.error(f"Problems fetching calendar by name '{cal_name}': {e}") + log.error(f"Problems fetching calendars by name: {e}") if raise_errors: raise diff --git a/caldav/base_client.py b/caldav/base_client.py index d1974bb7..5186a3c7 100644 --- a/caldav/base_client.py +++ b/caldav/base_client.py @@ -255,6 +255,99 @@ def _raise_authorization_error(self, url_str: str, reason_source: Any) -> NoRetu reason = "None given" raise error.AuthorizationError(url=url_str, reason=reason) + # ── Rate-limit handling ───────────────────────────────────────────────── + # Shared by the sync (DAVClient) and async (AsyncDAVClient) __init__ and + # request() retry loops, which are otherwise byte-identical apart from + # time.sleep vs asyncio.sleep. + + def _init_rate_limit_config( + self, + rate_limit_handle: bool | None, + rate_limit_default_sleep: int | None, + rate_limit_max_sleep: int | None, + ) -> None: + """Resolve and store rate-limit settings on self. + + When ``rate_limit_handle`` is None it is auto-detected from the + ``rate-limit`` feature; an enabled feature may also supply default and + max sleep durations (explicit constructor arguments are not overridden + because the feature values only fill in the auto-detected branch). + """ + rate_limit = self.features.is_supported("rate-limit", dict) + if rate_limit_handle is None: + if rate_limit and rate_limit.get("enable"): + rate_limit_handle = True + if "default_sleep" in rate_limit: + rate_limit_default_sleep = rate_limit["default_sleep"] + if "max_sleep" in rate_limit: + rate_limit_max_sleep = rate_limit["max_sleep"] + else: + rate_limit_handle = False + self.rate_limit_handle = rate_limit_handle + self.rate_limit_default_sleep = rate_limit_default_sleep + self.rate_limit_max_sleep = rate_limit_max_sleep + + def _rate_limit_sleep_seconds( + self, + exc: error.RateLimitError, + rate_limit_time_slept: float, + ) -> float: + """Decide how long to sleep before retrying a rate-limited request. + + Returns the sleep duration in seconds. Re-raises ``exc`` when no retry + should happen (rate-limit handling disabled, no usable duration, or the + accumulated sleep already exceeds ``rate_limit_max_sleep``). The caller + is responsible for actually sleeping (time.sleep vs asyncio.sleep) and + retrying with ``rate_limit_time_slept + ``. + """ + if not self.rate_limit_handle: + raise exc + sleep_seconds = error.compute_sleep_seconds( + exc.retry_after_seconds, + self.rate_limit_default_sleep, + self.rate_limit_max_sleep, + ) + if sleep_seconds is None or ( + self.rate_limit_max_sleep is not None + and rate_limit_time_slept > self.rate_limit_max_sleep + ): + raise exc + if rate_limit_time_slept: + sleep_seconds += rate_limit_time_slept / 2 + return sleep_seconds + + # ── Calendar discovery post-processing ────────────────────────────────── + # Pure result-handling shared by sync/async get_calendars; only the two + # awaited PROPFIND calls and the principal lookup differ between the twins. + + def _calendar_home_url(self, home_set_response: Any, principal: Any) -> str: + """Extract the calendar-home-set URL from a PROPFIND response. + + Falls back to the principal URL when the server does not advertise a + calendar-home-set (e.g. GMX), then makes the result absolute. + """ + from caldav.collection import ( + _extract_calendar_home_set_from_results as extract_home_set, + ) + + calendar_home_url = extract_home_set(home_set_response.results) + if not calendar_home_url: + calendar_home_url = str(principal.url) + return self._make_absolute_url(calendar_home_url) + + def _build_calendars_from_propfind(self, list_response: Any) -> list: + """Build Calendar objects from a calendar-home PROPFIND response.""" + from caldav.collection import Calendar + from caldav.collection import ( + _extract_calendars_from_propfind_results as extract_calendars, + ) + + calendar_infos = extract_calendars(list_response.results) + return [ + Calendar(client=self, url=info.url, name=info.name, id=info.cal_id) + for info in calendar_infos + ] + # ── XML builders ────────────────────────────────────────────────────────── # All methods are static: no I/O, no server interaction, pure data # transformation. Both DAVClient and AsyncDAVClient inherit these so @@ -645,6 +738,32 @@ def _normalize_to_list(obj: Any) -> list: return list(obj) +def _warn_unreadable_display_name(client: Any, calendar: Any, name: Any, exc: Exception) -> None: + """Log a warning when a calendar's display name couldn't be read during a + lookup by name -- unless the failure is expected per the compatibility matrix. + + Shared by the sync (:meth:`caldav.collection.CalendarSet.calendar`) and async + (:func:`caldav.async_davclient.get_calendars`) name-matching loops so the + warn-or-suppress decision lives in one place. + + The failure is treated as expected (and silently skipped) only when we + positively know the server doesn't support reading the DAV:displayname + property via PROPFIND (``propfind.displayname`` non-supported -- which falls + back to the ``propfind`` parent when not probed explicitly). When the + feature is supported, or when we have no feature matrix to consult, the + failure is unexpected and is warned about. + + The caller is responsible for continuing the loop afterwards, so that one + unreadable calendar never aborts the whole name lookup. + """ + features = getattr(client, "features", None) + if features is None or features.is_supported("propfind.displayname"): + log.warning( + f"Could not read display name for calendar " + f"{getattr(calendar, 'url', calendar)} while matching name '{name}': {exc}" + ) + + def _fetch_calendars_for_client( client: Any, calendar_url: Any | None, @@ -686,7 +805,7 @@ def _try(meth, kwargs, errmsg): calendar = principal.calendar(cal_url=cal_url) else: calendar = principal.calendar(cal_id=cal_url) - if _try(calendar.get_display_name, {}, f"calendar {cal_url}"): + if _try(calendar.get_display_name, {}, f"calendar {cal_url}") is not None: calendars.append(calendar) for cal_name in calendar_names: diff --git a/caldav/calendarobjectresource.py b/caldav/calendarobjectresource.py index a0f33976..9362a7e9 100644 --- a/caldav/calendarobjectresource.py +++ b/caldav/calendarobjectresource.py @@ -722,7 +722,7 @@ def add_attendee(self, attendee, no_default_parameters: bool = False, **paramete raise NotImplementedError( "do we need to support this anyway? Should be trivial, but can't figure out how to do it with the icalendar.Event/vCalAddress objects right now" ) - elif attendee.startswith("mailto:"): + elif attendee.lower().startswith("mailto:"): attendee_obj = vCalAddress(attendee) elif "@" in attendee and ":" not in attendee and ";" not in attendee: attendee_obj = vCalAddress("mailto:" + attendee) @@ -1000,11 +1000,7 @@ def load(self, only_if_unloaded: bool = False) -> "Self | Coroutine[Any, Any, Se except Exception: return self.load_by_multiget() - ## consider refactoring - this is repeated many places now - if "Etag" in r.headers: - self.props[dav.GetEtag.tag] = r.headers["Etag"] - if "Schedule-Tag" in r.headers: - self.props[cdav.ScheduleTag.tag] = r.headers["Schedule-Tag"] + self._update_tag_props(r) return self async def _async_load(self, only_if_unloaded: bool = False) -> Self: @@ -1046,10 +1042,7 @@ async def _async_load(self, only_if_unloaded: bool = False) -> Self: except Exception: return await self.load_by_multiget() - if "Etag" in r.headers: - self.props[dav.GetEtag.tag] = r.headers["Etag"] - if "Schedule-Tag" in r.headers: - self.props[cdav.ScheduleTag.tag] = r.headers["Schedule-Tag"] + self._update_tag_props(r) return self def load_by_multiget(self) -> "Self | Coroutine[Any, Any, Self]": @@ -1155,6 +1148,20 @@ async def _async_put(self, headers, retry_on_failure=True): # _post_put returned a retry coroutine (self._put(False) for async client) await result + def _update_tag_props(self, r) -> None: + """Capture the ETag / Schedule-Tag response headers into self.props. + + Called after both PUT (`_post_put`) and GET (`load`/`_async_load`); + keys are matched case-insensitively by the response header dict. + See RFC 6638 for Schedule-Tag. + """ + if not r.headers: + return + if "Etag" in r.headers: + self.props[dav.GetEtag.tag] = r.headers["Etag"] + if r.headers.get("Schedule-Tag"): + self.props[cdav.ScheduleTag.tag] = r.headers["Schedule-Tag"] + def _post_put(self, r, retry_on_failure): if r.status == 412: if self.schedule_tag: @@ -1164,7 +1171,7 @@ def _post_put(self, r, retry_on_failure): else: raise error.PutError(errmsg(r)) elif r.status == 302: - self.url = URL.objectify([x[1] for x in r.headers if x[0] == "location"][0]) + self.url = URL.objectify(r.headers.get("location")) elif r.status not in (204, 201): if retry_on_failure: try: @@ -1178,31 +1185,7 @@ def _post_put(self, r, retry_on_failure): return self._put(False) else: raise error.PutError(errmsg(r)) - if "Etag" in r.headers: - self.props[dav.GetEtag.tag] = r.headers["Etag"] - if r.headers and r.headers.get("schedule-tag"): - self.props[cdav.ScheduleTag.tag] = r.headers["schedule-tag"] - - if r.status == 302: - path = [x[1] for x in r.headers if x[0] == "location"][0] - self.url = URL.objectify(path) - elif r.status not in (204, 201): - if retry_on_failure: - try: - import vobject # noqa: F401 - except ImportError: - retry_on_failure = False - if retry_on_failure: - ## This seems like a noop, but it may "wash" the object - dummy = self.vobject_instance - return self._put(False) - else: - raise error.PutError(errmsg(r)) - ## TODO: refactor - those code lines are repeated all over the place - if "Etag" in r.headers: - self.props[dav.GetEtag.tag] = r.headers["Etag"] - if r.headers and r.headers.get("schedule-tag"): - self.props[cdav.ScheduleTag.tag] = r.headers["schedule-tag"] + self._update_tag_props(r) def _create( self, id=None, path=None, retry_on_failure=True @@ -1269,7 +1252,12 @@ def change_attendee_status(self, attendee: Any | None = None, **kwargs) -> None: return ical_obj = self.icalendar_component - attendee_lines = ical_obj["attendee"] + try: + attendee_lines = ical_obj["attendee"] + except KeyError: + raise error.NotFoundError( + f"Participant {attendee!r} not found in attendee list (no ATTENDEE properties)" + ) from None if isinstance(attendee_lines, str): attendee_lines = [attendee_lines] @@ -1281,7 +1269,7 @@ def strip_mailto(x): attendee_line.params.update(kwargs) cnt += 1 if not cnt: - raise error.NotFoundError("Participant %s not found in attendee list") + raise error.NotFoundError(f"Participant {attendee!r} not found in attendee list") error.assert_(cnt == 1) def save( @@ -1570,6 +1558,7 @@ def _set_data(self, data): self._data = vcal.fix(data) self._vobject_instance = None self._icalendar_instance = None + self._state = RawDataState(self._data) return self def _get_data(self): @@ -1940,7 +1929,7 @@ def _get_duration(self, i): start = datetime(start.year, start.month, start.day) end = datetime(end.year, end.month, end.day) return end - start - elif "DTSTART" in i and not isinstance(i["DTSTART"], datetime): + elif "DTSTART" in i and not isinstance(i["DTSTART"].dt, datetime): return timedelta(days=1) else: return timedelta(0) @@ -2119,55 +2108,53 @@ def _reduce_count(self, i=None) -> bool: i["RRULE"]["COUNT"][0] -= 1 return True - def _complete_recurring_safe(self, completion_timestamp): - """This mode will create a new independent task which is - marked as completed, and modify the existing recurring task. - It is probably the most safe way to handle the completion of a - recurrence of a recurring task, though the link between the - completed task and the original task is lost. + def _build_recurring_safe_completed(self, completion_timestamp) -> "Todo | None": + """Pure (no-I/O) part of the "safe" recurring-completion strategy. + + Advances ``self`` to its next occurrence in memory and returns a + freshly-built standalone copy marked as completed. Returns + ``None`` when the task is not (or no longer) recurring, in which + case the caller should fall back to a plain completion. The + caller is responsible for saving both ``self`` and the returned + copy (one PUT each). """ ## If count is one, then it is not really recurring if not self._reduce_count(): - return self.complete(handle_rrule=False) + return None next_dtstart = self._next(completion_timestamp) if not next_dtstart: - return self.complete(handle_rrule=False) + return None completed = self.copy() completed.url = self.parent.url.join(completed.id + ".ics") completed.icalendar_component.pop("RRULE") - completed.save() - completed.complete() + completed._complete_ical(completion_timestamp=completion_timestamp) duration = self.get_duration() i = self.icalendar_component i.pop("DTSTART", None) i.add("DTSTART", next_dtstart) self.set_duration(duration, movable_attr="DUE") + return completed + def _complete_recurring_safe(self, completion_timestamp): + """This mode will create a new independent task which is + marked as completed, and modify the existing recurring task. + It is probably the most safe way to handle the completion of a + recurrence of a recurring task, though the link between the + completed task and the original task is lost. + """ + completed = self._build_recurring_safe_completed(completion_timestamp) + if completed is None: + return self.complete(handle_rrule=False) + completed.save() self.save() - def _complete_recurring_thisandfuture(self, completion_timestamp) -> None: - """The RFC is not much helpful, a lot of guesswork is needed - to consider what the "right thing" to do wrg of a completion of - recurring tasks is ... but this is my shot at it. - - 1) The original, with rrule, will be kept as it is. The rrule - string is fetched from the first subcomponent of the - icalendar. - - 2) If there are multiple recurrence instances in subcomponents - and the last one is marked with RANGE=THISANDFUTURE, then - select this one. If it has the rrule property set, use this - rrule rather than the original one. Drop the RANGE parameter. - Calculate the next RECURRENCE-ID from the DTSTART of this - object. Mark task as completed. Increase SEQUENCE. - - 3) Create a new recurrence instance with RANGE=THISANDFUTURE, - without RRULE set (Ref - https://github.com/Kozea/Radicale/issues/1264). Set the - RECURRENCE-ID to the one calculated in #2. Calculate the - DTSTART based on rrule and completion timestamp/date. + def _prepare_recurring_thisandfuture(self, completion_timestamp) -> None: + """Pure (no-I/O) in-memory mutation behind + ``_complete_recurring_thisandfuture``; see that method for the + algorithm description. The caller does the single + ``save(increase_seqno=False)`` that follows. """ recurrences = self.icalendar_instance.subcomponents orig = recurrences[0] @@ -2219,7 +2206,6 @@ def _complete_recurring_thisandfuture(self, completion_timestamp) -> None: [x for x in recurrences if not self.is_pending(x)] ): self._complete_ical(recurrences[0], completion_timestamp=completion_timestamp) - self.save(increase_seqno=False) return rrule = rrule2 or rrule @@ -2231,6 +2217,30 @@ def _complete_recurring_thisandfuture(self, completion_timestamp) -> None: thisandfuture.add("DTSTART", next_dtstart) self._set_duration(i=thisandfuture, duration=duration, movable_attr="DUE") self.icalendar_instance.subcomponents.append(thisandfuture) + + def _complete_recurring_thisandfuture(self, completion_timestamp) -> None: + """The RFC is not much helpful, a lot of guesswork is needed + to consider what the "right thing" to do wrg of a completion of + recurring tasks is ... but this is my shot at it. + + 1) The original, with rrule, will be kept as it is. The rrule + string is fetched from the first subcomponent of the + icalendar. + + 2) If there are multiple recurrence instances in subcomponents + and the last one is marked with RANGE=THISANDFUTURE, then + select this one. If it has the rrule property set, use this + rrule rather than the original one. Drop the RANGE parameter. + Calculate the next RECURRENCE-ID from the DTSTART of this + object. Mark task as completed. Increase SEQUENCE. + + 3) Create a new recurrence instance with RANGE=THISANDFUTURE, + without RRULE set (Ref + https://github.com/Kozea/Radicale/issues/1264). Set the + RECURRENCE-ID to the one calculated in #2. Calculate the + DTSTART based on rrule and completion timestamp/date. + """ + self._prepare_recurring_thisandfuture(completion_timestamp) self.save(increase_seqno=False) def complete( @@ -2284,85 +2294,15 @@ async def _async_complete( async def _async_complete_recurring_safe(self, completion_timestamp: datetime) -> None: """Async version of _complete_recurring_safe.""" - if not self._reduce_count(): + completed = self._build_recurring_safe_completed(completion_timestamp) + if completed is None: return await self._async_complete(completion_timestamp, handle_rrule=False) - next_dtstart = self._next(completion_timestamp) - if not next_dtstart: - return await self._async_complete(completion_timestamp, handle_rrule=False) - - completed = self.copy() - completed.url = self.parent.url.join(completed.id + ".ics") - completed.icalendar_component.pop("RRULE") - await completed.save() - completed._complete_ical(completion_timestamp=completion_timestamp) await completed.save() - - duration = self.get_duration() - i = self.icalendar_component - i.pop("DTSTART", None) - i.add("DTSTART", next_dtstart) - self.set_duration(duration, movable_attr="DUE") await self.save() async def _async_complete_recurring_thisandfuture(self, completion_timestamp: datetime) -> None: """Async version of _complete_recurring_thisandfuture.""" - recurrences = self.icalendar_instance.subcomponents - orig = recurrences[0] - if "STATUS" not in orig: - orig["STATUS"] = "NEEDS-ACTION" - - if len(recurrences) == 1: - just_completed = orig.copy() - just_completed.pop("RRULE") - just_completed.add("RECURRENCE-ID", orig.get("DTSTART", completion_timestamp)) - seqno = just_completed.pop("SEQUENCE", 0) - just_completed.add("SEQUENCE", seqno + 1) - recurrences.append(just_completed) - - prev = recurrences[-1] - rrule = prev.get("RRULE", orig["RRULE"]) - thisandfuture = prev.copy() - seqno = thisandfuture.pop("SEQUENCE", 0) - thisandfuture.add("SEQUENCE", seqno + 1) - - if len(recurrences) > 2: - if prev["RECURRENCE-ID"].params.get("RANGE", None) == "THISANDFUTURE": - prev["RECURRENCE-ID"].params.pop("RANGE") - else: - raise NotImplementedError( - "multiple instances found, but last one is not of type THISANDFUTURE, possibly this has been created by some incompatible client, but we should deal with it" - ) - self._complete_ical(prev, completion_timestamp) - - thisandfuture.pop("RECURRENCE-ID", None) - thisandfuture.add("RECURRENCE-ID", self._next(i=prev, rrule=rrule)) - thisandfuture["RECURRENCE-ID"].params["RANGE"] = "THISANDFUTURE" - rrule2 = thisandfuture.pop("RRULE", None) - - if rrule2 is not None: - count = rrule2.get("COUNT", None) - if count is not None and count[0] in (0, 1): - for i in recurrences: - self._complete_ical(i, completion_timestamp=completion_timestamp) - thisandfuture.add("RRULE", rrule2) - else: - count = rrule.get("COUNT", None) - if count is not None and count[0] <= len( - [x for x in recurrences if not self.is_pending(x)] - ): - self._complete_ical(recurrences[0], completion_timestamp=completion_timestamp) - await self.save(increase_seqno=False) - return - - rrule = rrule2 or rrule - - duration = self._get_duration(i=prev) - thisandfuture.pop("DTSTART", None) - thisandfuture.pop("DUE", None) - next_dtstart = self._next(i=prev, rrule=rrule, ts=completion_timestamp) - thisandfuture.add("DTSTART", next_dtstart) - self._set_duration(i=thisandfuture, duration=duration, movable_attr="DUE") - self.icalendar_instance.subcomponents.append(thisandfuture) + self._prepare_recurring_thisandfuture(completion_timestamp) await self.save(increase_seqno=False) def _complete_ical(self, i=None, completion_timestamp=None) -> None: diff --git a/caldav/collection.py b/caldav/collection.py index 37b87b96..fb270852 100644 --- a/caldav/collection.py +++ b/caldav/collection.py @@ -10,6 +10,7 @@ A SynchronizableCalendarObjectCollection contains a local copy of objects from a calendar on the server. """ +import inspect import logging import uuid import warnings @@ -31,7 +32,7 @@ from collections.abc import Coroutine, Iterable, Iterator, Sequence from typing import Literal -from .base_client import ICALH +from .base_client import ICALH, _warn_unreadable_display_name from .calendarobjectresource import ( CalendarObjectResource, Event, @@ -78,6 +79,19 @@ def _extract_calendar_id_from_url(url: str) -> str | None: return None +def _safe_display_name(cal) -> str | None: + """Return ``cal``'s DAV:displayname, or None if it can't be read. + + Used when discovering a relocated calendar's canonical URL after creation + (see Calendar._adopt_canonical_url); a calendar that refuses to report its + display name simply isn't a match. + """ + try: + return cal.get_display_name() + except Exception: + return None + + def _quote_url_path(url: str) -> str: """Quote the path component of a URL to handle unencoded spaces (e.g. Zimbra).""" parsed = urlparse(url) @@ -251,7 +265,14 @@ def calendar(self, name: str | None = None, cal_id: str | None = None) -> "Calen # For name-based lookup, use calendars() which already uses async delegation if name and not cal_id: for calendar in self.get_calendars(): - display_name = calendar.get_display_name() + try: + display_name = calendar.get_display_name() + except Exception as e: + # Skip calendars whose display name can't be read; warn only + # when the failure is unexpected (see helper). Continuing + # ensures one unreadable calendar doesn't abort the lookup. + _warn_unreadable_display_name(self.client, calendar, name, e) + continue if display_name == name: return calendar if name and not cal_id: @@ -450,10 +471,15 @@ def calendar( name: str | None = None, cal_id: str | None = None, cal_url: str | None = None, - ) -> "Calendar": + ) -> "Calendar | Coroutine[Any, Any, Calendar]": """ The calendar method will return a calendar object. - It will not initiate any communication with the server. + + For a full-URL ``cal_id`` or a ``cal_url`` it does not initiate any + communication with the server and returns the Calendar directly (also + for async clients). For a bare ``cal_id``/``name`` it needs the + calendar home set, which on an async client is resolved with a PROPFIND; + in that case it returns a coroutine that must be awaited. """ if not cal_url: ## For full-URL cal_id, skip calendar_home_set (which may be async-lazy) @@ -467,6 +493,11 @@ def calendar( if self.client is None: raise ValueError("Unexpected value None for self.client") return Calendar(self.client, url=URL.objectify(cal_id)) + ## A bare cal_id/name needs the calendar home set. On async clients + ## that resolution awaits a PROPFIND, so we must hand back a coroutine + ## rather than evaluating the (lazy, coroutine-valued) home set here. + if self.is_async_client: + return self._async_calendar(name, cal_id) return self.calendar_home_set.calendar(name, cal_id) else: if self.client is None: @@ -474,6 +505,15 @@ def calendar( return Calendar(self.client, url=self.client.url.join(cal_url)) + async def _async_calendar( + self, + name: str | None = None, + cal_id: str | None = None, + ) -> "Calendar": + """Async implementation of calendar() for a bare cal_id/name.""" + calendar_home_set = await self._async_get_calendar_home_set() + return calendar_home_set.calendar(name, cal_id) + def get_vcal_address(self) -> "vCalAddress | Coroutine[Any, Any, vCalAddress]": """ Returns the principal, as an icalendar.vCalAddress object. @@ -598,22 +638,27 @@ def freebusy_request( freebusy_ical.add_component(freebusy_comp) outbox = self.schedule_outbox() caldavobj = FreeBusy(data=freebusy_ical, parent=self) - for attendee in attendees: - caldavobj.add_attendee(attendee, no_default_parameters=True) if self.is_async_client: - return self._async_freebusy_request(outbox, caldavobj) + return self._async_freebusy_request(outbox, caldavobj, attendees) + + for attendee in attendees: + caldavobj.add_attendee(attendee, no_default_parameters=True) caldavobj.add_organizer() response = self.client.post(outbox.url, caldavobj.data, headers=ICALH) return response._parse_scheduling_response_objects(parent=self) - async def _async_freebusy_request(self, outbox, fb_obj) -> dict: + async def _async_freebusy_request(self, outbox, fb_obj, attendees) -> dict: """Async implementation of freebusy_request() for async clients.""" ## TODO: could we have common headers as global variable? headers = ICALH outbox = await outbox + for attendee in attendees: + if isinstance(attendee, Principal): + attendee = await attendee.get_vcal_address() + fb_obj.add_attendee(attendee, no_default_parameters=True) ## TODO: it's really bad that arbitrary methods returns ## a coroutine in async mode. It's needed to make it much ## more clear what methods involves I/O and what methods @@ -747,16 +792,38 @@ def _create( prop = dav.Prop() display_name = None - # Some servers (e.g. Zimbra) use the DisplayName from the MKCALENDAR body - # as the calendar URL, ignoring the actual request path. When the server - # does not support setting a separate display name, omit it from the body so - # the request URL path is used as the calendar identifier. supports_displayname = not self.client or self.client.features.is_supported( "create-calendar.set-displayname" ) + stable_url = not self.client or self.client.features.is_supported( + "create-calendar.stable-url" + ) + # A few servers assign a calendar a canonical URL that differs from the + # requested cal_id when a display name is set: Zimbra relocates the + # collection to a display-name-derived path (a collection-level alias + # lingers at the cal_id and answers PROPFIND/REPORT, but a GET on a child + # object under it 404s, so the cal_id is not a usable address), while OX + # always exposes an opaque cal://0/NNN canonical URL. We still send the + # display name (it sticks); afterwards, for such servers + # (create-calendar.stable-url unsupported), we DISCOVER and ADOPT the + # canonical URL (see _adopt_canonical_url) so that self.url - and every + # later URL-based operation - points at the address that actually + # resolves. This replaces the older "drop the display name" workaround + # and behaves identically for Zimbra and OX. We only omit the display + # name when the server cannot set one at creation at all + # (create-calendar.set-displayname unsupported). if name and supports_displayname: display_name = dav.DisplayName(name) prop += [display_name] + elif name: # not supports_displayname + log.warning( + "Creating calendar %r without the requested display name %r: the " + "server does not support setting a display name when a calendar is " + "created (create-calendar.set-displayname). The calendar keeps its " + "requested URL but will have no display name.", + id, + name, + ) if supported_calendar_component_set: sccs = cdav.SupportedCalendarComponentSet() for scc in supported_calendar_component_set: @@ -769,7 +836,7 @@ def _create( mkcol = (dav.Mkcol() if method == "mkcol" else cdav.Mkcalendar()) + set if self.is_async_client: - return self._async_create(path, mkcol, method, name, display_name) + return self._async_create(path, mkcol, method, name, display_name, stable_url) self._query(root=mkcol, query_method=method, url=path, expected_return_value=201) @@ -792,7 +859,50 @@ def _create( exc_info=True, ) - async def _async_create(self, path, mkcol, method, name, display_name) -> None: + # On servers that don't keep the calendar at the requested cal_id when a + # display name is set (create-calendar.stable-url unsupported), re-point + # self.url to the canonical URL the server actually assigned. + if display_name and not stable_url: + self._adopt_canonical_url(name) + + def _adopt_canonical_url(self, name) -> None: + """Re-point ``self.url`` to the server's canonical URL for this calendar. + + Called only for servers where ``create-calendar.stable-url`` is + unsupported: the calendar just created is reachable under a canonical URL + that differs from the requested cal_id (Zimbra: a display-name-derived + path; OX: an opaque ``cal://0/NNN`` segment). The requested cal_id is not + a reliable address there (on Zimbra a collection alias answers + PROPFIND/REPORT but a GET on a child object 404s). We locate the calendar + by the display name we just set and adopt its URL so later URL-based + operations resolve. + + Best effort: if the calendar can't be located (or its name is ambiguous + because another calendar already shares it), ``self.url`` is left at the + requested URL. + """ + requested = self.url.canonical() + try: + relocated = [ + cal + for cal in self.parent.calendars() + if _safe_display_name(cal) == name and cal.url.canonical() != requested + ] + except Exception: + log.warning("Could not list calendars to discover canonical URL", exc_info=True) + return + if not relocated: + return + if len(relocated) > 1: + log.warning( + "Multiple calendars named %r found while discovering the canonical " + "URL for the calendar just created; adopting the first relocated one (%s)", + name, + relocated[0].url, + ) + self.url = relocated[0].url + + async def _async_create(self, path, mkcol, method, name, display_name, stable_url) -> None: """Async implementation of _create (call via _create, not directly).""" await self._query(root=mkcol, query_method=method, url=path, expected_return_value=201) @@ -810,6 +920,25 @@ async def _async_create(self, path, mkcol, method, name, display_name) -> None: exc_info=True, ) + # See _adopt_canonical_url (sync) - re-point self.url on unstable servers. + if display_name and not stable_url: + try: + cals = await self.parent.calendars() + requested = self.url.canonical() + for cal in cals: + try: + dn = await cal.get_display_name() + except Exception: + continue + if dn == name and cal.url.canonical() != requested: + self.url = cal.url + break + except Exception: + log.warning( + "Could not list calendars to discover canonical URL (async)", + exc_info=True, + ) + def delete(self, wipe=None): """Delete the calendar. @@ -1139,28 +1268,41 @@ async def _async_save(self, display_name, method=None): # def data2object_class - def _multiget(self, event_urls: Iterable[URL], raise_notfound: bool = False) -> Iterable[str]: - """ - get multiple events' data. - TODO: Does it overlap the _request_report_build_resultlist method - ## WARNING: async logic is duplicated in _async_multiget — mirror any changes there + def _build_multiget_root(self, event_urls: Iterable[URL]) -> cdav.CalendarMultiGet: + """Build the calendar-multiget REPORT body for the given hrefs. + + Pure (no I/O) — shared by the sync and async multiget twins. """ if self.url is None: raise ValueError("Unexpected value None for self.url") - prop = dav.Prop() + cdav.CalendarData() - root = cdav.CalendarMultiGet() + prop + [dav.Href(value=u.path) for u in event_urls] - # RFC 4791 section 7.9: "the 'Depth' header MUST be ignored by the - # server and SHOULD NOT be sent by the client" for calendar-multiget - response = self._query(root, None, "report") + return cdav.CalendarMultiGet() + prop + [dav.Href(value=u.path) for u in event_urls] + + def _extract_multiget_results( + self, response: Any, raise_notfound: bool + ) -> list[tuple[str, str]]: + """Turn a multiget REPORT response into ``(href, calendar_data)`` tuples. + + Pure (no I/O) — shared by the sync and async multiget twins. + """ results = response.expand_simple_props([cdav.CalendarData()]) if raise_notfound: - for href in response.statuses: - status = response.statuses[href] + for href, status in response.statuses.items(): if status and "404" in status: raise error.NotFoundError(f"Status {status} in {href}") - for r in results: - yield (r, results[r][cdav.CalendarData.tag]) + return [(r, results[r][cdav.CalendarData.tag]) for r in results] + + def _multiget( + self, event_urls: Iterable[URL], raise_notfound: bool = False + ) -> list[tuple[str, str]]: + """get multiple events' data. + + TODO: Does it overlap the _request_report_build_resultlist method? + """ + # RFC 4791 section 7.9: "the 'Depth' header MUST be ignored by the + # server and SHOULD NOT be sent by the client" for calendar-multiget + response = self._query(self._build_multiget_root(event_urls), None, "report") + return self._extract_multiget_results(response, raise_notfound) def _post_multiget(self, results: Iterable[tuple[str, str]]) -> list[_CC]: """Post-processing shared by multiget and _async_multiget_objects.""" @@ -1188,20 +1330,8 @@ def multiget(self, event_urls: Iterable[URL], raise_notfound: bool = False) -> I async def _async_multiget( self, event_urls: Iterable[URL], raise_notfound: bool = False ) -> list[tuple[str, str]]: - ## WARNING: sync logic is duplicated in _multiget — mirror any changes there - if self.url is None: - raise ValueError("Unexpected value None for self.url") - - prop = dav.Prop() + cdav.CalendarData() - root = cdav.CalendarMultiGet() + prop + [dav.Href(value=u.path) for u in event_urls] - response = await self._query(root, None, "report") - results = response.expand_simple_props([cdav.CalendarData()]) - if raise_notfound: - for href in response.statuses: - status = response.statuses[href] - if status and "404" in status: - raise error.NotFoundError(f"Status {status} in {href}") - return [(r, results[r][cdav.CalendarData.tag]) for r in results] + response = await self._query(self._build_multiget_root(event_urls), None, "report") + return self._extract_multiget_results(response, raise_notfound) async def _async_multiget_objects( self, event_urls: Iterable[URL], raise_notfound: bool = False @@ -1211,6 +1341,67 @@ async def _async_multiget_objects( await self._async_multiget(event_urls, raise_notfound=raise_notfound) ) + def _assign_multiget_data(self, unloaded: list, results: Iterable[tuple[str, str]]) -> None: + """Assign multiget (href, data) results onto the matching unloaded objects. + + Shared post-processing for _batch_load_objects and its async twin: index + the results by normalised URL (quoting to match servers that return + unencoded spaces, e.g. Zimbra) and set obj.data on each match. + """ + url_to_data = { + str(self.url.join(quote(unquote(str(href)), safe="/:@"))): data + for href, data in results + } + for obj in unloaded: + if str(obj.url) in url_to_data: + obj.data = url_to_data[str(obj.url)] + + def _batch_load_objects(self, objects: list) -> None: + """Load unloaded objects from the list in a single calendar-multiget REPORT. + + Already-loaded objects are skipped. If the REPORT fails, falls back to + individual obj.load(only_if_unloaded=True) calls per object, silently + swallowing per-object errors so callers can filter on is_loaded() afterward. + """ + unloaded = [o for o in objects if not o.is_loaded()] + if not unloaded: + return + try: + self._assign_multiget_data(unloaded, self._multiget([o.url for o in unloaded])) + except Exception: + logging.error("Batch multiget failed, falling back to individual loads", exc_info=True) + for obj in unloaded: + try: + obj.load(only_if_unloaded=True) + except Exception: + pass + + async def _async_batch_load_objects(self, objects: list) -> None: + """Async version of _batch_load_objects. + + The post-processing is shared via _assign_multiget_data(); the only + sync/async difference is the await on the multiget REPORT and on the + per-object fallback load(). + """ + unloaded = [o for o in objects if not o.is_loaded()] + if not unloaded: + return + try: + self._assign_multiget_data( + unloaded, await self._async_multiget([o.url for o in unloaded]) + ) + except Exception: + logging.error( + "Async batch multiget failed, falling back to individual loads", exc_info=True + ) + for obj in unloaded: + try: + load_result = obj.load(only_if_unloaded=True) + if inspect.isawaitable(load_result): + await load_result + except Exception: + pass + def calendar_multiget(self, *largs, **kwargs): """ get multiple events' data @@ -1417,6 +1608,7 @@ def search( filters=None, post_filter=None, _hacks=None, + compatibility_workarounds: bool | None = None, **searchargs, ) -> "list[_CC] | Coroutine[Any, Any, list[_CC]]": """Sends a search request towards the server, processes the @@ -1529,11 +1721,25 @@ def search( # For async clients, use async_search if self.is_async_client: return my_searcher.async_search( - self, server_expand, split_expanded, props, xml, post_filter, _hacks + self, + server_expand, + split_expanded, + props, + xml, + post_filter, + _hacks, + compatibility_workarounds, ) return my_searcher.search( - self, server_expand, split_expanded, props, xml, post_filter, _hacks + self, + server_expand, + split_expanded, + props, + xml, + post_filter, + _hacks, + compatibility_workarounds, ) def freebusy_request( @@ -1834,6 +2040,64 @@ def _generate_fake_sync_token(self, objects: list["CalendarObjectResource"]) -> hash_value = hashlib.md5(combined.encode(), usedforsecurity=False).hexdigest() return f"fake-{hash_value}" + ## The three helpers below carry the pure (no-I/O) logic shared between the + ## get_objects_by_sync_token sync/async twins, so only the awaited + ## server round-trips differ between them. + + def _should_use_sync_token(self, sync_token: Any, disable_fallback: bool) -> bool: + """Decide whether to attempt a real sync-collection REPORT. + + Raises ReportError when the server can't do sync-tokens and the caller + forbade the full-retrieval fallback. + """ + sync_support = self.client.features.is_supported("sync-token", return_type=dict) + if sync_support.get("support") == "unsupported": + if disable_fallback: + raise error.ReportError("Sync tokens are not supported by the server") + return False + ## A fake token means we emulated sync support last time; don't try a real one. + if sync_token and isinstance(sync_token, str) and sync_token.startswith("fake-"): + return False + return True + + def _apply_fallback_etags(self, response: Any, all_objects: list) -> None: + """Map ETags from a depth-1 PROPFIND response onto the given objects. + + ETags are crucial for detecting content changes in the fallback + mechanism (which can otherwise only see additions/deletions). + """ + etag_props = response.expand_simple_props([dav.GetEtag()]) + url_to_obj = {str(obj.url.canonical()): obj for obj in all_objects} + log.debug(f"Fallback: Fetching ETags for {len(url_to_obj)} objects") + for url_str, props in etag_props.items(): + canonical_url_str = str(self.url.join(url_str).canonical()) + if canonical_url_str in url_to_obj: + if not hasattr(url_to_obj[canonical_url_str], "props"): + url_to_obj[canonical_url_str].props = {} + url_to_obj[canonical_url_str].props.update(props) + log.debug(f"Fallback: Added ETag to {canonical_url_str}") + + def _build_fallback_sync_result( + self, all_objects: list, sync_token: Any + ) -> "SynchronizableCalendarObjectCollection": + """Build the fallback collection from a full object list, emulating + sync-token semantics: if the caller passed back our previous fake + token and nothing changed, return an empty collection. + """ + fake_sync_token = self._generate_fake_sync_token(all_objects) + if ( + sync_token + and isinstance(sync_token, str) + and sync_token.startswith("fake-") + and sync_token == fake_sync_token + ): + return SynchronizableCalendarObjectCollection( + calendar=self, objects=[], sync_token=fake_sync_token + ) + return SynchronizableCalendarObjectCollection( + calendar=self, objects=all_objects, sync_token=fake_sync_token + ) + def get_objects_by_sync_token( self, sync_token: Any | None = None, @@ -1869,23 +2133,9 @@ def get_objects_by_sync_token( the server truly supports sync tokens. """ if self.is_async_client: - ## TODO: lots of code duplication here. It's difficult, since there is a lot of - ## forth and back between the client and the server in this method. return self._async_get_objects_by_sync_token(sync_token, load_objects, disable_fallback) - ## Check if we should attempt to use sync tokens - ## (either server supports them, or we haven't checked yet, or this is a fake token) - use_sync_token = True - sync_support = self.client.features.is_supported("sync-token", return_type=dict) - if sync_support.get("support") == "unsupported": - if disable_fallback: - raise error.ReportError("Sync tokens are not supported by the server") - use_sync_token = False - ## If sync_token looks like a fake token, don't try real sync-collection - if sync_token and isinstance(sync_token, str) and sync_token.startswith("fake-"): - use_sync_token = False - - if use_sync_token: + if self._should_use_sync_token(sync_token, disable_fallback): try: root = self.client._build_sync_collection_body( sync_token=sync_token, props=["getetag"] @@ -1931,50 +2181,17 @@ def get_objects_by_sync_token( pass ## Fetch ETags for all objects if not already present - ## ETags are crucial for detecting changes in the fallback mechanism if all_objects and ( not hasattr(all_objects[0], "props") or dav.GetEtag.tag not in all_objects[0].props ): - ## Use PROPFIND to fetch ETags for all objects try: ## Do a depth-1 PROPFIND on the calendar to get all ETags response = self._query_properties([dav.GetEtag()], depth=1) - etag_props = response.expand_simple_props([dav.GetEtag()]) - - ## Map ETags to objects by URL (using string keys for reliable comparison) - url_to_obj = {str(obj.url.canonical()): obj for obj in all_objects} - log.debug(f"Fallback: Fetching ETags for {len(url_to_obj)} objects") - for url_str, props in etag_props.items(): - canonical_url_str = str(self.url.join(url_str).canonical()) - if canonical_url_str in url_to_obj: - if not hasattr(url_to_obj[canonical_url_str], "props"): - url_to_obj[canonical_url_str].props = {} - url_to_obj[canonical_url_str].props.update(props) - log.debug(f"Fallback: Added ETag to {canonical_url_str}") + self._apply_fallback_etags(response, all_objects) except Exception as e: - ## If fetching ETags fails, we'll fall back to URL-based tokens - ## which can't detect content changes, only additions/deletions log.debug(f"Failed to fetch ETags for fallback sync: {e}") - pass - ## Generate a fake sync token based on current state - fake_sync_token = self._generate_fake_sync_token(all_objects) - - ## If a sync_token was provided, check if anything has changed - if sync_token and isinstance(sync_token, str) and sync_token.startswith("fake-"): - ## Compare the provided token with the new token - if sync_token == fake_sync_token: - ## Nothing has changed, return empty collection - return SynchronizableCalendarObjectCollection( - calendar=self, objects=[], sync_token=fake_sync_token - ) - ## If tokens differ, return all objects (emulating a full sync) - ## In a real implementation, we'd return only changed objects, - ## but that requires storing previous state which we don't have - - return SynchronizableCalendarObjectCollection( - calendar=self, objects=all_objects, sync_token=fake_sync_token - ) + return self._build_fallback_sync_result(all_objects, sync_token) def objects_by_sync_token( self, *largs, **kwargs @@ -1996,20 +2213,7 @@ async def _async_get_objects_by_sync_token( disable_fallback: bool = False, ) -> "SynchronizableCalendarObjectCollection": """Async implementation of get_objects_by_sync_token.""" - - ## TODO: lots of code duplication here. It's difficult, since there is a lot of - ## forth and back between the client and the server in this method. - - use_sync_token = True - sync_support = self.client.features.is_supported("sync-token", return_type=dict) - if sync_support.get("support") == "unsupported": - if disable_fallback: - raise error.ReportError("Sync tokens are not supported by the server") - use_sync_token = False - if sync_token and isinstance(sync_token, str) and sync_token.startswith("fake-"): - use_sync_token = False - - if use_sync_token: + if self._should_use_sync_token(sync_token, disable_fallback): try: root = self.client._build_sync_collection_body( sync_token=sync_token, props=["getetag"] @@ -2048,30 +2252,11 @@ async def _async_get_objects_by_sync_token( ): try: response = await self._query_properties([dav.GetEtag()], depth=1) - etag_props = response.expand_simple_props([dav.GetEtag()]) - url_to_obj = {str(obj.url.canonical()): obj for obj in all_objects} - log.debug(f"Fallback: Fetching ETags for {len(url_to_obj)} objects") - for url_str, props in etag_props.items(): - canonical_url_str = str(self.url.join(url_str).canonical()) - if canonical_url_str in url_to_obj: - if not hasattr(url_to_obj[canonical_url_str], "props"): - url_to_obj[canonical_url_str].props = {} - url_to_obj[canonical_url_str].props.update(props) - log.debug(f"Fallback: Added ETag to {canonical_url_str}") + self._apply_fallback_etags(response, all_objects) except Exception as e: log.debug(f"Failed to fetch ETags for fallback sync: {e}") - fake_sync_token = self._generate_fake_sync_token(all_objects) - - if sync_token and isinstance(sync_token, str) and sync_token.startswith("fake-"): - if sync_token == fake_sync_token: - return SynchronizableCalendarObjectCollection( - calendar=self, objects=[], sync_token=fake_sync_token - ) - - return SynchronizableCalendarObjectCollection( - calendar=self, objects=all_objects, sync_token=fake_sync_token - ) + return self._build_fallback_sync_result(all_objects, sync_token) def get_journals(self) -> "list[Journal] | Coroutine[Any, Any, list[Journal]]": """ diff --git a/caldav/compatibility_hints.py b/caldav/compatibility_hints.py index 0ff4e690..059091fd 100644 --- a/caldav/compatibility_hints.py +++ b/caldav/compatibility_hints.py @@ -80,8 +80,17 @@ class FeatureSet: "url": { "type": "client-hints", }, + "well-known": { + "description": "Server handles /.well-known/caldav discovery as specified in RFC 6764 section 5. A conformant server should respond with a redirect (301/302/307/308) from /.well-known/caldav to the actual CalDAV endpoint. 'full' means a redirect was observed; 'unsupported' means the server returned 404 or similar; 'unknown' means the check was skipped (e.g. localhost or request failed). Note: well-known is often provided by infrastructure (reverse proxy/hosting) rather than the CalDAV server itself, so 'unknown' is the expected default for self-hosted or test setups.", + "default": {"support": "unknown"}, + "links": ["https://datatracker.ietf.org/doc/html/rfc6764#section-5"], + }, "get-current-user-principal": { "description": "Support for RFC5397, current principal extension. Most CalDAV servers have this, but it is an extension to the DAV standard. Possibly observed missing on mail.ru, DavMail gateway and it is possible to configure the support in some sabre-based servers", + ## Independent feature (directly probed): the default marks it so the + ## node uses its own probed value rather than being derived from + ## subfeatures such as .has-calendar. + "default": {"support": "full"}, "links": ["https://datatracker.ietf.org/doc/html/rfc5397"], }, "get-current-user-principal.has-calendar": { @@ -91,9 +100,36 @@ class FeatureSet: "description": "Server returns the supported-calendar-component-set property (RFC 4791 section 5.2.3). The property is optional: when absent the RFC mandates that all component types are accepted, so 'unsupported' here is not a protocol violation, but the client cannot determine the actual supported set without trying.", "links": ["https://datatracker.ietf.org/doc/html/rfc4791#section-5.2.3"], }, + "propfind": { + "description": "Server supports the PROPFIND method (RFC4918 section 9.1): a PROPFIND for a named property returns a multistatus response. Independent feature (not just a grouping node) so that a server lacking a sub-feature like propfind.allprop.resourcetype is not mistaken for one that does not support PROPFIND at all.", + "default": {"support": "full"}, + "links": ["https://datatracker.ietf.org/doc/html/rfc4918#section-9.1"], + }, + "propfind.allprop": { + "description": "An PROPFIND returns a multistatus response. This is independent of whether resourcetype in particular is included (see propfind.allprop.resourcetype).", + "default": {"support": "full"}, + "links": ["https://datatracker.ietf.org/doc/html/rfc4918#section-9.1"], + }, + "propfind.allprop.resourcetype": { + "description": "An PROPFIND returns the DAV:resourcetype live property. RFC4918 section 9.1 lists resourcetype among the live properties an allprop request should return, so 'full' (the default) is the conformant behaviour; a few servers (Bedework) omit it.", + "default": {"support": "full"}, + "links": ["https://datatracker.ietf.org/doc/html/rfc4918#section-9.1"], + }, "create-calendar.with-supported-component-types": { "description": "Server honours the supported-calendar-component-set restriction set at MKCALENDAR time. When 'full', the server both advertises (or enforces) the restriction; when 'unsupported', the restriction is silently ignored (wrong-type objects can be saved to the calendar). When 'ungraceful', the MKCALENDAR request itself fails when a component set is specified.", }, + "calendar-color": { + "description": "Server stores the nonstandard Apple/Mozilla {http://apple.com/ns/ical/}calendar-color property (set with a colour name like 'blue') on a calendar collection. 'full' covers servers that normalise the name to a hex value (the set value still tracks the input); 'broken' is a read-only property (the same value comes back regardless of what is set). Not described by RFC4791/RFC5545, so a server that rejects or ignores it ('unsupported') is not breaching any RFC. The default is 'fragile' because the behaviour varies a lot between servers and is rarely worth asserting on.", + "default": {"support": "fragile"}, + }, + "calendar-color.hex": { + "description": "Like calendar-color, but the property is set with a hex value (e.g. '#FF0000FF') rather than a colour name. Some servers accept one form but not the other.", + "default": {"support": "fragile"}, + }, + "calendar-order": { + "description": "Server stores the nonstandard Apple/Mozilla {http://apple.com/ns/ical/}calendar-order property on a calendar collection (a get/set round-trip). 'broken' is a read-only property (e.g. the server returns the calendar's own position regardless of what is set). Not described by RFC4791/RFC5545, so a server that rejects or ignores it ('unsupported') is not breaching any RFC. The default is 'fragile' because the behaviour varies a lot between servers.", + "default": {"support": "fragile"}, + }, "rate-limit": { "type": "client-feature", "description": "client (or test code) must sleep a bit between requests. Pro-active rate limiting is done through interval and count, server-flagged rate-limiting is controlled through default_sleep/max_sleep", @@ -110,6 +146,15 @@ class FeatureSet: "delay": "after this number of seconds, we may be reasonably sure that the search results are updated", } }, + "write-delay": { + "type": "server-peculiarity", + "default": {"support": "full"}, + "description": "The server processes write operations (PUT/DELETE/MKCALENDAR/PROPPATCH/...) asynchronously: the request returns success before the change has fully taken effect, so an immediate read-back (of any kind, not just a search) may 404 or return stale data. A client must wait a bit after every write. This is the general, write-side counterpart of 'search-cache' (which only delays searches). 'full' (the default) means writes take effect synchronously.", + "extra_keys": { + "behaviour": "'delay' to enable the post-write sleep", + "delay": "sleep this number of seconds after every write request before relying on the change being visible", + } + }, "tests-cleanup-calendar": { "type": "tests-behaviour", "description": "Deleting a calendar does not delete the objects, or perhaps create/delete of calendars does not work at all. For each test run, every calendar resource object should be deleted for every test run", @@ -127,10 +172,38 @@ class FeatureSet: "description": "Accessing a calendar which does not exist automatically creates it", }, "create-calendar.set-displayname": { - "description": "It's possible to set the displayname on a calendar upon creation" + "description": "It's possible to set the displayname on a calendar upon creation", + ## Independent feature (directly probed). + "default": {"support": "full"}, + }, + "create-calendar.stable-url": { + "description": ( + "After a calendar is created it remains addressable at the URL derived from the " + "requested cal_id. 'full' (the normal case): the calendar's canonical URL is the " + "requested URL. 'unsupported': the server assigns a DIFFERENT canonical URL and the " + "requested cal_id is not a reliable address for the calendar's object resources, so " + "clients must discover and adopt the canonical URL after creation (the caldav library " + "does this automatically). Two known patterns are handled identically: Zimbra " + "relocates the collection to a display-name-derived path - a collection-level alias " + "may linger at the cal_id and answer PROPFIND/REPORT, but a GET on a child object " + "(...//.ics) 404s, so it is not a usable address (cf. save-load.get-by-url); " + "OX always exposes an opaque cal://0/NNN (base64-segment) canonical URL. Note: on " + "Zimbra the URL only becomes unstable when a display name is supplied at creation; a " + "nameless MKCALENDAR stays at the requested cal_id." + ), + "default": {"support": "full"}, + }, + "propfind.displayname": { + "description": "Server returns the DAV:displayname property for a calendar collection via PROPFIND (RFC4918 section 15.2). This is a standard live property; virtually all CalDAV servers support it. 'broken' means the property is absent from the PROPFIND response even though a displayname was supplied at creation time.", + "default": {"support": "full"}, + "links": ["https://datatracker.ietf.org/doc/html/rfc4918#section-15.2"], }, "delete-calendar": { "description": "RFC4791 says nothing about deletion of calendars, so the server implementation is free to choose weather this should be supported or not. Section 3.2.3.2 in RFC 6638 says that if a calendar is deleted, all the calendarobjectresources on the calendar should also be deleted - but it's a bit unclear if this only applies to scheduling objects or not. Some calendar servers moves the object to a trashcan rather than deleting it", + ## Independent feature (directly probed): the default marks it so the + ## node uses its own probed value rather than being derived from + ## .free-namespace. + "default": {"support": "full"}, "links": ["https://datatracker.ietf.org/doc/html/rfc6638#section-3.2.3.2"], }, "delete-calendar.free-namespace": { @@ -142,9 +215,12 @@ class FeatureSet: "default": { "support": "fragile" }, }, "save-load": { - "description": "it's possible to save and load objects to the calendar" + "description": "it's possible to save and load objects to the calendar", + }, + "save-load.event": { ## TODO: make this DRY + "description": "it's possible to save and load events to the calendar", + "default": { "support": "full" } }, - "save-load.event": {"description": "it's possible to save and load events to the calendar"}, "save-load.event.recurrences": {"description": "it's possible to save and load recurring events to the calendar - events with an RRULE property set, including recurrence sets", "default": {"support": "full"}}, "save-load.event.recurrences.count": {"description": "The server will receive and store a recurring event with a count set in the RRULE", "default": {"support": "full"}}, ## This was Claude's suggestion and it works as of today, the @@ -157,17 +233,32 @@ class FeatureSet: ## information was simply discarded, and the current search behaviour would in ## such a case be incorrect if the exception is simply discarded. "save-load.event.recurrences.exception": {"description": "When a VCALENDAR containing a master VEVENT (with RRULE) and exception VEVENT(s) (with RECURRENCE-ID) is stored, the server keeps them together as a single calendar object resource. When unsupported, the server splits exception VEVENTs into separate calendar objects, making client-side expansion unreliable (the master expands without knowing about its exceptions)."}, - "save-load.todo": {"description": "it's possible to save and load tasks to the calendar"}, - "save-load.todo.recurrences": {"description": "it's possible to save and load recurring tasks to the calendar"}, + "save-load.event.recurrences.exception.reschedule": {"description": "The server accepts a PUT that reschedules an entire recurring event - changing the master VEVENT's DTSTART (re-anchoring the whole series) while detached exception VEVENT(s) (with RECURRENCE-ID) are present and their RECURRENCE-IDs are shifted to line up with the new series. This is unsupported for Ox, the server rejects such a PUT with 409 Conflict even when a matching If-Match etag is supplied. Rescheduling a recurring event that has no exceptions still works. Exercised by save(all_recurrences=True) after changing dtstart/dtend.", "default": {"support": "full"}}, + "save-load.todo": { + "description": "it's possible to save and load tasks to the calendar", + "default": { "support": "full" } + }, + "save-load.todo.recurrences": {"description": "it's possible to save and load recurring tasks to the calendar", "default": {"support": "full"}}, "save-load.todo.recurrences.count": {"description": "The server will receive and store a recurring task with a count set in the RRULE", "default": {"support": "full"}}, "save-load.todo.recurrences.thisandfuture": {"description": "Completing a recurring task with rrule_mode='thisandfuture' works (modifies RRULE and saves back to server)", "default": {"support": "full"}}, "save-load.todo.mixed-calendar": {"description": "The same calendar may contain both events and tasks (Zimbra only allows tasks to be placed on special task lists)", "default": {"support": "full"}}, - "save-load.journal": {"description": "The server will even accept journals"}, + "save-load.journal": { + "description": "The server will even accept journals", + "default": { "support": "full" } + }, ## TODO: zimbra cannot mix events and tasks, but then davis surprised me by not allowing journals on the same calendar. But this may be a miss in the checking script - it may be that mixing is allowed, but that the calendar has to be set up from scratch with explicit support for both VJOURNAL and other things "save-load.journal.mixed-calendar": {"description": "The same calendar may contain events, tasks and journals (some servers require journals on a dedicated VJOURNAL calendar)", "default": {"support": "full"}}, "save-load.get-by-url": { "description": "GET requests to calendar object resource URLs work correctly. When unsupported, the server returns 404 on GET even for valid object URLs. The client works around this by falling back to UID-based lookup.", }, + "non-existing-raises-not-found": { + "description": "Looking up a non-existing calendar object resource raises NotFoundError (the server answers 404). 'full' (the default) is the expected behaviour; some servers answer 403 instead (raising AuthorizationError) - e.g. Robur, probably to avoid leaking whether a resource exists - which is a legitimate choice rather than an RFC breach, so it is recorded as 'unsupported' rather than 'broken'.", + "default": {"support": "full"}, + }, + "save-load.stable-url": { + "description": "The server reports a calendar object resource under the same URL the client used to store it. When 'unsupported', the server canonicalizes the URL: e.g. OX App Suite exposes a calendar both under its display name and under an internal 'cal://0/NNN' identifier, so an object looked up via a calendar-query REPORT (object_by_uid / search) is reported under a different calendar path than the PUT URL. A direct GET on the original URL still works (the server keeps an alias). Clients should therefore not assume that a searched object's URL equals the URL it was created at.", + "default": {"support": "full"}, + }, "save-load.reuse-deleted-uid": { "description": "After deleting an event, the server allows creating a new event with the same UID. When 'broken', the server keeps deleted events in a trashbin with a soft-delete flag, causing unique constraint violations on UID reuse. See https://github.com/nextcloud/server/issues/30096" }, @@ -184,6 +275,15 @@ class FeatureSet: "description": "A saved calendar object resource can be modified and PUT back to the server; the server accepts the update and returns the modified data on the next GET/REPORT. When 'unsupported', the server treats calendar objects as immutable after initial creation (e.g. Google Calendar's legacy CalDAV API). Replaces the old 'no_overwrite' compatibility flag.", "default": {"support": "full"}, }, + "save-load.mutable.attendee-partstat": { + "description": "A client can modify an attendee's PARTSTAT on an existing event and PUT it back directly to the calendar. When 'unsupported', the server forbids direct modification of attendee participation status via PUT (e.g. OX App Suite returns 403 Forbidden even with a matching If-Match etag) and expects the change to be made through iTIP scheduling instead. See https://github.com/python-caldav/caldav/issues/399", + "default": {"support": "full"}, + "links": ["https://github.com/python-caldav/caldav/issues/399"], + }, + "save-load.mutable.if-match-optional": { + "description": "The If-Match precondition is optional when overwriting an existing calendar object resource: the server accepts a PUT that carries no If-Match etag (i.e. add_event()/save() on an object that was not first fetched). When 'unsupported', the server requires an If-Match etag for updates and rejects a no-If-Match overwrite with 409 Conflict (e.g. OX App Suite enforces optimistic concurrency). Such servers still support save-load.mutable via a fetch-then-save (etag-conditional) update; only the blind-overwrite path is affected.", + "default": {"support": "full"}, + }, "search": { "description": "calendar MUST support searching for objects using the REPORT method, as specified in RFC4791, section 7", "links": ["https://datatracker.ietf.org/doc/html/rfc4791#section-7"], @@ -208,7 +308,17 @@ class FeatureSet: "description": "Time-range searches should only return events/todos that actually fall within the requested time range. Some servers incorrectly return recurring events whose recurrences fall outside (after) the search interval, or events with no recurrences in the requested time range at all. RFC4791 section 9.9 specifies that a VEVENT component overlaps a time range if the condition (start < search_end AND end > search_start) is true.", "links": ["https://datatracker.ietf.org/doc/html/rfc4791#section-9.9"], }, + "search.time-range.comp-type-optional": { + "description": "Whether the server accepts a calendar-query carrying a time-range filter but NOT specifying a component type. Per RFC4791 section 9.7 a CALDAV:time-range element is only valid inside a comp-filter for VEVENT/VTODO/VJOURNAL/VFREEBUSY/VALARM - never directly under the VCALENDAR comp-filter. A query without a component type therefore has nowhere RFC-legal to put the time-range. Consequently 'unsupported' (the default) is FULLY RFC-COMPLIANT and is NOT a server defect: SabreDAV-based servers (Baikal, Nextcloud, ...) correctly reject such queries with HTTP 400 'You cannot add time-range filters on the VCALENDAR component'. When unsupported, the library splits the search into one query per component type. See https://github.com/python-caldav/caldav/issues/681", + "default": {"support": "unsupported"}, + "links": ["https://datatracker.ietf.org/doc/html/rfc4791#section-9.7"], + }, "search.time-range.todo": {"description": "basic time range searches for tasks works", "default": {"support": "full"}}, + "search.time-range.todo.no-dtstart": { + "description": "A VTODO without DTSTART (but with DUE) is returned by a date-range search. RFC5545 and RFC4791 section 9.9 say such a task has a defined time span and should be found, so 'full' (the default) is the compliant behaviour; some servers (Davical, Stalwart, Synology) skip any task lacking DTSTART. Probed with a closed window; servers that skip such tasks only in closed ranges (returning them in open-ended ones) are instead tracked by the 'vtodo_datesearch_nodtstart_task_is_skipped_in_closed_date_range' flag.", + "default": {"support": "full"}, + "links": ["https://datatracker.ietf.org/doc/html/rfc4791#section-9.9"], + }, "search.time-range.todo.old-dates": {"description": "time range searches for tasks with old dates (e.g. year 2000) work - some servers enforce a min-date-time restriction"}, "search.time-range.todo.strict": { "description": "Bounded VTODO time-range searches do not return tasks whose time span falls entirely outside the searched range (no false positives).", @@ -264,6 +374,11 @@ class FeatureSet: "search.text": { "description": "Search for text attributes should work" }, + "search.text.comp-type-optional": { + "description": "Whether the server returns matching objects for a calendar-query that carries a prop-filter (CATEGORIES, SUMMARY, ...) but does NOT specify a component type. Such a prop-filter ends up directly under the VCALENDAR comp-filter, where it filters on VCALENDAR's own properties - which do not include component properties like CATEGORIES - so most servers (e.g. Xandikos, SabreDAV) match nothing. 'unsupported' (the default) is therefore the common, RFC-reasonable case; when unsupported the library splits the search into one query per component type. Analogous to search.time-range.comp-type-optional. See https://github.com/python-caldav/caldav/issues/681", + "default": {"support": "unsupported"}, + "links": ["https://datatracker.ietf.org/doc/html/rfc4791#section-9.7"], + }, "search.text.case-sensitive": { "description": "In RFC4791, section-9.7.5, a text-match may pass a collation, and i;ascii-casemap MUST be the default, this is not checked (yet - TODO) by the caldav-server-checker project. Section 7.5 describes that the servers also are REQUIRED to support i;octet. The definitions of those collations are given in RFC4790, i;octet is a case-sensitive byte-by-byte comparition (fastest). search.text.case-sensitive is supported if passing the i;octet collation to search causes the search to be case-sensitive.", "links": [ @@ -285,6 +400,10 @@ class FeatureSet: }, "search.text.category": { "description": "Search for category should work. This is not explicitly specified in RFC4791, but covered in section 9.7.5. No examples targets categories explicitly, but there are some text match examples in section 7.8.6 and following sections", + ## Independent feature (directly probed): the default marks it so the + ## node uses its own probed value rather than being derived from + ## .substring. + "default": {"support": "full"}, "links": [ "https://datatracker.ietf.org/doc/html/rfc4791#section-9.7.5", "https://datatracker.ietf.org/doc/html/rfc4791#section-7.8.6", @@ -301,7 +420,11 @@ class FeatureSet: "links": ["https://datatracker.ietf.org/doc/html/rfc4791#section-7.4"], }, "search.recurrences.includes-implicit.todo": { - "description": "tasks can also be recurring" + "description": "tasks can also be recurring", + ## Independent feature (directly probed): the default marks it so the + ## node uses its own probed value rather than being derived from + ## .pending. + "default": {"support": "full"}, }, "search.recurrences.includes-implicit.todo.pending": { "description": "a future recurrence of a pending task should always be pending and appear in searches for pending tasks", @@ -332,6 +455,10 @@ class FeatureSet: }, "sync-token": { "description": "RFC6578 sync-collection reports are supported. Server provides sync tokens that can be used to efficiently retrieve only changed objects since last sync. Support can be 'full', 'fragile' (occasionally returns more content than expected), or 'unsupported'. Behaviour 'time-based' indicates second-precision tokens requiring sleep(1) between operations", + ## Independent feature (directly probed): the default marks it so the + ## node uses its own probed value rather than being derived from + ## .delete. + "default": {"support": "full"}, "links": ["https://datatracker.ietf.org/doc/html/rfc6578"], }, "sync-token.delete": { @@ -339,6 +466,10 @@ class FeatureSet: }, "scheduling": { "description": "Server supports CalDAV Scheduling (RFC6638). Detected via the presence of 'calendar-auto-schedule' in the DAV response header.", + ## Independent feature (directly probed via the DAV header): the default + ## marks it so the node uses its own probed value rather than being + ## derived from subfeatures such as .calendar-user-address-set. + "default": {"support": "full"}, "links": ["https://datatracker.ietf.org/doc/html/rfc6638"], }, "scheduling.mailbox": { @@ -390,6 +521,11 @@ class FeatureSet: }, "principal-search": { "description": "Server supports searching for principals (CalDAV users). Principal search may be restricted for privacy/security reasons on many servers. (not to be confused with get-current-user-principal)" + ## NB: genuine grouping node - 'supported' iff at least one search + ## method (.by-name / .list-all) works. The checker sets it directly + ## because that OR-semantics cannot be expressed by the library's + ## all-children-agree derivation; it deliberately has NO default so + ## that when all sub-searches fail the node is unsupported. }, "principal-search.by-name": { "description": "Server supports searching for principals by display name. Testing this properly requires setting up another user with a known name, so this check is not yet implemented" @@ -408,6 +544,10 @@ class FeatureSet: "save.duplicate-uid.cross-calendar": { "description": "Server allows events with the same UID to exist in different calendars and treats them as separate entities. Support can be 'full' (allowed), 'ungraceful' (rejected with error), or 'unsupported' (silently ignored or moved). Behaviour 'silently-ignored' means the duplicate is not saved but no error is thrown. Behaviour 'moved-instead-of-copied' means the event is moved from the original calendar to the new calendar (Zimbra behavior)" }, + "save.duplicate-event": { + "description": "Server allows two events with identical content but different UIDs to coexist in the same calendar. Some servers reject or de-duplicate such an event ('duplicates not allowed even with a different UID'), in which case this is 'unsupported' (silently dropped) or 'ungraceful' (rejected with an error). The default 'full' is the usual behaviour.", + "default": {"support": "full"}, + }, ## TODO: as for now, the tests will run towards the first calendar it will find, and most of the tests will assume the calendar is empty. This is bad. "test-calendar": { "type": "tests-behaviour", @@ -493,13 +633,14 @@ def copyFeatureSet(self, feature_set, collapse=True): UserWarning, stacklevel=3, ) + continue value = feature_set[feature] if feature not in self._server_features: self._server_features[feature] = {} server_node = self._server_features[feature] if isinstance(value, bool): server_node['support'] = "full" if value else "unsupported" - elif isinstance(value, str) and 'support' not in server_node: + elif isinstance(value, str): self._validate_support_level(value, feature) server_node['support'] = value elif isinstance(value, dict): @@ -541,44 +682,76 @@ def _collapse_key(self, feature_dict): def collapse(self): """ - If all subfeatures are the same, it should be collapsed into the parent - - Messy and complex logic :-( + Compact the stored feature set: a *grouping* parent (one without its own + explicit default) whose grouping children are all explicitly set to the + same status is replaced by a single entry on the parent, and the children + are dropped. + + The parent status comes from the single derivation path, + is_supported() -> _derive_from_subfeatures(). That path already: + * treats a node with an explicit default as an independent feature - + never derived/collapsed from its children (so e.g. save-load.mutable + stays "full" even when every child is "unsupported"), and + * ignores independent children (those with their own default) when + deriving a grouping parent. + collapse() adds only a losslessness check on top: it folds the children + in solely when every grouping child is explicitly set and matches the + derived value, so no per-child information is lost. """ - features = list(self._server_features.keys()) parents = set() - for feature in features: + for feature in self._server_features: if '.' in feature: parents.add(feature[:feature.rfind('.')]) - parents = list(parents) - ## Parents needs to be ordered by the number of dots. We proceed those with most dots first. - parents.sort(key = lambda x: (-x.count('.'), x)) - for parent in parents: + ## Deepest parents first, so a freshly collapsed child can feed its parent. + for parent in sorted(parents, key=lambda x: (-x.count('.'), x)): parent_info = self.find_feature(parent) - if len(parent_info['subfeatures']): - foo = self.is_supported(parent, return_type=dict, return_defaults=False) - if len(parent_info['subfeatures']) > 1 or foo is not None: - dont_collapse = False - foo_key = self._collapse_key(foo) if foo is not None else None - for sub in parent_info['subfeatures']: - bar = self._server_features.get(f"{parent}.{sub}") - if bar is None: - dont_collapse = True - break - bar_key = self._collapse_key(bar) - if foo is None: - foo = bar - foo_key = bar_key - elif bar_key != foo_key: - dont_collapse = True - break - if not dont_collapse: - if parent not in self._server_features: - self._server_features[parent] = {} - for sub in parent_info['subfeatures']: - self._server_features.pop(f"{parent}.{sub}") - self.copyFeatureSet({parent: foo}) + ## Independent node (its own explicit default) is never collapsed. + if 'default' in parent_info: + continue + + ## Independent children (their own default) are separate features: + ## neither folded in nor required to match. + grouping_children = [ + sub + for sub in parent_info['subfeatures'] + if 'default' not in self.find_feature(f"{parent}.{sub}") + ] + if not grouping_children: + continue + + derived = self.is_supported(parent, return_type=dict, return_defaults=False) + if derived is None: + continue + derived_key = self._collapse_key(derived) + + ## Lossless only if every grouping child is explicitly set and matches. + child_nodes = [self._server_features.get(f"{parent}.{sub}") for sub in grouping_children] + if any(node is None or self._collapse_key(node) != derived_key for node in child_nodes): + continue + + ## Folding sets the (previously unset) parent explicitly, which an + ## *independent* child (its own default) that is not itself explicitly + ## set would then inherit - changing its resolved status whenever its + ## default differs from the derived value. Skip the fold in that case + ## so is_supported() stays invariant under collapse(). (e.g. folding + ## save.duplicate-uid into save must not flip the independent sibling + ## save.duplicate-event from its default "full" to "ungraceful".) + independent_children = [ + sub + for sub in parent_info['subfeatures'] + if 'default' in self.find_feature(f"{parent}.{sub}") + ] + if any( + f"{parent}.{sub}" not in self._server_features + and self._collapse_key(self._default(f"{parent}.{sub}")) != derived_key + for sub in independent_children + ): + continue + + for sub in grouping_children: + self._server_features.pop(f"{parent}.{sub}", None) + self.copyFeatureSet({parent: derived}) def _default(self, feature_info): if isinstance(feature_info, str): @@ -619,7 +792,12 @@ def is_supported(self, feature, return_type=bool, return_defaults=True, accept_f if 'default' not in current_info: derived = self._derive_from_subfeatures(feature_, current_info, return_type, accept_fragile) if derived is not None: - return derived + # When visiting an ancestor node (feature_ != feature), only propagate + # the derived status downward if the *original* queried feature is also + # a grouping node (no explicit default). Independent features have their + # own explicit default and must not be overridden by a derived ancestor. + if feature_ == feature or 'default' not in feature_info: + return derived if '.' not in feature_: if not return_defaults: return None @@ -689,8 +867,11 @@ def _derive_from_subfeatures(self, feature, feature_info, return_type, accept_fr if has_positive: if all_same: derived_status = subfeature_statuses[0] + elif not is_complete: + # Incomplete mixed set: unset siblings might be unsupported; inconclusive + return None else: - # Mixed positive/negative → unknown + # All relevant children seen, but mixed positive/negative → unknown derived_status = 'unknown' elif is_complete and all_same: # All relevant subfeatures set, all the same negative status @@ -804,6 +985,67 @@ def dotted_feature_set_list(self, compact=False): ret[x] = feature.copy() return ret + ## Feature types that the server-tester cannot reliably probe and that + ## therefore must not be cross-checked against the declared config. + _UNCHECKABLE_FEATURE_TYPES = ( + "client-feature", + "server-observation", + "tests-behaviour", + "client-hints", + "server-peculiarity", + ) + + def compare(self, observed): + """Compare this *declared* (expected) feature set against an *observed* + feature set and return the list of mismatches. + + Each mismatch is a dict with keys ``feature``, ``expected`` and + ``observed`` holding the resolved (string) support levels that disagree. + + Only server-features are compared; anything resolving to ``fragile`` or + ``unknown`` on either side, and feature types the tester cannot probe + reliably (see ``_UNCHECKABLE_FEATURE_TYPES``), are ignored. + """ + ## Snapshot what the tester explicitly probed *before* compact=True + ## calls collapse(), which mutates _server_features by folding + ## subfeatures into their parent - making probed features look + ## untested. is_supported() still resolves the collapsed values + ## correctly afterwards via the parent. + checked_features = set(observed._server_features.keys()) + observed_dotted = observed.dotted_feature_set_list(compact=True) + expected_dotted = self.dotted_feature_set_list(compact=True) + + mismatches = [] + ## Iterate everything either side made an explicit statement about: + ## the compacted dotted dicts plus every feature the tester probed. + ## Probed features whose observed value equals the default are absent + ## from observed_dotted, yet may still conflict with a non-default + ## status the declared config inherits from a parent (e.g. Infomaniak + ## search.comp-type.optional vs an unsupported search.comp-type). + for feature in set(observed_dotted).union(expected_dotted).union(checked_features): + observation = observed.is_supported(feature, str) + expectation = self.is_supported(feature, str) + if "fragile" in (observation, expectation): + continue + if "unknown" in (observation, expectation): + continue + ## Skip features the tester never explicitly probed - the + ## observation would just be a default, not a real result. + if feature not in observed_dotted and feature not in checked_features: + continue + type_ = observed.find_feature(feature).get("type", "server-feature") + if type_ in self._UNCHECKABLE_FEATURE_TYPES: + continue + if expectation != observation: + mismatches.append( + { + "feature": feature, + "expected": expectation, + "observed": observation, + } + ) + return mismatches + #### OLD STYLE ## THE LIST BELOW IS TO BE REMOVED COMPLETELY. DO NOT USE IT. @@ -825,28 +1067,9 @@ def dotted_feature_set_list(self, compact=False): ## * Perhaps some more readable format should be considered (yaml?). ## * Consider how to get this into the documentation incompatibility_description = { - 'calendar_order': - """Server supports (nonstandard) calendar ordering property""", - - 'calendar_color': - """Server supports (nonstandard) calendar color property""", - - 'duplicates_not_allowed': - """Duplication of an event in the same calendar not allowed """ - """(even with different uid)""", - - 'event_by_url_is_broken': """A GET towards a valid calendar object resource URL will yield 404 (wtf?)""", - 'propfind_allprop_failure': - """The propfind test fails ... """ - """it asserts DAV:allprop response contains the text 'resourcetype', """ - """possibly this assert is wrong""", - - 'vtodo_datesearch_nodtstart_task_is_skipped': - """date searches for todo-items will not find tasks without a dtstart""", - 'vtodo_datesearch_nodtstart_task_is_skipped_in_closed_date_range': """only open-ended date searches for todo-items will find tasks without a dtstart""", @@ -866,24 +1089,21 @@ def dotted_feature_set_list(self, compact=False): """Events should be deleted before the calendar is deleted, """ """and/or deleting a calendar may not have immediate effect""", - 'no_overwrite': - """events cannot be edited""", - 'dav_not_supported': """when asked, the server may claim it doesn't support the DAV protocol. Observed by one baikal server, should be investigated more (TODO) and robur""", 'fastmail_buggy_noexpand_date_search': """The 'blissful anniversary' recurrent example event is returned when asked for a no-expand date search for some timestamps covering a completely different date""", - 'non_existing_raises_other': - """Robur raises AuthorizationError when trying to access a non-existing resource (while 404 is expected). Probably so one shouldn't probe a public name space?""", - 'robur_rrule_freq_yearly_expands_monthly': """Robur expands a yearly event into a monthly event. I believe I've reported this one upstream at some point, but can't find back to it""", } xandikos = { + ## Genuinely returns matching objects for a comp-type-less query that carries + ## a time-range (verified: the event is returned, not just "no error"). + "search.time-range.comp-type-optional": {"support": "full"}, ## Principal property search returns 403 (not implemented) "principal-search": "ungraceful", @@ -900,6 +1120,9 @@ def dotted_feature_set_list(self, compact=False): ## There is much development going on at Radicale as of summar 2025, ## so I'm expecting this list to shrink a lot soon. radicale = { + ## Genuinely returns matching objects for a comp-type-less query that carries + ## a time-range (verified: the event is returned, not just "no error"). + "search.time-range.comp-type-optional": {"support": "full"}, "search.is-not-defined": {"support": "full"}, "search.text.case-sensitive": {"support": "unsupported"}, "search.recurrences.includes-implicit.todo.pending": {"support": "fragile", "behaviour": "inconsistent results between runs"}, @@ -909,11 +1132,9 @@ def dotted_feature_set_list(self, compact=False): ## this only applies for very simple installations "auto-connect.url": {"domain": "localhost", "scheme": "http", "basepath": "/"}, "scheduling": {"support": "unsupported"}, - 'old_flags': [ - ## extra features not specified in RFC4791 - "calendar_order", - "calendar_color" - ] + ## extra properties not specified in RFC4791/RFC5545 + "calendar-color": {"support": "full"}, + "calendar-order": {"support": "full"}, } ## Be aware that nextcloud by default have different rate limits, including how often a user is allowed to create a new calendar. This may break test runs badly. @@ -921,8 +1142,15 @@ def dotted_feature_set_list(self, compact=False): 'auto-connect.url': { 'basepath': '/remote.php/dav', }, - ## I'm surprised, I'm quite sure this was reported ungraceful earlier. Passed with caldav commit a98d50490b872e9b9d8e93e2e401c936ad193003, caldav server checker commit 3cae24cf99da1702b851b5a74a9b88c8e5317dad 2026-02-15. The commit 3cae24cf99da1702b851b5a74a9b88c8e5317dad was however development done on the wrong branch and has been force-pushed awway. It was again observed ungraceful at commits be26d42b1ca3ff3b4fd183761b4a9b024ce12b84 / 537a23b145487006bb987dee5ab9e00cdebb0492 - 'search.comp-type.optional': {'support': 'ungraceful'}, + ## Historically this flip-flopped between "ungraceful" and "full" - that + ## instability was a checker bug (https://github.com/python-caldav/caldav/issues/681): + ## the comp-type.optional probe used to send a comp-type-less query carrying a + ## time-range, which SabreDAV rejects (the time-range belongs in a VEVENT/... + ## comp-filter, not under VCALENDAR). Now that the probe omits the time-range, + ## Nextcloud correctly accepts the bare comp-type-less query. The time-range + ## variant is tracked separately as search.time-range.comp-type-optional + ## (unsupported on SabreDAV, the default). + 'search.comp-type.optional': {'support': 'full'}, 'search.recurrences.expanded.todo': {'support': 'unsupported'}, "search.recurrences.includes-implicit.infinite-scope": False, 'delete-calendar': { @@ -970,13 +1198,42 @@ def dotted_feature_set_list(self, compact=False): ## Zimbra is not very good at it's caldav support zimbra = { 'auto-connect.url': {'basepath': '/dav/'}, + ## Genuinely returns matching objects for a comp-type-less query that carries + ## a time-range (verified: the event is returned, not just "no error"). + 'search.time-range.comp-type-optional': {'support': 'full'}, 'delete-calendar': {'support': 'fragile', 'behaviour': 'may move to trashbin instead of deleting immediately'}, ## This is a zimbra bug when creating calendars with a display ## name. Now mitigated in the calendar creation code. #'save-load.get-by-url': {'support': 'fragile', 'behaviour': '404 most of the time - but sometimes 200. Weird, should be investigated more'}, ## Zimbra treats same-UID events across calendars as aliases of the same event 'save.duplicate-uid.cross-calendar': {'support': 'unsupported'}, - 'create-calendar.set-displayname': {'support': 'unsupported'}, + ## Zimbra DOES apply a display name set at creation (the name sticks, so + ## set-displayname is 'full') - but it couples the display name to the + ## calendar URL. MKCALENDAR lands the calendar at the requested cal_id path; + ## the display name is then applied by a follow-up PROPPATCH, which Zimbra + ## implements as a rename that MOVES the collection: the canonical URL + ## relocates to a display-name-derived path (verified deterministic with a + ## unique name against zcs-foss:latest). + ## + ## So create-calendar.stable-url is 'unsupported': is_supported() returns + ## False, and Calendar._create() therefore discovers and adopts the canonical + ## URL after creation (re-pointing self.url), instead of dropping the display + ## name. This keeps the calendar fully usable (name retained AND object URLs + ## resolve) on both Zimbra and OX with no per-server branching. + ## + ## Two Zimbra quirks worth recording (and someday probing for explicitly), + ## mirrored in caldav-server-tester's CheckMakeDeleteCalendar: + ## * The URL is only unstable when a display name is supplied at creation; + ## a nameless MKCALENDAR stays put at the requested cal_id. + ## * Zimbra keeps a collection-level ALIAS at the original cal_id (PROPFIND/ + ## REPORT on it succeed), yet a GET on a child object under that alias + ## (...//.ics) 404s - the object is only retrievable under + ## the canonical relocated URL. So "the calendar collection is reachable + ## at cal_id" does NOT imply "objects are reachable at cal_id"; the canonical + ## URL must be used. (This also explains the old save-load.get-by-url + ## "404 most of the time but sometimes 200" observation.) + 'create-calendar.set-displayname': {'support': 'full'}, + 'create-calendar.stable-url': {'support': 'unsupported', 'behaviour': 'a display name set at creation relocates the collection to a display-name-derived canonical URL; a collection alias lingers at the requested cal_id but child object GETs under it 404'}, 'save-load.todo.mixed-calendar': {'support': 'unsupported'}, 'save-load.todo.recurrences.count': {'support': 'unsupported'}, ## This is a new problem? 'save-load.journal': {'support': 'ungraceful'}, @@ -987,7 +1244,10 @@ def dotted_feature_set_list(self, compact=False): # sometimes throws a 500 'search.text.category': {'support': 'ungraceful'}, 'search.recurrences.expanded.todo': { "support": "unsupported" }, - 'search.comp-type.optional': {'support': 'fragile'}, ## TODO: more research on this, looks like a bug in the checker, + ## was 'fragile' - that was the checker bug (it compared a comp-type-less + ## search against cnt, which counts objects stored in a separate + ## task/journal calendar). Confirmed full 2026-06-06. + 'search.comp-type.optional': {'support': 'full'}, 'search.time-range.alarm': {'support': 'unsupported'}, 'principal-search': "unsupported", ## Zimbra implements server-side automatic scheduling: invitations are @@ -1011,11 +1271,15 @@ def dotted_feature_set_list(self, compact=False): ## TODO: I just discovered that when searching for a date some ## years after a recurring daily event was made, the event does ## not appear. - - ## extra features not specified in RFC5545 - "calendar_order", - "calendar_color" - ] + ], + ## extra properties not specified in RFC4791/RFC5545. Zimbra stores + ## calendar-order, and stores calendar-color only when set as a hex value - + ## it rejects/ignores a colour name like "blue". (The old 'calendar_color' + ## flag was never actually exercised, because testSetCalendarProperties skips + ## on Zimbra: setting a display name relocates the calendar.) + "calendar-color": {"support": "unsupported"}, + "calendar-color.hex": {"support": "full"}, + "calendar-order": {"support": "full"}, } bedework = { @@ -1040,7 +1304,14 @@ def dotted_feature_set_list(self, compact=False): "search.recurrences": False, "sync-token": { "support": "fragile" }, 'search.comp-type': {'support': 'broken', 'behaviour': 'Server returns everything when searching for events and nothing when searching for todos'}, - 'search.comp-type.optional': {'support': 'ungraceful'}, + ## was 'ungraceful' - that was the checker bug (cnt counted the separately + ## stored journal); confirmed full 2026-06-06. + 'search.comp-type.optional': {'support': 'full'}, + ## Flaps between full and unsupported across runs - the comp-type-less + ## time-range query intermittently returns the in-range object vs nothing, + ## most likely the search-cache delay above. Marked fragile so the checker + ## skips it. Observed 2026-06-06. + 'search.time-range.comp-type-optional': {'support': 'fragile'}, 'search.is-not-defined.dtend': False, "principal-search": { "support": "ungraceful" }, ## Bedework hides past non-recurring events from REPORT without a time-range filter, @@ -1056,11 +1327,11 @@ def dotted_feature_set_list(self, compact=False): ## TODO: play with this and see if it's needed 'save-load.icalendar.related-to': {'support': 'broken', 'behaviour': 'first RELATED-TO line is preserved but subsequent RELATED-TO lines are stripped'}, - 'old_flags': [ - 'propfind_allprop_failure', - 'duplicates_not_allowed', - ], - + ## Bedework omits DAV:resourcetype from an allprop PROPFIND response. + "propfind.allprop.resourcetype": {"support": "unsupported"}, + ## (The old 'duplicates_not_allowed' flag was stale: Bedework does store a + ## second event with the same content under a different UID, so + ## save.duplicate-event is left at the default "full".) } synology = { @@ -1071,7 +1342,8 @@ def dotted_feature_set_list(self, compact=False): 'search.is-not-defined': {'support': 'fragile', 'behaviour': 'works for CLASS but not for CATEGORIES'}, 'search.text.case-sensitive': {'support': 'unsupported'}, 'search.time-range.alarm': {'support': 'unsupported'}, - 'old_flags': ['vtodo_datesearch_nodtstart_task_is_skipped'], + ## Synology skips VTODOs without DTSTART in date-range searches. + 'search.time-range.todo.no-dtstart': {'support': 'unsupported'}, 'test-calendar': {'cleanup-regime': 'wipe-calendar'}, 'scheduling.schedule-tag': False, 'scheduling.mailbox.inbox-delivery': False, @@ -1082,7 +1354,9 @@ def dotted_feature_set_list(self, compact=False): # into their calendar. "scheduling.schedule-tag": False, "http.multiplexing": "fragile", ## ref https://github.com/python-caldav/caldav/issues/564 - 'search.comp-type.optional': {'support': 'ungraceful'}, + ## was 'ungraceful' - that was the checker bug (cnt counted the journal that + ## SabreDAV stores in a separate calendar); confirmed full 2026-06-06. + 'search.comp-type.optional': {'support': 'full'}, 'search.recurrences.expanded.todo': {'support': 'unsupported'}, 'search.recurrences.includes-implicit.todo': {'support': 'unsupported'}, "search.recurrences.includes-implicit.infinite-scope": False, @@ -1091,11 +1365,9 @@ def dotted_feature_set_list(self, compact=False): 'principal-search.by-name.self': {'support': 'unsupported'}, 'principal-search.list-all': {'support': 'ungraceful'}, #'sync-token.delete': {'support': 'unsupported'}, ## Perhaps on some older servers? - 'old_flags': [ - ## extra features not specified in RFC5545 - "calendar_order", - "calendar_color", - ], + ## extra properties not specified in RFC4791/RFC5545 + "calendar-color": {"support": "full"}, + "calendar-order": {"support": "full"}, ## I'm surprised, I'm quite sure this was passing earlier. Caldav commit a98d50490b872e9b9d8e93e2e401c936ad193003, caldav server checker commit 3cae24cf99da1702b851b5a74a9b88c8e5317dad 'search.combined-is-logical-and': False } ## TODO: testPrincipals, testWrongAuthType, testTodoDatesearch fails @@ -1107,7 +1379,10 @@ def dotted_feature_set_list(self, compact=False): } cyrus = { - "search.comp-type.optional": {"support": "ungraceful"}, + ## A bare comp-type-less query is accepted; the previous "ungraceful" was a + ## checker bug where the probe carried a time-range + ## (https://github.com/python-caldav/caldav/issues/681). + "search.comp-type.optional": {"support": "full"}, "search.recurrences.includes-implicit.infinite-scope": False, "search.time-range.alarm": {"support": "ungraceful"}, 'principal-search': {'support': 'ungraceful'}, @@ -1146,29 +1421,41 @@ def dotted_feature_set_list(self, compact=False): # DAViCal delivers iTIP notifications to the attendee inbox AND auto-schedules # into their calendar. "scheduling.schedule-tag": False, - "search.comp-type.optional": { "support": "fragile" }, + ## was 'fragile' - that was the checker bug (cnt mismatch); confirmed full 2026-06-06. + "search.comp-type.optional": { "support": "full" }, + ## Genuinely returns matching objects for a comp-type-less query that carries + ## a time-range (verified: the event is returned, not just "no error"). + "search.time-range.comp-type-optional": { "support": "full" }, "search.time-range.alarm": { "support": "unsupported" }, 'sync-token': {'support': 'fragile'}, 'principal-search': {'support': 'unsupported'}, 'principal-search.list-all': {'support': 'unsupported'}, + ## DAViCal skips VTODOs without DTSTART in date-range searches. + 'search.time-range.todo.no-dtstart': {'support': 'unsupported'}, "old_flags": [ #'no_journal', ## it threw a 500 internal server error! ## for old versions #'nofreebusy', ## for old versions ## 'fragile_sync_tokens' removed - covered by 'sync-token': {'support': 'fragile'} - 'vtodo_datesearch_nodtstart_task_is_skipped', ## no issue raised yet - 'calendar_color', - 'calendar_order', 'vtodo_datesearch_notime_task_is_skipped', ], + ## extra properties not specified in RFC4791/RFC5545 + "calendar-color": {"support": "full"}, + "calendar-order": {"support": "full"}, } sogo = { "scheduling.schedule-tag": False, "scheduling.mailbox.inbox-delivery": False, + ## SOGo rejects the calendar-color property with an error (left at the + ## default "fragile" - rejecting a nonstandard extension is fine). It + ## accepts calendar-order but echoes back a server-computed position rather + ## than the value that was set, so that property is effectively read-only. + "calendar-order": {"support": "broken", "behaviour": "read-only; server returns its own calendar position rather than the value set"}, ## I'm surprised, I'm quite sure this was passing earlier. reported unsupported with caldav commit a98d50490b872e9b9d8e93e2e401c936ad193003, caldav server checker commit 3cae24cf99da1702b851b5a74a9b88c8e5317dad 2026-02-15 "search.text.category": False, - "search.time-range.event.old-dates": False, - "search.time-range.todo.old-dates": False, + ## old-date time-range search works (probe found, definite-future object + ## correctly excluded); the earlier "False" was an artifact of the old + ## count==1 check, which a next-year open-start DUE-only task inflated. "save-load.journal": {"support": "ungraceful"}, "search.is-not-defined": {"support": "unsupported"}, "search.text.case-sensitive": { @@ -1180,9 +1467,13 @@ def dotted_feature_set_list(self, compact=False): "search.time-range.alarm": { "support": "unsupported" }, - ## was unsupported. reported ungraceful with caldav commit a98d50490b872e9b9d8e93e2e401c936ad193003, caldav server checker commit 3cae24cf99da1702b851b5a74a9b88c8e5317dad 2026-02-15 + ## A comp-type-less query returns nothing - with or without a time-range - + ## so both search.comp-type.optional and search.time-range.comp-type-optional + ## are unsupported (the latter is the default). The previous "ungraceful" was + ## a checker bug where the comp-type.optional probe carried a time-range that + ## SabreDAV-likes reject (https://github.com/python-caldav/caldav/issues/681). "search.comp-type.optional": { - "support": "ungraceful" + "support": "unsupported" }, ## includes-implicit.todo has been observed as both supported and unsupported ## across different test runs. Other includes-implicit children are unsupported. @@ -1260,9 +1551,12 @@ def dotted_feature_set_list(self, compact=False): 'principal-search': {'support': 'ungraceful'}, 'freebusy-query': {'support': 'ungraceful'}, "scheduling": {"support": "unsupported"}, - 'old_flags': [ - 'non_existing_raises_other', ## AuthorizationError instead of NotFoundError - ], + ## Robur answers 403 (AuthorizationError) instead of 404 (NotFoundError) when + ## looking up a non-existing resource - probably to avoid leaking whether a + ## resource exists. (Not re-probed during this migration: the Robur test + ## server was down; value carried over from the old 'non_existing_raises_other' + ## flag.) + 'non-existing-raises-not-found': {'support': 'unsupported', 'behaviour': 'raises AuthorizationError (403) instead of NotFoundError (404)'}, 'save-load.icalendar.related-to': {'support': 'unsupported'}, 'test-calendar': {'cleanup-regime': 'wipe-calendar'}, "sync-token": {"support": "ungraceful"}, @@ -1330,11 +1624,12 @@ def dotted_feature_set_list(self, compact=False): "principal-search.by-name.self": {"support": "unsupported"}, "principal-search": {"support": "ungraceful"}, "save-load.journal.mixed-calendar": {"support": "unsupported"}, - "search.comp-type.optional": {"support": "ungraceful"}, - "old_flags": [ - "calendar_order", - "calendar_color", - ], + ## was 'ungraceful' - that was the checker bug (cnt counted the journal that + ## SabreDAV stores in a separate calendar); confirmed full 2026-06-06. + "search.comp-type.optional": {"support": "full"}, + ## extra properties not specified in RFC4791/RFC5545 + "calendar-color": {"support": "full"}, + "calendar-order": {"support": "full"}, ## I'm surprised, I'm quite sure this was passing earlier. Caldav commit a98d50490b872e9b9d8e93e2e401c936ad193003, caldav server checker commit 3cae24cf99da1702b851b5a74a9b88c8e5317dad 'search.combined-is-logical-and': False } @@ -1356,25 +1651,35 @@ def dotted_feature_set_list(self, compact=False): "save.duplicate-uid.cross-calendar": {"support": "ungraceful"}, # CCS rejects multi-instance VTODOs (thisandfuture recurring completion) "save-load.todo.recurrences.thisandfuture": {"support": "unsupported"}, - "search.comp-type.optional": {"support": "ungraceful"}, - ## "full" observed, 70938dc1cbb6a839978eee4315699746d38ee5f0/3cae24cf99da1702b851b5a74a9b88c8e5317dad, 2026-02-17. - ## However, this may be due to mess with the caldav-server-checker branches. "unsupported" again at be26d42b1ca3ff3b4fd183761b4a9b024ce12b84 / 537a23b145487006bb987dee5ab9e00cdebb0492 + ## was 'ungraceful' - that was the checker bug (cnt mismatch: it counted a + ## journal object that CCS could not store, so the comp-type-less count never + ## matched). Confirmed full 2026-06-06. + ## ("full" had also been observed 2026-02-17, then "unsupported"/"ungraceful" + ## - all that flapping was the same checker bug, now fixed.) + "search.comp-type.optional": {"support": "full"}, "search.text.case-sensitive": {"support": "unsupported"}, "search.time-range.event": {"support": "full"}, "search.time-range.event.old-dates": {"support": "ungraceful"}, "search.time-range.todo": {"support": "full"}, "search.time-range.todo.old-dates": {"support": "ungraceful"}, - "search.time-range.open": {"support": "ungraceful"}, + ## open-ended time-range searches work with the near-future fixtures; CCS only + ## rejected them (ungraceful) for the old year-2000 range, so the leaves default + ## to "full" (a grouping "search.time-range.open: ungraceful" was removed here). "search.time-range.alarm": {"support": "unsupported"}, - "search.recurrences": {"support": "unsupported"}, + ## Recurrence expansion actually works within the (near-future) search window; + ## this was previously reported "unsupported" only because the test fixtures + ## lived in year 2000, which CCS's min-date-time restriction hid. Only infinite + ## scope (far-future) and server-side VTODO expansion remain unsupported. + "search.recurrences.includes-implicit.infinite-scope": {"support": "unsupported"}, + "search.recurrences.expanded.todo": {"support": "unsupported"}, "principal-search": {"support": "unsupported"}, # Ephemeral Docker container: wipe objects (avoids UID conflicts across calendars) "test-calendar": {"cleanup-regime": "wipe-calendar"}, - ## Did pass earlier, ungraceful at be26d42b1ca3ff3b4fd183761b4a9b024ce12b84 / 537a23b145487006bb987dee5ab9e00cdebb0492 - 'freebusy-query': {'support': 'ungraceful'}, - "old_flags": [ - "propfind_allprop_failure", - ], + ## freebusy-query works with the near-future fixtures; CCS rejected the + ## year-2000 range with an error, so this defaults to "full" now. + ## (The old 'propfind_allprop_failure' flag was stale: CCS does return + ## DAV:resourcetype in an allprop PROPFIND, so propfind.allprop.resourcetype + ## is left at the default "full".) } ## Stalwart - all-in-one mail & collaboration server (CalDAV added 2024/2025) @@ -1390,24 +1695,39 @@ def dotted_feature_set_list(self, compact=False): 'create-calendar.auto': True, 'principal-search': {'support': 'ungraceful'}, 'search.time-range.alarm': False, + ## Stalwart accepts comp-type-less queries fully, including the time-range + ## and prop-filter variants (both unsupported on most other servers). + ## Confirmed 2026-06-06. + 'search.time-range.comp-type-optional': {'support': 'full'}, + 'search.text.comp-type-optional': {'support': 'full'}, ## Stalwart supports implicit recurrence for datetime events but not for ## all-day (VALUE=DATE) recurring events in time-range searches. 'search.recurrences.includes-implicit.event': {'support': 'fragile', 'behaviour': 'broken for all-day (VALUE=DATE) events'}, ## Stalwart returns the recurring todo in search results but doesn't return the ## RRULE intact, so client-side expansion can't expand it to specific occurrences. 'search.recurrences.includes-implicit.todo': {'support': 'fragile'}, - ## Stalwart correctly handles exceptions in server-side CALDAV:expand (observed supported). - ## Stalwart stores master+exception VEVENTs as a single resource with 2 VEVENTs. + ## Stalwart stores master+exception VEVENTs as a single resource with 2 VEVENTs, + ## so client-side expand of the recurrence set works. 'save-load.event.recurrences.exception': {'support': 'full'}, + ## ...but server-side CALDAV:expand only suppresses the exception-overridden + ## occurrence when SEQUENCE is absent. With SEQUENCE present (as real clients + ## always emit) it returns both the original occurrence and the override. + ## Detected by the server-tester's csc_monthly_recurring_with_exception_seq fixture. + 'search.recurrences.expanded.exception': { + 'support': 'fragile', + 'behaviour': 'server-side expand fails to suppress the exception-overridden occurrence when SEQUENCE is present', + }, 'search.time-range.open': True, ## Stalwart delivers iTIP notifications to the attendee inbox AND auto-schedules ## into their calendar (verified by running CheckSchedulingInboxDelivery). "scheduling.mailbox.inbox-delivery": True, "scheduling.auto-schedule": True, - 'old_flags': [ - ## Stalwart does not return VTODO items without DTSTART in date searches - 'vtodo_datesearch_nodtstart_task_is_skipped', - ], + ## Stalwart's handling of DTSTART-less VTODOs in date searches is date + ## dependent: a near-future DUE-only task is returned (the server-tester + ## probe sees 'full'), but the old-date fixtures used by testTodoDatesearch + ## are skipped. Marked 'fragile' so the checker skips it and the integration + ## test (is_supported -> False) still treats the old-date task as skipped. + 'search.time-range.todo.no-dtstart': {'support': 'fragile'}, } ## Lots of transient problems with purelymail @@ -1494,12 +1814,24 @@ def dotted_feature_set_list(self, compact=False): ] } -## https://www.open-xchange.com/ +## https://ox.io/ ## OX App Suite CalDAV served at /caldav/ (Apache proxies to /servlet/dav/caldav on port 8009). ## The Docker image must be built locally before use (see tests/docker-test-servers/ox/build.sh). ox = { - ## Renaming a calendar after creation via PROPPATCH is not supported - 'create-calendar.set-displayname': {'support': 'unsupported'}, + ## Renaming a calendar after creation via PROPPATCH is not supported, but + ## setting the display name AT creation time is - and that's what the probe + ## tests. Was 'unsupported' (conflated the two operations, and masked by the + ## checker's display-name-lookup bug). Confirmed full 2026-06-07. + 'create-calendar.set-displayname': {'support': 'full'}, + ## OX gives EVERY calendar an opaque internal 'cal://0/NNN' canonical URL + ## (base64-encoded in the path, e.g. /caldav/Y2FsOi8vMC8xMzYw/), whether or + ## not a display name is set. The requested cal_id does resolve as a usable + ## alias (object GETs under it work, unlike Zimbra), but the canonical URL + ## still differs from the requested URL - so under the URL-stability semantics + ## this is 'unsupported', exactly like Zimbra and with no special-casing: the + ## library discovers and adopts the canonical URL after creation. (The + ## display name itself sticks, so create-calendar.set-displayname is 'full'.) + 'create-calendar.stable-url': {'support': 'unsupported', 'behaviour': "the calendar's canonical URL is an opaque cal://0/NNN (base64 path segment) that differs from the requested cal_id; the cal_id alias is usable but clients should adopt the canonical URL"}, ## VTODOs must be in a dedicated VTODO-only calendar; mixed calendars not supported 'save-load.todo.mixed-calendar': {'support': 'unsupported'}, ## Basic VTODO support works fine; only recurrences are broken @@ -1508,20 +1840,63 @@ def dotted_feature_set_list(self, compact=False): 'save-load.todo.recurrences': {'support': 'ungraceful'}, ## VJOURNAL is not supported 'save-load.journal': {'support': 'unsupported'}, + ## OX exposes the calendar both under its display name and under an internal + ## "cal://0/NNN" id, so objects looked up via REPORT come back under a + ## different calendar URL than the one used to PUT them (GET on the original + ## URL still works via an alias). + 'save-load.stable-url': {'support': 'unsupported'}, + ## OX enforces optimistic concurrency: a no-If-Match overwrite PUT is rejected + ## with 409 Conflict (etag-conditional save() still works). + 'save-load.mutable.if-match-optional': {'support': 'unsupported'}, + ## OX forbids changing an attendee's PARTSTAT via a direct PUT (403 Forbidden + ## even with a matching etag); it must go through iTIP scheduling. + 'save-load.mutable.attendee-partstat': {'support': 'unsupported'}, ## Search limitations 'search.time-range.event.old-dates': {'support': 'unsupported'}, 'search.time-range.todo.old-dates': {'support': 'unsupported'}, 'search.time-range.alarm': {'support': 'unsupported'}, 'search.unlimited-time-range': {'support': 'broken'}, - 'search.comp-type.optional': {'support': 'ungraceful'}, - 'search.text': {'support': 'unsupported'}, + ## was 'ungraceful' - that was the checker bug (cnt mismatch across the + ## separate VTODO calendar); confirmed full 2026-06-06. + 'search.comp-type.optional': {'support': 'full'}, + ## OX silently ignores the CALDAV comp-filter: a calendar-query that + ## specifies a component type returns the calendar's whole contents + ## regardless of the requested type (a VEVENT-calendar answers a VTODO query + ## with its VEVENT, and vice versa). No right-typed objects are dropped, so + ## the library recovers the correct result by post-filtering - hence + ## "unsupported" (silently ignored), not "broken". Confirmed by direct probe + ## 2026-06-09. Contrast bedework, which drops the todos (data loss = broken). + 'search.comp-type': {'support': 'unsupported'}, + ## Text search (case-sensitive, case-insensitive, substring) now works in OX. + ## Confirmed full 2026-06-13. Category search remains unsupported. + 'search.text': {'support': 'full'}, 'search.text.category': {'support': 'unsupported'}, - 'search.text.case-sensitive': {'support': 'unsupported'}, - 'search.text.case-insensitive': {'support': 'unsupported'}, - ## Recurrence searching broken (sliding window + old-dates limitation) - 'search.recurrences.includes-implicit': {'support': 'unsupported'}, + ## Recurrence searching: the sliding window hides far-past/far-future + ## occurrences, but implicit expansion of *datetime* events and server-side + ## expansion of exceptions work within the window (detectable now that the + ## fixtures are in the near future rather than year 2000). VTODO recurrence, + ## datetime-event server-side expansion, and infinite scope remain unsupported. + ## (event and exception expansion are left at the default "full".) + 'search.recurrences.includes-implicit.todo': {'support': 'unsupported'}, 'search.recurrences.includes-implicit.todo.pending': {'support': 'unsupported'}, - 'search.recurrences.expanded': {'support': 'unsupported'}, + 'search.recurrences.includes-implicit.infinite-scope': {'support': 'unsupported'}, + 'search.recurrences.expanded.event': {'support': 'unsupported'}, + 'search.recurrences.expanded.todo': {'support': 'unsupported'}, + ## Rescheduling the whole series (changing the master DTSTART) is rejected with + ## 409 Conflict once detached exceptions exist - even with a matching If-Match + ## etag. Shifting the DTSTART of an exception-free recurring event still works. + ## Confirmed by direct probe 2026-06-14. + 'save-load.event.recurrences.exception.reschedule': {'support': 'unsupported'}, + ## OX ignores the time-range on VTODO queries and returns every task + 'search.time-range.todo.strict': {'support': 'broken'}, + ## OX silently ignores the is-not-defined prop-filter and returns the whole + ## calendar regardless (confirmed by direct probe 2026-06-09: a no_category + ## search still returns the categorised event; a no_class search still + ## returns the CONFIDENTIAL event). Same "filter ignored" behaviour as + ## search.comp-type above - silently ignored, hence unsupported. + 'search.is-not-defined': {'support': 'unsupported'}, + 'search.is-not-defined.category': {'support': 'unsupported'}, + 'search.is-not-defined.class': {'support': 'unsupported'}, ## is-not-defined for DTEND is not supported 'search.is-not-defined.dtend': {'support': 'unsupported'}, ## Freebusy queries are not supported (returns 400) @@ -1538,9 +1913,63 @@ def dotted_feature_set_list(self, compact=False): "scheduling.freebusy-query": "ungraceful", 'search.time-range.open.start': "broken", 'search.time-range.open.end': True, - ## time-range.open is "broken", while time-range.open.start.duration is "unsupported"? - ## this may possibly be some problems with the checker rather than with Ox - 'search.time-range.open.start.duration': "unsupported" + ## DTSTART+DURATION components ARE found by an overlapping time-range search: + ## confirmed by direct probe 2026-06-09 for VEVENT, and the VTODO duration + ## fixture is returned too. The VTODO time-range is not honoured strictly + ## (out-of-range tasks leak in - tracked separately as + ## search.time-range.todo.strict=broken), so the checker now treats the VTODO + ## duration probe as inconclusive rather than a failure and judges this + ## feature from the conclusive VEVENT result. (Previously mis-reported as a + ## VTODO/VEVENT asymmetry; see the old "checker problem" note here.) + 'search.time-range.open.start.duration': {'support': 'full'}, +} + +## Infomaniak (https://www.infomaniak.com/) - kSuite calendar, CalDAV served at +## https://sync.infomaniak.com/ (/.well-known/caldav redirects there). Runs +## SabreDAV 4.3.1. Profiled 2026-06-15 against a freshly created dedicated +## calendar; save-load and most search features work well. +infomaniak = { + ## SabreDAV processes writes asynchronously - MKCALENDAR/PUT/DELETE return + ## before the change is queryable, so an immediate read-back 404s or returns + ## stale data for several seconds. This is server-wide (not just searches), + ## so we sleep after every write rather than only before searches. + 'write-delay': {'behaviour': 'delay', 'delay': 16}, + ## VJOURNAL is not supported. + 'save-load.journal': {'support': 'unsupported'}, + ## Calendar colour/order work once the post-write delay is honoured (the + ## hex form is normalised, e.g. '#FF0000FF' is stored as '#ff0000'). These + ## previously looked 'broken' (read-only): a read-back issued too soon + ## returned the stale value, an artifact of the asynchronous writes above. + ## Set explicitly to 'full' since the feature default is the weaker 'fragile'. + 'calendar-color': {'support': 'full'}, + 'calendar-color.hex': {'support': 'full'}, + 'calendar-order': {'support': 'full'}, + ## The CALDAV comp-filter is silently ignored: a calendar-query that requests + ## one component type returns the calendar's whole contents regardless (a + ## VJOURNAL query returned a VEVENT). No right-typed objects are dropped, so + ## the library recovers by post-filtering - hence "unsupported", not "broken". + 'search.comp-type': {'support': 'unsupported', 'behaviour': 'comp-filter silently ignored - returns the whole calendar regardless of requested component type'}, + ## Because the comp-filter is ignored, omitting it (which the RFC permits) + ## also returns the whole calendar - so the "optional comp-type" behaviour + ## works. The parent is 'unsupported', so this child must say so explicitly, + ## otherwise it inherits 'unsupported' and disagrees with the observation. + 'search.comp-type.optional': {'support': 'full'}, + ## A combined (logical-AND) filter is not honoured. + 'search.combined-is-logical-and': {'support': 'unsupported'}, + ## VTODO recurrence searching is not supported (datetime VEVENT recurrence + ## search, including server-side expand and infinite scope, works fine). + 'search.recurrences.includes-implicit.todo': {'support': 'unsupported'}, + 'search.recurrences.includes-implicit.todo.pending': {'support': 'unsupported'}, + 'search.recurrences.expanded.todo': {'support': 'unsupported'}, + ## Scheduling is advertised and the calendar-user-address-set and scheduling + ## mailbox are present, but the server never returns a Schedule-Tag (neither + ## on GET nor via PROPFIND). + 'scheduling.schedule-tag': {'support': 'unsupported', 'behaviour': 'no Schedule-Tag returned on GET or via PROPFIND'}, + 'scheduling.schedule-tag.stable-partstat': {'support': 'unsupported'}, + ## Principal search is effectively unsupported (lists nothing / errors out). + 'principal-search': {'support': 'ungraceful'}, + 'principal-search.by-name.self': {'support': 'unsupported'}, + 'principal-search.list-all': {'support': 'ungraceful'}, } # fmt: on diff --git a/caldav/config.py b/caldav/config.py index 05c8af72..451d0cfc 100644 --- a/caldav/config.py +++ b/caldav/config.py @@ -33,6 +33,8 @@ def expand_config_section(config, section="default", blacklist=None): ## If it's not a glob-pattern ... if set(section).isdisjoint(set("[*?")): + if section not in config: + return [] ## If it's referring to a "meta section" with the "contains" keyword if "contains" in config[section]: results = [] @@ -47,7 +49,7 @@ def expand_config_section(config, section="default", blacklist=None): return results else: ## Disabled sections should be ignored - if config.get("section", {}).get("disable", False): + if config.get(section, {}).get("disable", False): return [] ## NORMAL CASE - return [ section ] @@ -181,7 +183,7 @@ def resolve_features(features): feature_name = features if feature_name.startswith("compatibility_hints."): feature_name = feature_name[len("compatibility_hints.") :] - return getattr(caldav.compatibility_hints, feature_name) + return copy.deepcopy(getattr(caldav.compatibility_hints, feature_name)) if isinstance(features, dict) and "base" in features: base_name = features["base"] if isinstance(base_name, str): @@ -263,15 +265,15 @@ def get_connection_params( or None if no configuration found. """ # 1. Explicit parameters take highest priority - if explicit_params: - # Filter to valid connection keys - conn_params = {k: v for k, v in explicit_params.items() if k in CONNKEYS} - if conn_params.get("url") or conn_params.get("features"): - # Return when URL is given, or when features are given (the - # client constructor resolves URL from auto-connect.url hints - # via _auto_url()). Don't fall through to env vars/config - # files when the caller explicitly provided connection info. - return conn_params + explicit_conn = ( + {k: v for k, v in explicit_params.items() if k in CONNKEYS} if explicit_params else {} + ) + if explicit_conn.get("url") or explicit_conn.get("features"): + # Return when URL is given, or when features are given (the + # client constructor resolves URL from auto-connect.url hints + # via _auto_url()). Don't fall through to env vars/config + # files when the caller explicitly provided connection info. + return explicit_conn # Check for config file path from environment early (needed for test server config too) if environment: @@ -297,14 +299,19 @@ def get_connection_params( if environment: conn_params = _get_env_config() if conn_params: + conn_params.update(explicit_conn) return conn_params # 4. Config file if check_config_file: conn_params = _get_file_config(config_file, config_section) if conn_params: + conn_params.update(explicit_conn) return conn_params + # No env/config source matched. At this point explicit_conn has neither + # 'url' nor 'features' (those return early above), so it cannot produce a + # connectable client — treat it as "no configuration found". return None @@ -334,7 +341,7 @@ def _get_file_config(file_path: str | None, section_name: str | None) -> dict[st return None section_data = config_section(cfg, section_name) - return _extract_conn_params_from_section(section_data) + return extract_conn_params_from_section(section_data) def _get_test_server_config( @@ -496,14 +503,22 @@ def _test_server_to_params(server: Any, was_already_started: bool) -> dict[str, return params -def _extract_conn_params_from_section(section_data: dict[str, Any]) -> dict[str, Any] | None: +def extract_conn_params_from_section(section_data: dict[str, Any]) -> dict[str, Any] | None: """Extract connection parameters from a config section dict. - Returns a dict containing only CONNKEYS entries. Returns ``None`` if no - server URL is present. Calendar filter keys (``calendar_name``, - ``calendar_url``) are intentionally excluded — callers that need them - (e.g. :func:`get_all_file_connection_params`) read ``section_data`` - directly. + Keys prefixed with ``caldav_`` are mapped to client constructor parameters + (with ``caldav_user``/``caldav_pass`` accepted as aliases for + username/password), environment variable references are expanded, and a + ``features`` key is resolved through :func:`resolve_features`. Public so + that downstream tools (e.g. plann) can reuse it on plann-style config + sections. + + Returns a dict containing only CONNKEYS entries. Returns ``None`` if + neither a server URL nor features are present (with features, the client + constructor can resolve the URL from auto-connect.url hints). Calendar + filter keys (``calendar_name``, ``calendar_url``) are intentionally + excluded — callers that need them (e.g. + :func:`get_all_file_connection_params`) read ``section_data`` directly. """ conn_params: dict[str, Any] = {} for k in section_data: @@ -522,7 +537,7 @@ def _extract_conn_params_from_section(section_data: dict[str, Any]) -> dict[str, elif k == "features" and section_data[k]: conn_params["features"] = resolve_features(section_data[k]) - return conn_params if conn_params.get("url") else None + return conn_params if (conn_params.get("url") or conn_params.get("features")) else None def get_all_file_connection_params( @@ -540,7 +555,7 @@ def get_all_file_connection_params( ``calendar_url`` calendar-filter keys read from the config section. Returns an empty list when the config file is absent or the section has - no usable URL. + neither a usable URL nor features to derive one from. """ if not section_name: section_name = "default" @@ -553,7 +568,7 @@ def get_all_file_connection_params( result: list[dict[str, Any]] = [] for s in sections: section_data = config_section(cfg, s) - params = _extract_conn_params_from_section(section_data) + params = extract_conn_params_from_section(section_data) if params: # Add calendar filter keys — these must NOT flow into DAVClient() for k in ("calendar_name", "calendar_url"): @@ -588,7 +603,7 @@ def get_all_test_servers( for section_name in cfg: section_data = config_section(cfg, section_name) if section_data.get("testing_allowed"): - conn_params = _extract_conn_params_from_section(section_data) + conn_params = extract_conn_params_from_section(section_data) if conn_params: # Also copy the raw section data for keys not in CONNKEYS # (e.g., testing_allowed itself, or custom keys) diff --git a/caldav/datastate.py b/caldav/datastate.py index 72c89dc2..85836cb4 100644 --- a/caldav/datastate.py +++ b/caldav/datastate.py @@ -64,18 +64,18 @@ def get_uid(self) -> str | None: """ cal = self.get_icalendar_copy() for comp in cal.subcomponents: - if comp.name in ("VEVENT", "VTODO", "VJOURNAL", "FREEBUSY") and "UID" in comp: + if comp.name in ("VEVENT", "VTODO", "VJOURNAL", "VFREEBUSY") and "UID" in comp: return str(comp["UID"]) return None def get_component_type(self) -> str | None: - """Get the component type (VEVENT, VTODO, VJOURNAL, FREEBUSY) without full parsing. + """Get the component type (VEVENT, VTODO, VJOURNAL, VFREEBUSY) without full parsing. Default implementation parses the data, but subclasses can optimize. """ cal = self.get_icalendar_copy() for comp in cal.subcomponents: - if comp.name in ("VEVENT", "VTODO", "VJOURNAL", "FREEBUSY"): + if comp.name in ("VEVENT", "VTODO", "VJOURNAL", "VFREEBUSY"): return comp.name return None @@ -149,7 +149,7 @@ def get_component_type(self) -> str | None: return "VTODO" elif "BEGIN:VJOURNAL" in self._data: return "VJOURNAL" - elif "BEGIN:FREEBUSY" in self._data: + elif "BEGIN:VFREEBUSY" in self._data: return "VFREEBUSY" return None diff --git a/caldav/davclient.py b/caldav/davclient.py index 74d37803..6a84d446 100644 --- a/caldav/davclient.py +++ b/caldav/davclient.py @@ -223,7 +223,12 @@ def __init__( preventing DNS-based downgrade attacks where malicious DNS could redirect to unencrypted HTTP. Set to False ONLY if you need to support non-TLS servers and trust your DNS infrastructure. - This parameter has no effect if enable_rfc6764=False. + SCOPE: this only gates the RFC6764 discovery path. It has no + effect when enable_rfc6764=False, and does NOT reject an + explicitly-passed http:// URL (e.g. url="http://your.server.example.com/dav/" + still connects over plaintext despite require_tls=True). + Making enforcement global is deferred to 4.0 — see + https://github.com/python-caldav/caldav/issues/687 rate_limit_handle: boolean, whether to automatically sleep and retry when the server responds with 429 Too Many Requests or 503 Service Unavailable. Default: False (raise RateLimitError immediately). @@ -298,8 +303,10 @@ def __init__( ) self.headers.update(headers or CaseInsensitiveDict()) if self.url.username is not None: - username = unquote(self.url.username) - password = unquote(self.url.password) + if username is None: + username = unquote(self.url.username) + if password is None and self.url.password is not None: + password = unquote(self.url.password) # Use discovered username if no explicit username was provided if username is None and discovered_username is not None: @@ -331,19 +338,9 @@ def __init__( self._principal = None - rate_limit = self.features.is_supported("rate-limit", dict) - if rate_limit_handle is None: - if rate_limit and rate_limit.get("enable"): - rate_limit_handle = True - if "default_sleep" in rate_limit: - rate_limit_default_sleep = rate_limit["default_sleep"] - if "max_sleep" in rate_limit: - rate_limit_max_sleep = rate_limit["max_sleep"] - else: - rate_limit_handle = False - self.rate_limit_handle = rate_limit_handle - self.rate_limit_default_sleep = rate_limit_default_sleep - self.rate_limit_max_sleep = rate_limit_max_sleep + self._init_rate_limit_config( + rate_limit_handle, rate_limit_default_sleep, rate_limit_max_sleep + ) def __enter__(self) -> Self: ## Used for tests, to set up a temporarily test server @@ -466,13 +463,6 @@ def get_calendars(self, principal: Principal | None = None) -> list[Calendar]: for cal in calendars: print(f"Calendar: {cal.get_display_name()}") """ - from caldav.collection import ( - _extract_calendar_home_set_from_results as extract_home_set, - ) - from caldav.collection import ( - _extract_calendars_from_propfind_results as extract_calendars, - ) - if principal is None: principal = self.principal() @@ -482,14 +472,7 @@ def get_calendars(self, principal: Principal | None = None) -> list[Calendar]: props=self.CALENDAR_HOME_SET_PROPS, depth=0, ) - calendar_home_url = extract_home_set(response.results) - if not calendar_home_url: - # Fall back to the principal URL as calendar home - # (some servers like GMX don't support calendar-home-set) - calendar_home_url = str(principal.url) - - # Make URL absolute if relative - calendar_home_url = self._make_absolute_url(calendar_home_url) + calendar_home_url = self._calendar_home_url(response, principal) # Fetch calendars via PROPFIND response = self.propfind( @@ -498,14 +481,7 @@ def get_calendars(self, principal: Principal | None = None) -> list[Calendar]: depth=1, ) - # Process results using shared helper - calendar_infos = extract_calendars(response.results) - - # Convert CalendarInfo objects to Calendar objects - return [ - Calendar(client=self, url=info.url, name=info.name, id=info.cal_id) - for info in calendar_infos - ] + return self._build_calendars_from_propfind(response) def search_calendar( self, @@ -825,20 +801,7 @@ def request( try: return self._sync_request(url, method, body, headers) except error.RateLimitError as e: - if not self.rate_limit_handle: - raise - sleep_seconds = error.compute_sleep_seconds( - e.retry_after_seconds, - self.rate_limit_default_sleep, - self.rate_limit_max_sleep, - ) - if rate_limit_time_slept: - sleep_seconds += rate_limit_time_slept / 2 - if sleep_seconds is None or ( - self.rate_limit_max_sleep is not None - and rate_limit_time_slept > self.rate_limit_max_sleep - ): - raise + sleep_seconds = self._rate_limit_sleep_seconds(e, rate_limit_time_slept) time.sleep(sleep_seconds) return self.request(url, method, body, headers, rate_limit_time_slept + sleep_seconds) diff --git a/caldav/discovery.py b/caldav/discovery.py index c240858b..08c74387 100644 --- a/caldav/discovery.py +++ b/caldav/discovery.py @@ -393,6 +393,10 @@ def discover_service( DNS-based downgrade attacks to plaintext HTTP. Set to False only if you explicitly need to support non-TLS servers and trust your DNS infrastructure. + NOTE: this gates the discovery path only; the client does not + enforce TLS on explicitly-passed URLs. Global enforcement is + deferred to 4.0 — see + https://github.com/python-caldav/caldav/issues/687 Returns: ServiceInfo object with discovered service details, or None if discovery fails @@ -481,10 +485,16 @@ def discover_service( well_known_info = _well_known_lookup(domain, service_type, timeout, ssl_verify_cert) if well_known_info: - # Preserve username from email address - well_known_info.username = username - log.info(f"Discovered {service_type} service via well-known URI: {well_known_info.url}") - return well_known_info + if require_tls and not well_known_info.tls: + log.warning( + f"require_tls=True: Rejecting well-known redirect to non-TLS URL " + f"{well_known_info.url!r} — possible misconfiguration or downgrade attack" + ) + else: + # Preserve username from email address + well_known_info.username = username + log.info(f"Discovered {service_type} service via well-known URI: {well_known_info.url}") + return well_known_info # All discovery methods failed log.warning(f"Failed to discover {service_type} service for {domain}") diff --git a/caldav/jmap/async_client.py b/caldav/jmap/async_client.py index d87fa682..07f64e57 100644 --- a/caldav/jmap/async_client.py +++ b/caldav/jmap/async_client.py @@ -3,6 +3,9 @@ Mirrors JMAPClient with all public methods as coroutines. Uses niquests.AsyncSession for HTTP — niquests is a core dependency. + +All response-parsing logic lives in _JMAPClientBase (client.py); each method +here is a ~3-line async wrapper: get session, send request, delegate to parser. """ from __future__ import annotations @@ -12,16 +15,13 @@ from niquests import AsyncSession -from caldav.jmap._methods.calendar import build_calendar_get, parse_calendar_get +from caldav.jmap._methods.calendar import build_calendar_get from caldav.jmap._methods.event import ( build_event_changes, build_event_get, build_event_set_create, build_event_set_destroy, build_event_set_update, - parse_event_changes, - parse_event_get, - parse_event_set, ) from caldav.jmap._methods.task import ( build_task_get, @@ -29,8 +29,6 @@ build_task_set_create, build_task_set_destroy, build_task_set_update, - parse_task_list_get, - parse_task_set, ) from caldav.jmap.client import _DEFAULT_USING, _TASK_USING, _JMAPClientBase from caldav.jmap.convert import ical_to_jscal @@ -64,11 +62,23 @@ class AsyncJMAPClient(_JMAPClientBase): timeout: HTTP request timeout in seconds. """ + def _get_http_session(self) -> AsyncSession: + """Return the persistent async HTTP session, creating it on first call.""" + if self._http_session is None: + sess = AsyncSession() + sess.auth = self._auth + sess.headers.update({"Content-Type": "application/json", "Accept": "application/json"}) + self._http_session = sess + return self._http_session + async def __aenter__(self) -> AsyncJMAPClient: + self._get_http_session() return self async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: - return None + if self._http_session is not None: + await self._http_session.close() + self._http_session = None async def _get_session(self) -> Session: """Return the cached Session, fetching it on first call.""" @@ -103,14 +113,11 @@ async def _request(self, method_calls: list[tuple], using: list[str] | None = No log.debug("JMAP POST to %s: %d method call(s)", session.api_url, len(method_calls)) - async with AsyncSession() as http: - response = await http.post( - session.api_url, - json=payload, - auth=self._auth, - headers={"Content-Type": "application/json", "Accept": "application/json"}, - timeout=self.timeout, - ) + response = await self._get_http_session().post( + session.api_url, + json=payload, + timeout=self.timeout, + ) if response.status_code in (401, 403): raise JMAPAuthError( @@ -142,18 +149,8 @@ async def get_calendars(self) -> list[JMAPCalendar]: List of :class:`~caldav.jmap.objects.calendar.JMAPCalendar` objects. """ session = await self._get_session() - call = build_calendar_get(session.account_id) - responses = await self._request([call]) - - for method_name, resp_args, _ in responses: - if method_name == "Calendar/get": - calendars = parse_calendar_get(resp_args) - for cal in calendars: - cal._client = self - cal._is_async = True - return calendars - - return [] + responses = await self._request([build_calendar_get(session.account_id)]) + return self._parse_get_calendars(responses, self, True) async def create_event(self, calendar_id: str, ical_str: str) -> str: """Create a calendar event from an iCalendar string. @@ -172,20 +169,7 @@ async def create_event(self, calendar_id: str, ical_str: str) -> str: jscal = ical_to_jscal(ical_str, calendar_id=calendar_id) call = build_event_set_create(session.account_id, {"new-0": jscal}) responses = await self._request([call]) - - for method_name, resp_args, _ in responses: - if method_name == "CalendarEvent/set": - created, _, _, not_created, _, _ = parse_event_set(resp_args) - if "new-0" in not_created: - self._raise_set_error(session, not_created["new-0"]) - if "new-0" not in created: - raise JMAPMethodError( - url=session.api_url, - reason="CalendarEvent/set response missing created entry for new-0", - ) - return created["new-0"]["id"] - - raise JMAPMethodError(url=session.api_url, reason="No CalendarEvent/set response") + return self._parse_create_event_response(responses, session.api_url) async def get_event(self, event_id: str) -> JMAPCalendarObject: """Fetch a calendar event as an iCalendar string. @@ -203,21 +187,8 @@ async def get_event(self, event_id: str) -> JMAPCalendarObject: JMAPMethodError: If the event is not found. """ session = await self._get_session() - call = build_event_get(session.account_id, ids=[event_id]) - responses = await self._request([call]) - - for method_name, resp_args, _ in responses: - if method_name == "CalendarEvent/get": - items = parse_event_get(resp_args) - if not items: - raise JMAPMethodError( - url=session.api_url, - reason=f"Event not found: {event_id}", - error_type="notFound", - ) - return JMAPCalendarObject(data=items[0], parent=None) - - raise JMAPMethodError(url=session.api_url, reason="No CalendarEvent/get response") + responses = await self._request([build_event_get(session.account_id, ids=[event_id])]) + return self._parse_get_event_response(responses, session.api_url, event_id) async def update_event(self, event_id: str, ical_str: str) -> None: """Update a calendar event from an iCalendar string. @@ -230,19 +201,17 @@ async def update_event(self, event_id: str, ical_str: str) -> None: JMAPMethodError: If the server rejects the update. """ session = await self._get_session() - patch = ical_to_jscal(ical_str) - patch.pop("uid", None) # uid is server-immutable after creation; patch must omit it - call = build_event_set_update(session.account_id, {event_id: patch}) - responses = await self._request([call]) - - for method_name, resp_args, _ in responses: - if method_name == "CalendarEvent/set": - _, _, _, _, not_updated, _ = parse_event_set(resp_args) - if event_id in not_updated: - self._raise_set_error(session, not_updated[event_id]) - return - - raise JMAPMethodError(url=session.api_url, reason="No CalendarEvent/set response") + patch, nulled = self._build_event_update_patch(ical_str) + while True: + responses = await self._request( + [build_event_set_update(session.account_id, {event_id: patch})] + ) + drop = self._unsupported_null_keys(responses, event_id, patch, nulled) + if not drop: + break + for key in drop: + patch.pop(key, None) + self._parse_update_event_response(responses, session.api_url, event_id) async def _search( self, @@ -255,15 +224,7 @@ async def _search( session = await self._get_session() calls = self._build_event_search_calls(session.account_id, calendar_id, start, end, text) responses = await self._request(calls) - - for method_name, resp_args, _ in responses: - if method_name == "CalendarEvent/get": - return [ - JMAPCalendarObject(data=item, parent=parent) - for item in parse_event_get(resp_args) - ] - - return [] + return self._parse_search_response(responses, parent) async def search_events( self, @@ -302,16 +263,12 @@ async def get_sync_token(self) -> str: retrieve only what changed since this point. """ session = await self._get_session() - call = build_event_get(session.account_id, ids=[]) - responses = await self._request([call]) - for method_name, resp_args, _ in responses: - if method_name == "CalendarEvent/get": - return resp_args.get("state", "") - raise JMAPMethodError(url=session.api_url, reason="No CalendarEvent/get response") + responses = await self._request([build_event_get(session.account_id, ids=[])]) + return self._parse_get_sync_token_response(responses, session.api_url) async def get_objects_by_sync_token( self, sync_token: str - ) -> tuple[list[JMAPCalendarObject], list[JMAPCalendarObject], list[str]]: + ) -> tuple[list[JMAPCalendarObject], list[JMAPCalendarObject], list[str], str]: """Fetch events changed since a previous sync token. Calls ``CalendarEvent/changes`` to discover which events were created, @@ -325,53 +282,28 @@ async def get_objects_by_sync_token( or by a prior call to this method. Returns: - A 3-tuple ``(added, modified, deleted)``: + A 4-tuple ``(added, modified, deleted, new_sync_token)``: - ``added``: objects for newly created events (``parent`` is ``None``). - ``modified``: objects for updated events (``parent`` is ``None``). - ``deleted``: Event IDs that were destroyed. + - ``new_sync_token``: Pass to the next call to this method as ``sync_token``. Raises: JMAPMethodError: If the server reports ``hasMoreChanges: true``. """ session = await self._get_session() - changes_call = build_event_changes(session.account_id, sync_token) - responses = await self._request([changes_call]) - - created_ids: list[str] = [] - updated_ids: list[str] = [] - destroyed: list[str] = [] - - for method_name, resp_args, _ in responses: - if method_name == "CalendarEvent/changes": - _, _, has_more, created_ids, updated_ids, destroyed = parse_event_changes(resp_args) - if has_more: - raise JMAPMethodError( - url=session.api_url, - reason=( - "CalendarEvent/changes response was truncated by the server " - "(hasMoreChanges=true). Call get_sync_token() to obtain a " - "fresh baseline and re-sync." - ), - error_type="serverPartialFail", - ) - + responses = await self._request([build_event_changes(session.account_id, sync_token)]) + created_ids, updated_ids, destroyed, new_sync_token = self._parse_event_changes_response( + responses, session.api_url + ) fetch_ids = created_ids + updated_ids if not fetch_ids: - return [], [], destroyed - - get_call = build_event_get(session.account_id, ids=fetch_ids) - get_responses = await self._request([get_call]) - - events_by_id: dict[str, JMAPCalendarObject] = {} - for method_name, resp_args, _ in get_responses: - if method_name == "CalendarEvent/get": - for item in parse_event_get(resp_args): - events_by_id[item["id"]] = JMAPCalendarObject(data=item, parent=None) - - added = [events_by_id[i] for i in created_ids if i in events_by_id] - modified = [events_by_id[i] for i in updated_ids if i in events_by_id] - return added, modified, destroyed + return [], [], destroyed, new_sync_token + get_responses = await self._request([build_event_get(session.account_id, ids=fetch_ids)]) + return self._assemble_sync_token_result( + get_responses, created_ids, updated_ids, destroyed, new_sync_token + ) async def delete_event(self, event_id: str) -> None: """Delete a calendar event. @@ -383,17 +315,8 @@ async def delete_event(self, event_id: str) -> None: JMAPMethodError: If the server rejects the delete. """ session = await self._get_session() - call = build_event_set_destroy(session.account_id, [event_id]) - responses = await self._request([call]) - - for method_name, resp_args, _ in responses: - if method_name == "CalendarEvent/set": - _, _, _, _, _, not_destroyed = parse_event_set(resp_args) - if event_id in not_destroyed: - self._raise_set_error(session, not_destroyed[event_id]) - return - - raise JMAPMethodError(url=session.api_url, reason="No CalendarEvent/set response") + responses = await self._request([build_event_set_destroy(session.account_id, [event_id])]) + self._parse_delete_event_response(responses, session.api_url, event_id) async def _get_object_by_uid( self, uid: str, calendar_id: str | None = None, parent: JMAPCalendar | None = None @@ -414,14 +337,10 @@ async def get_task_lists(self) -> list[dict]: List of raw JMAP TaskList dicts as returned by the server. """ session = await self._get_session() - call = build_task_list_get(session.account_id) - responses = await self._request([call], using=_TASK_USING) - - for method_name, resp_args, _ in responses: - if method_name == "TaskList/get": - return parse_task_list_get(resp_args) - - return [] + responses = await self._request( + [build_task_list_get(session.account_id)], using=_TASK_USING + ) + return self._parse_get_task_lists_response(responses) async def create_task(self, task_list_id: str, title: str, **kwargs) -> str: """Create a task in a task list. @@ -452,15 +371,7 @@ async def create_task(self, task_list_id: str, title: str, **kwargs) -> str: task_dict.update(kwargs) call = build_task_set_create(session.account_id, {"new-0": task_dict}) responses = await self._request([call], using=_TASK_USING) - - for method_name, resp_args, _ in responses: - if method_name == "Task/set": - created, _, _, not_created, _, _ = parse_task_set(resp_args) - if "new-0" in not_created: - self._raise_set_error(session, not_created["new-0"]) - return created["new-0"]["id"] - - raise JMAPMethodError(url=session.api_url, reason="No Task/set response") + return self._parse_create_task_response(responses, session.api_url) async def get_task(self, task_id: str) -> dict: """Fetch a task by ID. @@ -475,21 +386,10 @@ async def get_task(self, task_id: str) -> dict: JMAPMethodError: If the task is not found. """ session = await self._get_session() - call = build_task_get(session.account_id, ids=[task_id]) - responses = await self._request([call], using=_TASK_USING) - - for method_name, resp_args, _ in responses: - if method_name == "Task/get": - items = resp_args.get("list", []) - if not items: - raise JMAPMethodError( - url=session.api_url, - reason=f"Task not found: {task_id}", - error_type="notFound", - ) - return items[0] - - raise JMAPMethodError(url=session.api_url, reason="No Task/get response") + responses = await self._request( + [build_task_get(session.account_id, ids=[task_id])], using=_TASK_USING + ) + return self._parse_get_task_response(responses, session.api_url, task_id) async def update_task(self, task_id: str, patch: dict) -> None: """Update a task with a partial patch. @@ -504,15 +404,7 @@ async def update_task(self, task_id: str, patch: dict) -> None: session = await self._get_session() call = build_task_set_update(session.account_id, {task_id: patch}) responses = await self._request([call], using=_TASK_USING) - - for method_name, resp_args, _ in responses: - if method_name == "Task/set": - _, _, _, _, not_updated, _ = parse_task_set(resp_args) - if task_id in not_updated: - self._raise_set_error(session, not_updated[task_id]) - return - - raise JMAPMethodError(url=session.api_url, reason="No Task/set response") + self._parse_update_task_response(responses, session.api_url, task_id) async def delete_task(self, task_id: str) -> None: """Delete a task. @@ -524,14 +416,7 @@ async def delete_task(self, task_id: str) -> None: JMAPMethodError: If the server rejects the delete. """ session = await self._get_session() - call = build_task_set_destroy(session.account_id, [task_id]) - responses = await self._request([call], using=_TASK_USING) - - for method_name, resp_args, _ in responses: - if method_name == "Task/set": - _, _, _, _, _, not_destroyed = parse_task_set(resp_args) - if task_id in not_destroyed: - self._raise_set_error(session, not_destroyed[task_id]) - return - - raise JMAPMethodError(url=session.api_url, reason="No Task/set response") + responses = await self._request( + [build_task_set_destroy(session.account_id, [task_id])], using=_TASK_USING + ) + self._parse_delete_task_response(responses, session.api_url, task_id) diff --git a/caldav/jmap/client.py b/caldav/jmap/client.py index 59cd333b..ebbba237 100644 --- a/caldav/jmap/client.py +++ b/caldav/jmap/client.py @@ -43,6 +43,7 @@ ) from caldav.jmap.constants import CALENDAR_CAPABILITY, CORE_CAPABILITY, TASK_CAPABILITY from caldav.jmap.convert import ical_to_jscal +from caldav.jmap.convert._patch import _NULL_FOR_UPDATE from caldav.jmap.error import JMAPAuthError, JMAPMethodError from caldav.jmap.objects.calendar import JMAPCalendar from caldav.jmap.objects.calendar_object import JMAPCalendarObject @@ -70,6 +71,7 @@ def __init__( self.password = password self.timeout = timeout self._session_cache: Session | None = None + self._http_session = None if auth is not None: self._auth = auth @@ -118,9 +120,10 @@ def _build_auth(self, auth_type: str | None): reason=f"Unsupported auth_type {effective_type!r}. Use 'basic' or 'bearer'.", ) - def _raise_set_error(self, session: Session, err: dict) -> None: + @staticmethod + def _raise_set_error(api_url: str, err: dict) -> None: raise JMAPMethodError( - url=session.api_url, + url=api_url, reason=f"set failed: {err}", error_type=err.get("type", "serverError"), ) @@ -158,6 +161,256 @@ def _build_event_search_calls( ) return [query_call, get_call] + @staticmethod + def _build_event_update_patch(ical_str: str) -> tuple[dict, frozenset[str]]: + """Build a JSCalendar PatchObject for a ``CalendarEvent/set`` update. + + RFC 8620 merge semantics preserve properties absent from the patch, so + any optional property removed client-side must be explicitly nulled to + actually clear it server-side. Returns the patch together with the set + of keys that were null-injected purely for this cleanup (i.e. were not + present in the converted iCalendar) so the caller can drop them if the + server refuses to null a property it does not support. + """ + patch = ical_to_jscal(ical_str) + patch.pop("uid", None) # uid is server-immutable after creation; patch must omit it + nulled: set[str] = set() + for key in _NULL_FOR_UPDATE: + if key not in patch: + patch[key] = None + nulled.add(key) + return patch, frozenset(nulled) + + @staticmethod + def _unsupported_null_keys( + responses: list, event_id: str, patch: dict, nulled: frozenset[str] + ) -> set[str] | None: + """Detect an update that failed *only* because the server rejects + null-clearing of properties it does not support. + + Some servers (e.g. Stalwart for ``recurrenceRules``) reject a property + outright in ``CalendarEvent/set``, even when it is being set to ``null``. + Nulling such a property is harmless cleanup — it was absent from the new + iCalendar — so we report it as droppable, letting the caller retry the + update without it. + + Returns the set of droppable keys when the failure is exactly this case, + or ``None`` when the update succeeded or failed for a genuine reason (in + which case the caller proceeds to :meth:`_parse_update_event_response`, + which raises the real error). Some servers report only one offending + property per response, so the caller retries in a loop, dropping the + reported keys until the update succeeds or hits a genuine error; each + returned key is guaranteed still present in ``patch``, so the loop + strictly shrinks the patch and terminates. + """ + for method_name, resp_args, _ in responses: + if method_name == "CalendarEvent/set": + _, _, _, _, not_updated, _ = parse_event_set(resp_args) + err = not_updated.get(event_id) + if not err or err.get("type") != "invalidProperties": + return None + props = set(err.get("properties") or []) + droppable = {p for p in props if p in nulled and p in patch and patch[p] is None} + # Only retry when every offending property is null-cleanup we can + # safely omit; if the client actually set one of them to a value, + # the rejection is genuine and must surface. + if props and props == droppable: + return droppable + return None + return None + + # --------------------------------------------------------------------------- + # Shared response parsers — pure synchronous; used by both sync and async + # clients. Each method takes the raw ``methodResponses`` list returned by + # ``_request()`` plus whatever extra context is needed to build the result + # or raise an informative error, and returns/raises exactly what the public + # method should return/raise. + # --------------------------------------------------------------------------- + + @staticmethod + def _parse_get_calendars(responses: list, client, is_async: bool) -> list[JMAPCalendar]: + for method_name, resp_args, _ in responses: + if method_name == "Calendar/get": + calendars = parse_calendar_get(resp_args) + for cal in calendars: + cal._client = client + cal._is_async = is_async + return calendars + return [] + + @staticmethod + def _parse_create_event_response(responses: list, api_url: str) -> str: + for method_name, resp_args, _ in responses: + if method_name == "CalendarEvent/set": + created, _, _, not_created, _, _ = parse_event_set(resp_args) + if "new-0" in not_created: + _JMAPClientBase._raise_set_error(api_url, not_created["new-0"]) + if "new-0" not in created: + raise JMAPMethodError( + url=api_url, + reason="CalendarEvent/set response missing created entry for new-0", + ) + return created["new-0"]["id"] + raise JMAPMethodError(url=api_url, reason="No CalendarEvent/set response") + + @staticmethod + def _parse_get_event_response( + responses: list, api_url: str, event_id: str + ) -> JMAPCalendarObject: + for method_name, resp_args, _ in responses: + if method_name == "CalendarEvent/get": + items = parse_event_get(resp_args) + if not items: + raise JMAPMethodError( + url=api_url, + reason=f"Event not found: {event_id}", + error_type="notFound", + ) + return JMAPCalendarObject(data=items[0], parent=None) + raise JMAPMethodError(url=api_url, reason="No CalendarEvent/get response") + + @staticmethod + def _parse_update_event_response(responses: list, api_url: str, event_id: str) -> None: + for method_name, resp_args, _ in responses: + if method_name == "CalendarEvent/set": + _, _, _, _, not_updated, _ = parse_event_set(resp_args) + if event_id in not_updated: + _JMAPClientBase._raise_set_error(api_url, not_updated[event_id]) + return + raise JMAPMethodError(url=api_url, reason="No CalendarEvent/set response") + + @staticmethod + def _parse_search_response( + responses: list, parent: JMAPCalendar | None + ) -> list[JMAPCalendarObject]: + for method_name, resp_args, _ in responses: + if method_name == "CalendarEvent/get": + return [ + JMAPCalendarObject(data=item, parent=parent) + for item in parse_event_get(resp_args) + ] + return [] + + @staticmethod + def _parse_get_sync_token_response(responses: list, api_url: str) -> str: + for method_name, resp_args, _ in responses: + if method_name == "CalendarEvent/get": + return resp_args.get("state", "") + raise JMAPMethodError(url=api_url, reason="No CalendarEvent/get response") + + @staticmethod + def _parse_event_changes_response( + responses: list, api_url: str + ) -> tuple[list[str], list[str], list[str], str]: + """Parse a CalendarEvent/changes response. + + Returns ``(created_ids, updated_ids, destroyed_ids, new_sync_token)``. + Raises :class:`JMAPMethodError` when the server truncated the result. + """ + created_ids: list[str] = [] + updated_ids: list[str] = [] + destroyed: list[str] = [] + new_sync_token: str = "" + for method_name, resp_args, _ in responses: + if method_name == "CalendarEvent/changes": + _, new_sync_token, has_more, created_ids, updated_ids, destroyed = ( + parse_event_changes(resp_args) + ) + if has_more: + raise JMAPMethodError( + url=api_url, + reason=( + "CalendarEvent/changes response was truncated by the server " + "(hasMoreChanges=true). Call get_sync_token() to obtain a " + "fresh baseline and re-sync." + ), + error_type="serverPartialFail", + ) + return created_ids, updated_ids, destroyed, new_sync_token + + @staticmethod + def _assemble_sync_token_result( + get_responses: list, + created_ids: list[str], + updated_ids: list[str], + destroyed: list[str], + new_sync_token: str, + ) -> tuple[list[JMAPCalendarObject], list[JMAPCalendarObject], list[str], str]: + events_by_id: dict[str, JMAPCalendarObject] = {} + for method_name, resp_args, _ in get_responses: + if method_name == "CalendarEvent/get": + for item in parse_event_get(resp_args): + events_by_id[item["id"]] = JMAPCalendarObject(data=item, parent=None) + added = [events_by_id[i] for i in created_ids if i in events_by_id] + modified = [events_by_id[i] for i in updated_ids if i in events_by_id] + return added, modified, destroyed, new_sync_token + + @staticmethod + def _parse_delete_event_response(responses: list, api_url: str, event_id: str) -> None: + for method_name, resp_args, _ in responses: + if method_name == "CalendarEvent/set": + _, _, _, _, _, not_destroyed = parse_event_set(resp_args) + if event_id in not_destroyed: + _JMAPClientBase._raise_set_error(api_url, not_destroyed[event_id]) + return + raise JMAPMethodError(url=api_url, reason="No CalendarEvent/set response") + + @staticmethod + def _parse_get_task_lists_response(responses: list) -> list[dict]: + for method_name, resp_args, _ in responses: + if method_name == "TaskList/get": + return parse_task_list_get(resp_args) + return [] + + @staticmethod + def _parse_create_task_response(responses: list, api_url: str) -> str: + for method_name, resp_args, _ in responses: + if method_name == "Task/set": + created, _, _, not_created, _, _ = parse_task_set(resp_args) + if "new-0" in not_created: + _JMAPClientBase._raise_set_error(api_url, not_created["new-0"]) + if "new-0" not in created: + raise JMAPMethodError( + url=api_url, + reason="Task/set response missing created entry for new-0", + ) + return created["new-0"]["id"] + raise JMAPMethodError(url=api_url, reason="No Task/set response") + + @staticmethod + def _parse_get_task_response(responses: list, api_url: str, task_id: str) -> dict: + for method_name, resp_args, _ in responses: + if method_name == "Task/get": + items = resp_args.get("list", []) + if not items: + raise JMAPMethodError( + url=api_url, + reason=f"Task not found: {task_id}", + error_type="notFound", + ) + return items[0] + raise JMAPMethodError(url=api_url, reason="No Task/get response") + + @staticmethod + def _parse_update_task_response(responses: list, api_url: str, task_id: str) -> None: + for method_name, resp_args, _ in responses: + if method_name == "Task/set": + _, _, _, _, not_updated, _ = parse_task_set(resp_args) + if task_id in not_updated: + _JMAPClientBase._raise_set_error(api_url, not_updated[task_id]) + return + raise JMAPMethodError(url=api_url, reason="No Task/set response") + + @staticmethod + def _parse_delete_task_response(responses: list, api_url: str, task_id: str) -> None: + for method_name, resp_args, _ in responses: + if method_name == "Task/set": + _, _, _, _, _, not_destroyed = parse_task_set(resp_args) + if task_id in not_destroyed: + _JMAPClientBase._raise_set_error(api_url, not_destroyed[task_id]) + return + raise JMAPMethodError(url=api_url, reason="No Task/set response") + class JMAPClient(_JMAPClientBase): """Synchronous JMAP client for calendar operations. @@ -179,11 +432,23 @@ class JMAPClient(_JMAPClientBase): timeout: HTTP request timeout in seconds. """ + def _get_http_session(self): + """Return the persistent HTTP session, creating it on first call.""" + if self._http_session is None: + sess = requests.Session() + sess.auth = self._auth + sess.headers.update({"Content-Type": "application/json", "Accept": "application/json"}) + self._http_session = sess + return self._http_session + def __enter__(self) -> JMAPClient: + self._get_http_session() return self def __exit__(self, exc_type, exc_val, exc_tb) -> None: - return None + if self._http_session is not None: + self._http_session.close() + self._http_session = None def _get_session(self) -> Session: """Return the cached Session, fetching it on first call.""" @@ -217,11 +482,9 @@ def _request(self, method_calls: list[tuple], using: list[str] | None = None) -> log.debug("JMAP POST to %s: %d method call(s)", session.api_url, len(method_calls)) - response = requests.post( + response = self._get_http_session().post( session.api_url, json=payload, - auth=self._auth, - headers={"Content-Type": "application/json", "Accept": "application/json"}, timeout=self.timeout, ) @@ -255,18 +518,8 @@ def get_calendars(self) -> list[JMAPCalendar]: List of :class:`~caldav.jmap.objects.calendar.JMAPCalendar` objects. """ session = self._get_session() - call = build_calendar_get(session.account_id) - responses = self._request([call]) - - for method_name, resp_args, _ in responses: - if method_name == "Calendar/get": - calendars = parse_calendar_get(resp_args) - for cal in calendars: - cal._client = self - cal._is_async = False - return calendars - - return [] + responses = self._request([build_calendar_get(session.account_id)]) + return self._parse_get_calendars(responses, self, False) def create_event(self, calendar_id: str, ical_str: str) -> str: """Create a calendar event from an iCalendar string. @@ -285,20 +538,7 @@ def create_event(self, calendar_id: str, ical_str: str) -> str: jscal = ical_to_jscal(ical_str, calendar_id=calendar_id) call = build_event_set_create(session.account_id, {"new-0": jscal}) responses = self._request([call]) - - for method_name, resp_args, _ in responses: - if method_name == "CalendarEvent/set": - created, _, _, not_created, _, _ = parse_event_set(resp_args) - if "new-0" in not_created: - self._raise_set_error(session, not_created["new-0"]) - if "new-0" not in created: - raise JMAPMethodError( - url=session.api_url, - reason="CalendarEvent/set response missing created entry for new-0", - ) - return created["new-0"]["id"] - - raise JMAPMethodError(url=session.api_url, reason="No CalendarEvent/set response") + return self._parse_create_event_response(responses, session.api_url) def get_event(self, event_id: str) -> JMAPCalendarObject: """Fetch a calendar event by JMAP event ID. @@ -316,21 +556,8 @@ def get_event(self, event_id: str) -> JMAPCalendarObject: JMAPMethodError: If the event is not found. """ session = self._get_session() - call = build_event_get(session.account_id, ids=[event_id]) - responses = self._request([call]) - - for method_name, resp_args, _ in responses: - if method_name == "CalendarEvent/get": - items = parse_event_get(resp_args) - if not items: - raise JMAPMethodError( - url=session.api_url, - reason=f"Event not found: {event_id}", - error_type="notFound", - ) - return JMAPCalendarObject(data=items[0], parent=None) - - raise JMAPMethodError(url=session.api_url, reason="No CalendarEvent/get response") + responses = self._request([build_event_get(session.account_id, ids=[event_id])]) + return self._parse_get_event_response(responses, session.api_url, event_id) def update_event(self, event_id: str, ical_str: str) -> None: """Update a calendar event from an iCalendar string. @@ -343,19 +570,17 @@ def update_event(self, event_id: str, ical_str: str) -> None: JMAPMethodError: If the server rejects the update. """ session = self._get_session() - patch = ical_to_jscal(ical_str) - patch.pop("uid", None) # uid is server-immutable after creation; patch must omit it - call = build_event_set_update(session.account_id, {event_id: patch}) - responses = self._request([call]) - - for method_name, resp_args, _ in responses: - if method_name == "CalendarEvent/set": - _, _, _, _, not_updated, _ = parse_event_set(resp_args) - if event_id in not_updated: - self._raise_set_error(session, not_updated[event_id]) - return - - raise JMAPMethodError(url=session.api_url, reason="No CalendarEvent/set response") + patch, nulled = self._build_event_update_patch(ical_str) + while True: + responses = self._request( + [build_event_set_update(session.account_id, {event_id: patch})] + ) + drop = self._unsupported_null_keys(responses, event_id, patch, nulled) + if not drop: + break + for key in drop: + patch.pop(key, None) + self._parse_update_event_response(responses, session.api_url, event_id) def _search( self, @@ -368,15 +593,7 @@ def _search( session = self._get_session() calls = self._build_event_search_calls(session.account_id, calendar_id, start, end, text) responses = self._request(calls) - - for method_name, resp_args, _ in responses: - if method_name == "CalendarEvent/get": - return [ - JMAPCalendarObject(data=item, parent=parent) - for item in parse_event_get(resp_args) - ] - - return [] + return self._parse_search_response(responses, parent) def search_events( self, @@ -417,16 +634,12 @@ def get_sync_token(self) -> str: retrieve only what changed since this point. """ session = self._get_session() - call = build_event_get(session.account_id, ids=[]) - responses = self._request([call]) - for method_name, resp_args, _ in responses: - if method_name == "CalendarEvent/get": - return resp_args.get("state", "") - raise JMAPMethodError(url=session.api_url, reason="No CalendarEvent/get response") + responses = self._request([build_event_get(session.account_id, ids=[])]) + return self._parse_get_sync_token_response(responses, session.api_url) def get_objects_by_sync_token( self, sync_token: str - ) -> tuple[list[JMAPCalendarObject], list[JMAPCalendarObject], list[str]]: + ) -> tuple[list[JMAPCalendarObject], list[JMAPCalendarObject], list[str], str]: """Fetch events changed since a previous sync token. Calls ``CalendarEvent/changes`` to discover which events were created, @@ -440,53 +653,28 @@ def get_objects_by_sync_token( or by a prior call to this method. Returns: - A 3-tuple ``(added, modified, deleted)``: + A 4-tuple ``(added, modified, deleted, new_sync_token)``: - ``added``: objects for newly created events (``parent`` is ``None``). - ``modified``: objects for updated events (``parent`` is ``None``). - ``deleted``: Event IDs that were destroyed. + - ``new_sync_token``: Pass to the next call to this method as ``sync_token``. Raises: JMAPMethodError: If the server reports ``hasMoreChanges: true``. """ session = self._get_session() - changes_call = build_event_changes(session.account_id, sync_token) - responses = self._request([changes_call]) - - created_ids: list[str] = [] - updated_ids: list[str] = [] - destroyed: list[str] = [] - - for method_name, resp_args, _ in responses: - if method_name == "CalendarEvent/changes": - _, _, has_more, created_ids, updated_ids, destroyed = parse_event_changes(resp_args) - if has_more: - raise JMAPMethodError( - url=session.api_url, - reason=( - "CalendarEvent/changes response was truncated by the server " - "(hasMoreChanges=true). Call get_sync_token() to obtain a " - "fresh baseline and re-sync." - ), - error_type="serverPartialFail", - ) - + responses = self._request([build_event_changes(session.account_id, sync_token)]) + created_ids, updated_ids, destroyed, new_sync_token = self._parse_event_changes_response( + responses, session.api_url + ) fetch_ids = created_ids + updated_ids if not fetch_ids: - return [], [], destroyed - - get_call = build_event_get(session.account_id, ids=fetch_ids) - get_responses = self._request([get_call]) - - events_by_id: dict[str, JMAPCalendarObject] = {} - for method_name, resp_args, _ in get_responses: - if method_name == "CalendarEvent/get": - for item in parse_event_get(resp_args): - events_by_id[item["id"]] = JMAPCalendarObject(data=item, parent=None) - - added = [events_by_id[i] for i in created_ids if i in events_by_id] - modified = [events_by_id[i] for i in updated_ids if i in events_by_id] - return added, modified, destroyed + return [], [], destroyed, new_sync_token + get_responses = self._request([build_event_get(session.account_id, ids=fetch_ids)]) + return self._assemble_sync_token_result( + get_responses, created_ids, updated_ids, destroyed, new_sync_token + ) def delete_event(self, event_id: str) -> None: """Delete a calendar event. @@ -498,17 +686,8 @@ def delete_event(self, event_id: str) -> None: JMAPMethodError: If the server rejects the delete. """ session = self._get_session() - call = build_event_set_destroy(session.account_id, [event_id]) - responses = self._request([call]) - - for method_name, resp_args, _ in responses: - if method_name == "CalendarEvent/set": - _, _, _, _, _, not_destroyed = parse_event_set(resp_args) - if event_id in not_destroyed: - self._raise_set_error(session, not_destroyed[event_id]) - return - - raise JMAPMethodError(url=session.api_url, reason="No CalendarEvent/set response") + responses = self._request([build_event_set_destroy(session.account_id, [event_id])]) + self._parse_delete_event_response(responses, session.api_url, event_id) def _get_object_by_uid( self, uid: str, calendar_id: str | None = None, parent: JMAPCalendar | None = None @@ -529,14 +708,8 @@ def get_task_lists(self) -> list[dict]: List of raw JMAP TaskList dicts as returned by the server. """ session = self._get_session() - call = build_task_list_get(session.account_id) - responses = self._request([call], using=_TASK_USING) - - for method_name, resp_args, _ in responses: - if method_name == "TaskList/get": - return parse_task_list_get(resp_args) - - return [] + responses = self._request([build_task_list_get(session.account_id)], using=_TASK_USING) + return self._parse_get_task_lists_response(responses) def create_task(self, task_list_id: str, title: str, **kwargs) -> str: """Create a task in a task list. @@ -567,15 +740,7 @@ def create_task(self, task_list_id: str, title: str, **kwargs) -> str: task_dict.update(kwargs) call = build_task_set_create(session.account_id, {"new-0": task_dict}) responses = self._request([call], using=_TASK_USING) - - for method_name, resp_args, _ in responses: - if method_name == "Task/set": - created, _, _, not_created, _, _ = parse_task_set(resp_args) - if "new-0" in not_created: - self._raise_set_error(session, not_created["new-0"]) - return created["new-0"]["id"] - - raise JMAPMethodError(url=session.api_url, reason="No Task/set response") + return self._parse_create_task_response(responses, session.api_url) def get_task(self, task_id: str) -> dict: """Fetch a task by ID. @@ -590,21 +755,10 @@ def get_task(self, task_id: str) -> dict: JMAPMethodError: If the task is not found. """ session = self._get_session() - call = build_task_get(session.account_id, ids=[task_id]) - responses = self._request([call], using=_TASK_USING) - - for method_name, resp_args, _ in responses: - if method_name == "Task/get": - items = resp_args.get("list", []) - if not items: - raise JMAPMethodError( - url=session.api_url, - reason=f"Task not found: {task_id}", - error_type="notFound", - ) - return items[0] - - raise JMAPMethodError(url=session.api_url, reason="No Task/get response") + responses = self._request( + [build_task_get(session.account_id, ids=[task_id])], using=_TASK_USING + ) + return self._parse_get_task_response(responses, session.api_url, task_id) def update_task(self, task_id: str, patch: dict) -> None: """Update a task with a partial patch. @@ -619,15 +773,7 @@ def update_task(self, task_id: str, patch: dict) -> None: session = self._get_session() call = build_task_set_update(session.account_id, {task_id: patch}) responses = self._request([call], using=_TASK_USING) - - for method_name, resp_args, _ in responses: - if method_name == "Task/set": - _, _, _, _, not_updated, _ = parse_task_set(resp_args) - if task_id in not_updated: - self._raise_set_error(session, not_updated[task_id]) - return - - raise JMAPMethodError(url=session.api_url, reason="No Task/set response") + self._parse_update_task_response(responses, session.api_url, task_id) def delete_task(self, task_id: str) -> None: """Delete a task. @@ -639,14 +785,7 @@ def delete_task(self, task_id: str) -> None: JMAPMethodError: If the server rejects the delete. """ session = self._get_session() - call = build_task_set_destroy(session.account_id, [task_id]) - responses = self._request([call], using=_TASK_USING) - - for method_name, resp_args, _ in responses: - if method_name == "Task/set": - _, _, _, _, _, not_destroyed = parse_task_set(resp_args) - if task_id in not_destroyed: - self._raise_set_error(session, not_destroyed[task_id]) - return - - raise JMAPMethodError(url=session.api_url, reason="No Task/set response") + responses = self._request( + [build_task_set_destroy(session.account_id, [task_id])], using=_TASK_USING + ) + self._parse_delete_task_response(responses, session.api_url, task_id) diff --git a/caldav/jmap/convert/_patch.py b/caldav/jmap/convert/_patch.py new file mode 100644 index 00000000..db31a2ab --- /dev/null +++ b/caldav/jmap/convert/_patch.py @@ -0,0 +1,33 @@ +""" +RFC 8620 PatchObject helpers for CalendarEvent/set update calls. + +When updating an event, absent keys preserve the server's current value. +To delete an optional property the patch must set it to null explicitly. +""" + +from __future__ import annotations + +# Optional JSCalendar top-level properties that must be explicitly nulled in +# a CalendarEvent/set update when they are absent from the converted result. +# This ensures properties removed client-side (e.g. LOCATION deleted from +# the iCalendar) are actually removed on the server, not silently preserved. +_NULL_FOR_UPDATE: frozenset[str] = frozenset( + { + "description", + "color", + "locations", + "keywords", + "priority", + "privacy", + "freeBusyStatus", + "status", + "sequence", + "showWithoutTime", + "timeZone", + "recurrenceRules", + "excludedRecurrenceRules", + "recurrenceOverrides", + "participants", + "alerts", + } +) diff --git a/caldav/jmap/convert/_utils.py b/caldav/jmap/convert/_utils.py index 12b12263..475eca85 100644 --- a/caldav/jmap/convert/_utils.py +++ b/caldav/jmap/convert/_utils.py @@ -111,11 +111,12 @@ def _duration_to_timedelta(duration_str: str) -> timedelta: def _format_local_dt(dt: datetime | date) -> str: - """Format a datetime or date as a JSCalendar LocalDateTime or UTCDateTime string. + """Format a datetime or date as a JSCalendar LocalDateTime string. - JSCalendar uses: - - LocalDateTime: "2024-03-15T09:00:00" (no TZ suffix) - - UTCDateTime: "2024-03-15T09:00:00Z" (uppercase Z) + RFC 8984 requires LocalDateTime (no Z suffix) for override keys and RRULE + ``until`` values. Timezone information is stripped — callers must convert + UTC datetimes to the event's local timezone before calling if the event uses + TZID; for floating or all-day events the naive value is already correct. For date objects (all-day), uses T00:00:00 suffix. @@ -123,10 +124,8 @@ def _format_local_dt(dt: datetime | date) -> str: dt: A datetime (with or without tzinfo) or a date. Returns: - Formatted string suitable for use as a JSCalendar override key or datetime value. + Formatted string suitable for use as a JSCalendar override key or RRULE until. """ if isinstance(dt, datetime): - if dt.tzinfo is not None and dt.utcoffset() == timedelta(0): - return dt.strftime("%Y-%m-%dT%H:%M:%SZ") return dt.strftime("%Y-%m-%dT%H:%M:%S") return f"{dt.isoformat()}T00:00:00" diff --git a/caldav/jmap/convert/ical_to_jscal.py b/caldav/jmap/convert/ical_to_jscal.py index 00e9ce59..dbafd157 100644 --- a/caldav/jmap/convert/ical_to_jscal.py +++ b/caldav/jmap/convert/ical_to_jscal.py @@ -386,6 +386,17 @@ def ical_to_jscal(ical_str: str, calendar_id: str | None = None) -> dict: if location: jscal["locations"] = _location_str_to_jscal(str(location)) + status = master.get("STATUS") + if status: + _STATUS_ICAL_TO_JSCAL = { + "CONFIRMED": "confirmed", + "TENTATIVE": "tentative", + "CANCELLED": "cancelled", + } + jscal_status = _STATUS_ICAL_TO_JSCAL.get(str(status).upper()) + if jscal_status: + jscal["status"] = jscal_status + participants: dict = {} organizer = master.get("ORGANIZER") if organizer is not None: diff --git a/caldav/jmap/convert/jscal_to_ical.py b/caldav/jmap/convert/jscal_to_ical.py index 2a449d1b..4ed4c03d 100644 --- a/caldav/jmap/convert/jscal_to_ical.py +++ b/caldav/jmap/convert/jscal_to_ical.py @@ -353,6 +353,17 @@ def jscal_to_ical(jscal: dict) -> str: if loc_name: event.add("location", loc_name) + status = jscal.get("status") + if status: + _STATUS_JSCAL_TO_ICAL = { + "confirmed": "CONFIRMED", + "tentative": "TENTATIVE", + "cancelled": "CANCELLED", + } + ical_status = _STATUS_JSCAL_TO_ICAL.get(status) + if ical_status: + event.add("status", ical_status) + for rule in jscal.get("recurrenceRules") or []: ical_rule = _jscal_rrule_to_rrule(rule) if ical_rule: @@ -371,6 +382,15 @@ def jscal_to_ical(jscal: dict) -> str: rid_dt: datetime | date = datetime.strptime(override_key, "%Y-%m-%dT%H:%M:%SZ").replace( tzinfo=timezone.utc ) + elif show_without_time: + rid_dt = date.fromisoformat(override_key[:10]) + elif time_zone: + try: + rid_dt = datetime.strptime(override_key[:19], "%Y-%m-%dT%H:%M:%S").replace( + tzinfo=ZoneInfo(time_zone) + ) + except ZoneInfoNotFoundError: + rid_dt = datetime.strptime(override_key[:19], "%Y-%m-%dT%H:%M:%S") else: rid_dt = datetime.strptime(override_key[:19], "%Y-%m-%dT%H:%M:%S") @@ -381,7 +401,8 @@ def jscal_to_ical(jscal: dict) -> str: child.add("uid", uid) child.add("dtstamp", datetime.now(tz=timezone.utc)) child.add("recurrence-id", rid_dt) - child_start = patch.get("start", start_str) + # Default child start to the occurrence time (override key), not the master start. + child_start = patch.get("start", override_key) child_tz = patch.get("timeZone", time_zone) child_swt = patch.get("showWithoutTime", show_without_time) if child_start: diff --git a/caldav/jmap/objects/calendar.py b/caldav/jmap/objects/calendar.py index 28812ac9..6be47bd7 100644 --- a/caldav/jmap/objects/calendar.py +++ b/caldav/jmap/objects/calendar.py @@ -8,7 +8,7 @@ from __future__ import annotations from dataclasses import dataclass, field -from datetime import datetime +from datetime import datetime, timezone from typing import TYPE_CHECKING from caldav.jmap.objects.calendar_object import JMAPCalendarObject @@ -18,6 +18,17 @@ from caldav.jmap.client import JMAPClient +def _to_utcdate(dt: datetime) -> str: + """Convert a datetime to JMAP UTCDate format (YYYY-MM-DDTHH:MM:SSZ). + + Naive datetimes are assumed to be UTC. Aware datetimes are converted to + UTC before formatting. Microseconds are dropped as JMAP does not allow them. + """ + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + @dataclass class JMAPCalendar: """A JMAP Calendar object. @@ -111,9 +122,9 @@ def search(self, **searchargs): start = searchargs.get("start") end = searchargs.get("end") if isinstance(start, datetime): - start = start.isoformat() + start = _to_utcdate(start) if isinstance(end, datetime): - end = end.isoformat() + end = _to_utcdate(end) return self._client._search( calendar_id=self.id, start=start, @@ -126,9 +137,9 @@ async def _async_search(self, **searchargs) -> list[JMAPCalendarObject]: start = searchargs.get("start") end = searchargs.get("end") if isinstance(start, datetime): - start = start.isoformat() + start = _to_utcdate(start) if isinstance(end, datetime): - end = end.isoformat() + end = _to_utcdate(end) return await self._client._search( calendar_id=self.id, start=start, diff --git a/caldav/lib/auth.py b/caldav/lib/auth.py index fa4d351e..c15b9ef1 100644 --- a/caldav/lib/auth.py +++ b/caldav/lib/auth.py @@ -28,7 +28,7 @@ def extract_auth_types(header: str) -> set[str]: Reference: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/WWW-Authenticate#syntax """ - return {h.split()[0] for h in header.lower().split(",")} + return {h.split()[0] for h in header.lower().split(",") if h.strip()} def select_auth_type( diff --git a/caldav/lib/error.py b/caldav/lib/error.py index 16d79883..2822c0b7 100644 --- a/caldav/lib/error.py +++ b/caldav/lib/error.py @@ -13,6 +13,12 @@ ## Environmental variables prepended with "PYTHON_CALDAV" are used for debug purposes, ## environmental variables prepended with "CALDAV_" are for connection parameters debug_dump_communication = os.environ.get("PYTHON_CALDAV_COMMDUMP", False) + if debug_dump_communication: + logging.getLogger("caldav").warning( + "PYTHON_CALDAV_COMMDUMP is set: request/response bodies and headers " + "(including credentials and calendar PII) will be written to uniquely-named " + "files under /tmp. These files accumulate indefinitely — remove them when done." + ) ## one of DEBUG_PDB, DEBUG, DEVELOPMENT, PRODUCTION debugmode = os.environ["PYTHON_CALDAV_DEBUGMODE"] except KeyError: diff --git a/caldav/lib/url.py b/caldav/lib/url.py index c2b426e0..3390371b 100644 --- a/caldav/lib/url.py +++ b/caldav/lib/url.py @@ -140,7 +140,13 @@ def canonical(self) -> "URL": """ url = self.unauth() - arr = list(cast(urllib.parse.ParseResult, self.url_parsed)) + # Use url's parsed form (credentials already stripped), not self's. + # Also always build a fresh URL so self is never mutated — unauth() + # returns self when there are no credentials, and the old code then + # overwrote url.url_raw/url_parsed which are the same object as self. + if url.url_parsed is None: + url.url_parsed = cast(urllib.parse.ParseResult, urlparse(str(url))) + arr = list(url.url_parsed) ## quoting path and removing double slashes arr[2] = quote(unquote(url.path.replace("//", "/"))) ## sensible defaults @@ -155,11 +161,7 @@ def canonical(self) -> "URL": portpart = "" arr[1] += portpart - # make sure to delete the string version - url.url_raw = urlunparse(arr) - url.url_parsed = None - - return url + return URL(urlunparse(arr)) def join(self, path: Any) -> "URL": """ diff --git a/caldav/lib/vcal.py b/caldav/lib/vcal.py index fb29f7cd..4748e723 100644 --- a/caldav/lib/vcal.py +++ b/caldav/lib/vcal.py @@ -77,20 +77,24 @@ def fix(event): ## TODO: add ^ before COMPLETED and CREATED? ## 1) Add an arbitrary time if completed is given as date - fixed = re.sub(r"COMPLETED(?:;VALUE=DATE)?:(\d+)\s", r"COMPLETED:\g<1>T120000Z", event) + fixed = re.sub(r"COMPLETED(?:;VALUE=DATE)?:(\d+)(?=\s)", r"COMPLETED:\g<1>T120000Z", event) ## 2) CREATED timestamps prior to epoch does not make sense, ## change from year 0001 to epoch. fixed = re.sub("CREATED:00001231T000000Z", "CREATED:19700101T000000Z", fixed) - fixed = re.sub(r"\\+('\")", r"\1", fixed) + fixed = re.sub(r"\\+(['\"])", r"\1", fixed) ## 4) trailing whitespace probably never makes sense - fixed = re.sub(" *$", "", fixed) + fixed = re.sub(" *$", "", fixed, flags=re.MULTILINE) ## 6) add DTSTAMP if not given ## (corner case that DTSTAMP is given in one but not all the recurrences is ignored) if "\nDTSTAMP:" not in fixed: - assert "\nEND" in fixed + if "\nEND" not in fixed: + logging.getLogger(__name__).warning( + "vcal.fix(): truncated iCalendar data (no END: line) — skipping DTSTAMP fixup" + ) + return fixed dtstamp = datetime.datetime.now(tz=datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ") fixed = re.sub("(\nEND:(VTODO|VEVENT|VJOURNAL))", f"\nDTSTAMP:{dtstamp}\\1", fixed) @@ -240,8 +244,8 @@ def create_ical(ical_fragment=None, objtype=None, language="en_DK", **props): ret = to_normal_str(my_instance.to_ical()) if ical_fragment and ical_fragment.strip(): ret = re.sub( - "^END:V", - ical_fragment.strip() + "\nEND:V", + "^(END:V(?:EVENT|TODO|JOURNAL))", + ical_fragment.strip() + "\n\\1", ret, flags=re.MULTILINE, count=1, diff --git a/caldav/response.py b/caldav/response.py index 0b27d5b9..35b0b0cb 100644 --- a/caldav/response.py +++ b/caldav/response.py @@ -116,22 +116,31 @@ def _strip_to_multistatus(tree: _Element) -> "_Element | list[_Element]": return [tree] -def _extract_properties(propstats: "list[_Element]") -> "dict[str, Any]": - """Extract properties from propstat elements into a flat dict.""" - properties: dict[str, Any] = {} +def _collect_prop_elements(propstats: "list[_Element]") -> "dict[str, _Element]": + """Collect ``{proptag: element}`` from a list of propstat elements. + + Propstats whose status reports 404 are skipped — that is the single, + shared expression of the "a 404 propstat means the property is absent on + the resource" quirk. This helper is the one place both the dataclass + parsers (via :func:`_extract_properties`) and the legacy + :meth:`DAVResponse._find_objects_and_props` collect prop children, so the + quirk no longer has to be maintained in two parallel loops (code-review + §5.7). + """ + collected: dict[str, _Element] = {} for propstat in propstats: status_elem = propstat.find(dav.Status.tag) if status_elem is not None and status_elem.text and " 404 " in status_elem.text: continue - prop = propstat.find(dav.Prop.tag) - if prop is None: - continue - for child in prop: - if len(child) == 0: - properties[child.tag] = child.text - else: - properties[child.tag] = _element_to_value(child) - return properties + for prop in propstat.iterfind(dav.Prop.tag): + for child in prop: + collected[child.tag] = child + return collected + + +def _extract_properties(propstats: "list[_Element]") -> "dict[str, Any]": + """Extract properties from propstat elements into a flat dict of parsed values.""" + return {tag: _element_to_value(el) for tag, el in _collect_prop_elements(propstats).items()} def _element_to_value(elem: _Element) -> Any: @@ -274,7 +283,12 @@ def _init_from_response(self, response: "Response", davclient: Any = None) -> No # We'll try to parse the content as XML no matter the content type. self.tree = etree.XML( self._raw, - parser=etree.XMLParser(remove_blank_text=True, huge_tree=self.huge_tree), + parser=etree.XMLParser( + remove_blank_text=True, + huge_tree=self.huge_tree, + resolve_entities=False, + no_network=True, + ), ) except Exception: # Content wasn't XML. What does the content-type say? @@ -474,28 +488,24 @@ def _parse_response(self, response: _Element) -> tuple[str, list[_Element], Any href = _normalize_href(elem.text or "") elif elem.tag == dav.PropStat.tag: propstats.append(elem) - elif elem.tag == "{DAV:}responsedescription": - ## This happens with Stalwart on a 404. - ## This code is mostly moot, but in debug - ## mode I want to be sure we do not toss away any data - error.assert_(elem.text == "No resources found") - check_404 = True - elif elem.tag == "{DAV:}error": - ## This happens with purelymail on a 404. - ## This code is mostly moot, but in debug - ## mode I want to be sure we do not toss away any data - children = elem.getchildren() - error.assert_(len(children) == 1) - error.assert_(children[0].tag == "{https://purelymail.com}does-not-exist") + elif elem.tag in ("{DAV:}responsedescription", "{DAV:}error"): + ## Both are optional children of per RFC 4918 + ## and carry server-defined content. We've seen them on + ## 404s (Stalwart sends No resources + ## found, purelymail sends + ## <…:does-not-exist/>), so accept any such + ## element generically rather than fingerprinting servers. check_404 = True else: - ## i.e. purelymail may contain one more tag, ... - ## This is probably not a breach of the standard. It may - ## probably be ignored. But it's something we may want to - ## know. + ## A tag we don't recognise at all (e.g. a server inventing + ## an element). Not necessarily a standards + ## breach and probably ignorable, but worth surfacing. error.weirdness("unexpected element found in response", elem) error.assert_(href) - if check_404: + if check_404 and status: + ## We've only ever observed / on + ## 404s; flag it in debug mode if a server pairs them with some + ## other status so we notice and revisit this handling. error.assert_("404" in status) return (cast(str, href), propstats, status) @@ -606,27 +616,11 @@ def _find_objects_and_props(self) -> dict[str, dict[str, _Element]]: self.objects[href] = {} self.statuses[href] = status - ## The properties may be delivered either in one - ## propstat with multiple props or in multiple - ## propstat - for propstat in propstats: - cnt = 0 - status = propstat.find(dav.Status.tag) - error.assert_(status is not None) - if status is not None and status.text is not None: - error.assert_(len(status) == 0) - cnt += 1 - self.validate_status(status.text) - ## if a prop was not found, ignore it - if " 404 " in status.text: - continue - for prop in propstat.iterfind(dav.Prop.tag): - cnt += 1 - for theprop in prop: - self.objects[href][theprop.tag] = theprop - - ## there shouldn't be any more elements except for status and prop - error.assert_(cnt == len(propstat)) + ## The properties may be delivered either in one propstat + ## with multiple props or in multiple propstats; the 404-skip + ## quirk is shared with the dataclass parsers via + ## _collect_prop_elements (code-review §5.7). + self.objects[href].update(_collect_prop_elements(propstats)) return self.objects diff --git a/caldav/search.py b/caldav/search.py index 5b853382..7b0c1351 100644 --- a/caldav/search.py +++ b/caldav/search.py @@ -17,6 +17,8 @@ from .lib import error if TYPE_CHECKING: + from collections.abc import Generator + from .calendarobjectresource import ( CalendarObjectResource as AsyncCalendarObjectResource, ) @@ -190,7 +192,8 @@ def _build_search_xml_query( for property in searcher._property_operator: if searcher._property_operator[property] == "undef": match = cdav.NotDefined() - filters.append(cdav.PropFilter(property.upper()) + match) + prop_name = "CATEGORIES" if property.lower() == "category" else property.upper() + filters.append(cdav.PropFilter(prop_name) + match) else: value = searcher._property_filters[property] property_ = property.upper() @@ -267,9 +270,39 @@ class SearchAction(Enum): SEARCH_WITH_COMPTYPES = auto() # (args) -> search with all comp types REQUEST_REPORT = auto() # (xml, comp_class, props) -> make CalDAV request LOAD_OBJECT = auto() # (obj) -> load object data + LOAD_OBJECTS_BATCH = ( + auto() + ) # (calendar, objects) -> batch-load via calendar._batch_load_objects RETURN = auto() # (result) -> return this value +def _advance_search_gen( + gen: "Generator[tuple[SearchAction, Any], Any, None]", + result: Any = None, + exc: BaseException | None = None, +) -> "tuple[SearchAction, Any] | None": + """Phase 2 of the search driver protocol, shared by sync and async drivers. + + Feed the Phase-1 ``result`` (or the ``exc`` raised while executing the + yielded action) back into the search generator. Feeding an exception via + ``gen.throw()`` lets the search logic's own try/except blocks act on it + (the issue #681 time-range fallback, per-object load error handling, ...); + if the generator does not handle it, ``gen.throw()`` re-raises it out of + here, which is the correct propagation. + + Passing ``result=None, exc=None`` on a fresh generator primes it. + + :return: the next ``(action, data)`` to execute, or ``None`` when the + generator is exhausted (StopIteration → the driver returns ``[]``). + """ + try: + if exc is not None: + return gen.throw(exc) + return gen.send(result) + except StopIteration: + return None + + @dataclass class CalDAVSearcher(Searcher): """The baseclass (which is generic, and not CalDAV-specific) @@ -316,6 +349,12 @@ class CalDAVSearcher(Searcher): comp_class: Optional["CalendarObjectResource"] = None _explicit_operators: set = field(default_factory=set) _calendar: Optional["Calendar"] = field(default=None, repr=False) + ## When False, all server-compatibility workarounds in _search_impl are + ## disabled and the query the searcher describes is sent verbatim (a single + ## REPORT, no comp-type splitting, no filter rewriting, no fallback retries). + ## Used by the server-compatibility checker to observe raw server behaviour. + ## Propagates to clones automatically via dataclasses.replace(). + _compatibility_workarounds: bool = True def add_property_filter( self, @@ -455,12 +494,18 @@ def _search_impl( "create the searcher via calendar.searcher()" ) + ## When disabled, every server-compatibility workaround below is skipped + ## and the query is sent verbatim (used by the compatibility checker to + ## observe raw server behaviour). + cw = self._compatibility_workarounds + ## Workaround for servers where REPORT without a time range only returns ## objects within a sliding window (search.unlimited-time-range: broken). ## Inject a wide time range covering 1970–2126 so that year-2000 test ## objects and other old data are returned. if ( - not self.start + cw + and not self.start and not self.end and not (self.expand or server_expand) and not calendar.client.features.is_supported("search.unlimited-time-range") @@ -480,7 +525,8 @@ def _search_impl( ## Handle servers with broken component-type filtering (e.g., Bedework) comp_type_support = calendar.client.features.is_supported("search.comp-type", str) no_comp_filter = ( - (self.comp_class or self.todo or self.event or self.journal) + cw + and (self.comp_class or self.todo or self.event or self.journal) and comp_type_support == "broken" and post_filter is not False ) @@ -490,13 +536,18 @@ def _search_impl( post_filter = True ## Setting default value for post_filter - if post_filter is None and ( - (self.todo and not self.include_completed) - or self.expand - or "categories" in self._property_filters - or "category" in self._property_filters - or not calendar.client.features.is_supported("search.text.case-sensitive") - or not calendar.client.features.is_supported("search.time-range.accurate") + if ( + cw + and post_filter is None + and ( + (self.todo and not self.include_completed) + or self.expand + or "categories" in self._property_filters + or "category" in self._property_filters + or any(op == "==" for op in self._property_operator.values()) + or not calendar.client.features.is_supported("search.text.case-sensitive") + or not calendar.client.features.is_supported("search.time-range.accurate") + ) ): post_filter = True @@ -508,7 +559,8 @@ def _search_impl( ## expansion is unreliable (the master expands without knowing its exceptions, yielding ## duplicate occurrences). Fall back to server-side expansion when it handles exceptions. if ( - self.expand + cw + and self.expand and not server_expand and not calendar.client.features.is_supported("save-load.event.recurrences.exception") and calendar.client.features.is_supported("search.recurrences.expanded.exception") @@ -523,7 +575,8 @@ def _search_impl( ## (e.g. purelymail where both i;octet and i;ascii-casemap collations are unsupported). ## Remove all text-value filters and rely on client-side post_filter instead. if ( - not calendar.client.features.is_supported("search.text") + cw + and not calendar.client.features.is_supported("search.text") and self._property_filters and post_filter is not False ): @@ -545,7 +598,8 @@ def _search_impl( ## special compatbility-case for servers that does not ## support category search properly if ( - not calendar.client.features.is_supported("search.text.category") + cw + and not calendar.client.features.is_supported("search.text.category") and ("categories" in self._property_filters or "category" in self._property_filters) and post_filter is not False ): @@ -562,7 +616,7 @@ def _search_impl( ## special compatibility-case for servers that do not support is-not-defined ## for specific properties (e.g. search.is-not-defined.category or .dtend) - if post_filter is not False: + if cw and post_filter is not False: undef_props_without_support = [ prop for prop, op in self._property_operator.items() @@ -587,7 +641,8 @@ def _search_impl( ## special compatibility-case for servers that do not support substring search if ( - not calendar.client.features.is_supported("search.text.substring") + cw + and not calendar.client.features.is_supported("search.text.substring") and post_filter is not False ): explicit_contains = [ @@ -614,7 +669,7 @@ def _search_impl( ## special compatibility-case for servers that does not ## support combined searches very well - if not calendar.client.features.is_supported("search.combined-is-logical-and"): + if cw and not calendar.client.features.is_supported("search.combined-is-logical-and"): if self.start or self.end: if self._property_filters: clone = self._clone_without_filters(clear_all_filters=True) @@ -624,7 +679,7 @@ def _search_impl( ) yield ( SearchAction.RETURN, - self.filter(objects, post_filter, split_expanded, server_expand), + self.filter(objects, True, split_expanded, server_expand), ) return @@ -657,7 +712,7 @@ def _search_impl( ## TODO: consider if not ignore_completed3 is sufficient, ## then the recursive part of the query here is moot, and ## we wouldn't waste so much time on repeated queries - if self.todo and self.include_completed is False: + if cw and self.todo and self.include_completed is False: clone = replace(self, include_completed=True) clone.include_completed = True ## Why? Isn't this redundant? clone.expand = False @@ -703,9 +758,43 @@ def _search_impl( server_expand, props=props, filters=xml, _hacks=_hacks ) - if not self.comp_class and not calendar.client.features.is_supported( - "search.comp-type.optional" - ): + ## A CALDAV:time-range (and VALARM) filter is a component-level filter: + ## RFC4791 section 9.7 only allows it inside a comp-filter for + ## VEVENT/VTODO/VJOURNAL/VFREEBUSY/VALARM, never directly under VCALENDAR. + ## So when no component type is given we cannot place such a filter in an + ## RFC-legal way - we must split the search into one query per component + ## type (search.time-range.comp-type-optional). This is independent of + ## search.comp-type.optional, which only governs comp-type-less queries + ## WITHOUT any filter. + ## The same applies to a prop-filter (CATEGORIES, SUMMARY, ...): under + ## VCALENDAR it would filter on VCALENDAR's own properties (which lack + ## component properties), so servers match nothing + ## (search.text.comp-type-optional). + ## See https://github.com/python-caldav/caldav/issues/681 + has_component_level_filter = bool( + self.start or self.end or self.alarm_start or self.alarm_end + ) + has_property_filter = bool(self._property_filters) + needs_comptype_split = ( + cw + and not self.comp_class + and ( + not calendar.client.features.is_supported("search.comp-type.optional") + or ( + has_component_level_filter + and not calendar.client.features.is_supported( + "search.time-range.comp-type-optional" + ) + ) + or ( + has_property_filter + and not calendar.client.features.is_supported( + "search.text.comp-type-optional" + ) + ) + ) + ) + if needs_comptype_split: if self.include_completed is None: self.include_completed = True @@ -722,8 +811,37 @@ def _search_impl( (calendar, xml, self.comp_class, props), ) except error.ReportError as err: + ## Reactive workaround for https://github.com/python-caldav/caldav/issues/681: + ## if the server was (optimistically) configured as supporting + ## search.time-range.comp-type-optional but actually rejects the + ## comp-type-less time-range query (e.g. SabreDAV's HTTP 400 "You cannot + ## add time-range filters on the VCALENDAR component"), retry by splitting + ## into one query per component type. Also covers prop-filters + ## (search.text.comp-type-optional). orig_xml must be empty - if the + ## caller passed a full calendar-query we cannot rebuild it per comp-type. + if ( + cw + and not self.comp_class + and not orig_xml + and (has_component_level_filter or has_property_filter) + ): + result = yield ( + SearchAction.SEARCH_WITH_COMPTYPES, + ( + calendar, + server_expand, + split_expanded, + props, + orig_xml, + _hacks, + post_filter, + ), + ) + yield (SearchAction.RETURN, result) + return if ( - calendar.client.features.backward_compatibility_mode + cw + and calendar.client.features.backward_compatibility_mode and not self.comp_class and "400" not in err.reason ): @@ -780,29 +898,17 @@ def _search_impl( ) return - # Post-process: load objects - obj2 = [] - for o in objects: - try: - yield (SearchAction.LOAD_OBJECT, o) - obj2.append(o) - except Exception: - logging.error( - "Server does not want to reveal details about the calendar object", - exc_info=True, - ) - objects = obj2 + # Post-process: batch-load unloaded objects in one REPORT instead of N GETs + yield (SearchAction.LOAD_OBJECTS_BATCH, (calendar, objects)) + objects = [o for o in objects if o.is_loaded() or o.has_component()] # Google sometimes returns empty objects objects = [o for o in objects if o.has_component()] objects = self.filter(objects, post_filter, split_expanded, server_expand) # Partial workaround for https://github.com/python-caldav/caldav/issues/201 - for obj in objects: - try: - yield (SearchAction.LOAD_OBJECT, obj) - except Exception: - pass + # Re-issue a batch load in case any objects need a second fetch + yield (SearchAction.LOAD_OBJECTS_BATCH, (calendar, objects)) yield (SearchAction.RETURN, self.sort(objects)) @@ -815,6 +921,7 @@ def search( xml: str = None, post_filter=None, _hacks: str = None, + compatibility_workarounds: bool | None = None, ) -> list[CalendarObjectResource]: """Do the search on a CalDAV calendar. @@ -831,6 +938,13 @@ def search( :param xml: XML query to be sent to the server (string or elements) :param post_filter: Do client-side filtering after querying the server :param _hacks: Please don't ask! + :param compatibility_workarounds: When ``False``, all server-compatibility + workarounds are disabled and the query is sent verbatim + (single REPORT, no comp-type splitting, no filter + rewriting, no fallback retries). Mainly for the + server-compatibility checker, to observe raw server + behaviour. ``None`` (the default) leaves the searcher's + current setting unchanged. Make sure not to confuse he CalDAV properties with iCalendar properties. @@ -851,36 +965,52 @@ def search( flag on. """ + if compatibility_workarounds is not None: + self._compatibility_workarounds = compatibility_workarounds gen = self._search_impl( calendar, server_expand, split_expanded, props, xml, post_filter, _hacks ) - result = None - - try: - action, data = gen.send(result) - except StopIteration: - return [] - while True: + ## The driver alternates Phase 1 (execute the yielded action, here) and + ## Phase 2 (feed the result/exception back, in _advance_search_gen). Only + ## Phase 1 differs between sync and async; the generator protocol is shared. + step = _advance_search_gen(gen) # prime the generator + while step is not None: + action, data = step + if action == SearchAction.RETURN: + return data + result = exc = None try: - if action == SearchAction.RECURSIVE_SEARCH: - clone, cal, srv_exp, spl_exp, prp, xm, pf, hk = data - result = clone.search(cal, srv_exp, spl_exp, prp, xm, pf, hk) - elif action == SearchAction.SEARCH_WITH_COMPTYPES: - cal, srv_exp, spl_exp, prp, xm, hk, pf = data - result = self._search_with_comptypes(cal, srv_exp, spl_exp, prp, xm, hk, pf) - elif action == SearchAction.REQUEST_REPORT: - cal, xm, comp_cls, prp = data - result = cal._request_report_build_resultlist(xm, comp_cls, props=prp) - elif action == SearchAction.LOAD_OBJECT: - data.load(only_if_unloaded=True) - result = None - elif action == SearchAction.RETURN: - return data - - action, data = gen.send(result) - except StopIteration: - return [] + result = self._dispatch_search_action(action, data) + except Exception as e: + exc = e + step = _advance_search_gen(gen, result, exc) + return [] + + def _dispatch_search_action(self, action: SearchAction, data: Any) -> Any: + """Phase 1 of the sync search driver: execute one yielded SearchAction. + + Returns the value to feed back into the generator (``None`` for actions + whose effect is a side effect). ``RETURN`` is handled by the driver + loop itself. Sync twin of :meth:`_async_dispatch_search_action`. + """ + if action == SearchAction.RECURSIVE_SEARCH: + clone, cal, srv_exp, spl_exp, prp, xm, pf, hk = data + return clone.search(cal, srv_exp, spl_exp, prp, xm, pf, hk) + if action == SearchAction.SEARCH_WITH_COMPTYPES: + cal, srv_exp, spl_exp, prp, xm, hk, pf = data + return self._search_with_comptypes(cal, srv_exp, spl_exp, prp, xm, hk, pf) + if action == SearchAction.REQUEST_REPORT: + cal, xm, comp_cls, prp = data + return cal._request_report_build_resultlist(xm, comp_cls, props=prp) + if action == SearchAction.LOAD_OBJECT: + data.load(only_if_unloaded=True) + return None + if action == SearchAction.LOAD_OBJECTS_BATCH: + cal, objs = data + cal._batch_load_objects(objs) + return None + raise AssertionError(f"unhandled search action {action!r}") def _search_with_comptypes( self, @@ -927,6 +1057,7 @@ async def async_search( xml: str = None, post_filter=None, _hacks: str = None, + compatibility_workarounds: bool | None = None, ) -> list["AsyncCalendarObjectResource"]: """Async version of search() - does the search on an AsyncCalendar. @@ -935,40 +1066,52 @@ async def async_search( See the sync search() method for full documentation. """ + if compatibility_workarounds is not None: + self._compatibility_workarounds = compatibility_workarounds gen = self._search_impl( calendar, server_expand, split_expanded, props, xml, post_filter, _hacks ) - result = None - try: - action, data = gen.send(result) - except StopIteration: - return [] - - while True: + ## See the sync search() driver: only Phase 1 (the action execution) is + ## awaited here; the Phase-2 generator protocol is shared via + ## _advance_search_gen. + step = _advance_search_gen(gen) # prime the generator + while step is not None: + action, data = step + if action == SearchAction.RETURN: + return data + result = exc = None try: - if action == SearchAction.RECURSIVE_SEARCH: - clone, cal, srv_exp, spl_exp, prp, xm, pf, hk = data - result = await clone.async_search(cal, srv_exp, spl_exp, prp, xm, pf, hk) - elif action == SearchAction.SEARCH_WITH_COMPTYPES: - cal, srv_exp, spl_exp, prp, xm, hk, pf = data - result = await self._async_search_with_comptypes( - cal, srv_exp, spl_exp, prp, xm, hk, pf - ) - elif action == SearchAction.REQUEST_REPORT: - cal, xm, comp_cls, prp = data - result = await cal._request_report_build_resultlist(xm, comp_cls, props=prp) - elif action == SearchAction.LOAD_OBJECT: - load_result = data.load(only_if_unloaded=True) - if inspect.isawaitable(load_result): - await load_result - result = None - elif action == SearchAction.RETURN: - return data - - action, data = gen.send(result) - except StopIteration: - return [] + result = await self._async_dispatch_search_action(action, data) + except Exception as e: + exc = e + step = _advance_search_gen(gen, result, exc) + return [] + + async def _async_dispatch_search_action(self, action: SearchAction, data: Any) -> Any: + """Phase 1 of the async search driver: execute one yielded SearchAction. + + Async twin of :meth:`_dispatch_search_action`; see it for semantics. + """ + if action == SearchAction.RECURSIVE_SEARCH: + clone, cal, srv_exp, spl_exp, prp, xm, pf, hk = data + return await clone.async_search(cal, srv_exp, spl_exp, prp, xm, pf, hk) + if action == SearchAction.SEARCH_WITH_COMPTYPES: + cal, srv_exp, spl_exp, prp, xm, hk, pf = data + return await self._async_search_with_comptypes(cal, srv_exp, spl_exp, prp, xm, hk, pf) + if action == SearchAction.REQUEST_REPORT: + cal, xm, comp_cls, prp = data + return await cal._request_report_build_resultlist(xm, comp_cls, props=prp) + if action == SearchAction.LOAD_OBJECT: + load_result = data.load(only_if_unloaded=True) + if inspect.isawaitable(load_result): + await load_result + return None + if action == SearchAction.LOAD_OBJECTS_BATCH: + cal, objs = data + await cal._async_batch_load_objects(objs) + return None + raise AssertionError(f"unhandled search action {action!r}") async def _async_search_with_comptypes( self, diff --git a/caldav/testing.py b/caldav/testing.py index 1211f30b..f87c2730 100644 --- a/caldav/testing.py +++ b/caldav/testing.py @@ -9,6 +9,7 @@ Docker and external server support lives in tests/test_servers/ (source only). """ +import copy import socket import tempfile import threading @@ -123,7 +124,7 @@ def __init__(self, config: dict[str, Any] | None = None) -> None: if "features" not in config: from caldav import compatibility_hints - features = compatibility_hints.xandikos.copy() + features = copy.deepcopy(compatibility_hints.xandikos) features["auto-connect.url"]["domain"] = f"{config['host']}:{config['port']}" config["features"] = features super().__init__(config) @@ -265,7 +266,7 @@ def __init__(self, config: dict[str, Any] | None = None) -> None: if "features" not in config: from caldav import compatibility_hints - features = compatibility_hints.radicale.copy() + features = copy.deepcopy(compatibility_hints.radicale) features["auto-connect.url"]["domain"] = f"{config['host']}:{config['port']}" config["features"] = features super().__init__(config) diff --git a/docs/design/FULL_CODE_REVIEW_2026-06.md b/docs/design/FULL_CODE_REVIEW_2026-06.md index 988887c3..61fac23f 100644 --- a/docs/design/FULL_CODE_REVIEW_2026-06.md +++ b/docs/design/FULL_CODE_REVIEW_2026-06.md @@ -51,7 +51,7 @@ real bugs, clustering around four themes: ## 1. Crash bugs (realistic trigger → unhandled exception) -### 1.1 `calendarobjectresource.py:1167` + `:1187` — 302 handling iterates headers as tuples `[repro]` +### 1.1 `calendarobjectresource.py:1167` + `:1187` — 302 handling iterates headers as tuples `[repro]` ✅ FIXED (commit 22b9cc66+1) `[x[1] for x in r.headers if x[0] == "location"][0]` — iterating a dict-like `Headers` object (niquests `CaseInsensitiveDict` sync, `httpx.Headers` async) yields key *strings*, so `x[0]` is the first character of each header name. @@ -60,7 +60,7 @@ instead of following the redirect. The same broken pattern appears twice because the whole block is pasted twice (see §6.2; the second copy is partly dead code). Fix: `r.headers.get("location")`. -### 1.2 `davclient.py:302` — URL with username but no password → TypeError `[repro]` +### 1.2 `davclient.py:302` — URL with username but no password → TypeError `[repro]` ✅ FIXED `DAVClient(url='https://user@example.com/dav/', password='secret')`: `self.url.username` is set, so `unquote(self.url.password)` runs with `password=None` → TypeError inside `urllib.parse.unquote`. The async client @@ -68,7 +68,7 @@ dead code). Fix: `r.headers.get("location")`. also gives **explicit kwargs precedence over URL credentials, while sync does the opposite**. Pick one precedence (kwargs should win) and share the code. -### 1.3 `davclient.py:836` / `async_davclient.py:376` — rate-limit retry: `None + float` `[code]` +### 1.3 `davclient.py:836` / `async_davclient.py:376` — rate-limit retry: `None + float` `[code]` ✅ FIXED `sleep_seconds += rate_limit_time_slept / 2` executes *before* the `sleep_seconds is None` check. With `rate_limit_handle=True` and `rate_limit_default_sleep=None`: first 429 has `Retry-After: 5` → retried; @@ -76,7 +76,7 @@ second 429 has no usable Retry-After (`compute_sleep_seconds` returns None, e.g. `Retry-After: 0`) → `None += 2.5` → TypeError instead of the documented `RateLimitError`. Same bug copy-pasted in both clients. -### 1.4 `async_davclient.py:1272` — `aio.get_calendars(calendar_name=...)` can never work `[code]` +### 1.4 `async_davclient.py:1272` — `aio.get_calendars(calendar_name=...)` can never work `[code]` ✅ FIXED The async module-level helper awaits the *synchronous* `Principal.calendar()`, which has no async dispatch (`collection.py:448–475`): `calendar_home_set` → `get_property` returns a coroutine for async clients, and @@ -85,7 +85,7 @@ coroutine → TypeError (swallowed into an empty result when `raise_errors=False`). Name-based calendar lookup via `caldav.aio` is broken end-to-end. -### 1.5 `collection.py:601` — async `freebusy_request` with Principal attendees → AttributeError `[code]` +### 1.5 `collection.py:601` — async `freebusy_request` with Principal attendees → AttributeError `[code]` ✅ FIXED `add_attendee(attendee)` is called *before* the `is_async_client` branch at line 604. For a `Principal` attendee on an async client, `get_vcal_address()` returns a coroutine, and `add_attendee` then does @@ -93,13 +93,13 @@ line 604. For a `Principal` attendee on an async client, `_async_save_with_invites` (`collection.py:983–984`) already does the awaited conversion correctly — the same dance is missing here. -### 1.6 `calendarobjectresource.py:727` — `add_attendee("MAILTO:user@example.com")` → UnboundLocalError `[code]` +### 1.6 `calendarobjectresource.py:727` — `add_attendee("MAILTO:user@example.com")` → UnboundLocalError `[code]` ✅ FIXED The string-branch chain is case-sensitive: uppercase `MAILTO:` (common in real-world iCalendar; RFC 3986 schemes are case-insensitive) fails `startswith("mailto:")` and fails the `":" not in attendee` branch, so `attendee_obj` is never assigned and line 742 raises UnboundLocalError. -### 1.7 `calendarobjectresource.py:1272` — `change_attendee_status` raises bare KeyError; `:1284` literal `%s` `[repro]` +### 1.7 `calendarobjectresource.py:1272` — `change_attendee_status` raises bare KeyError; `:1284` literal `%s` `[repro]` ✅ FIXED When the component has no ATTENDEE property at all, `ical_obj['attendee']` raises `KeyError('ATTENDEE')` — not `error.NotFoundError`, which is the only thing the principal-address loops catch — so the "Principal is not invited" @@ -107,38 +107,38 @@ fallback is unreachable. Additionally the genuine not-found raise is `error.NotFoundError("Participant %s not found in attendee list")` with no `% attendee`: the user literally sees `%s`. -### 1.8 `lib/auth.py:31` — IndexError on malformed WWW-Authenticate `[repro]` +### 1.8 `lib/auth.py:31` — IndexError on malformed WWW-Authenticate `[repro]` ✅ FIXED `extract_auth_types('Basic realm="x",')` (trailing comma — seen in the wild) → the empty segment makes `h.split()[0]` raise IndexError, aborting the auth negotiation with an unrelated traceback. Guard with `for h in header.split(",") if h.strip()`. -### 1.9 `config.py:37` — missing section raises KeyError instead of returning empty `[repro]` +### 1.9 `config.py:37` — missing section raises KeyError instead of returning empty `[repro]` ✅ FIXED `expand_config_section` does `config[section]` for non-glob names. A config file with only named sections (no `default`) makes plain `caldav.get_calendars()` crash with `KeyError: 'default'` instead of falling through to "no configuration found". -### 1.10 `compatibility_hints.py:611` — `copyFeatureSet` crashes merging plain-string features `[repro]` +### 1.10 `compatibility_hints.py:611` — `copyFeatureSet` crashes merging plain-string features `[repro]` ✅ FIXED `FeatureSet({'scheduling': 'unsupported'}).copyFeatureSet({'scheduling': 'fragile'})` → bare AssertionError: the `'support' not in server_node` guard makes string-valued updates of an existing feature fall through to the final `else: raise AssertionError`. Plain strings are the dominant style in the hint dicts, so any two-layer merge expressing the same feature crashes. -### 1.11 `compatibility_hints.py:605` — unknown feature names: warn now, crash later `[repro]` +### 1.11 `compatibility_hints.py:605` — unknown feature names: warn now, crash later `[repro]` ✅ FIXED A typoed feature name in a user's config produces only a UserWarning at set time, but the bad key is still stored — a later `collapse()` / `is_supported()` hits a message-less AssertionError in `find_feature`, far from the config that caused it. Reject (or drop) the key at intake instead. -### 1.12 `lib/vcal.py:93` — bare `assert` on server-supplied data `[repro]` +### 1.12 `lib/vcal.py:93` — bare `assert` on server-supplied data `[repro]` ✅ FIXED Truncated/garbage iCalendar without DTSTAMP and without an `END:` line makes `fix()` raise a bare AssertionError. Under `python -O` the assert (and thus the DTSTAMP fixup logic it guards) is silently skipped. Should be `error.assert_` or a proper parse error. -### 1.13 `jmap/client.py:576` / `jmap/async_client.py:461` — `create_task` missing the guard `create_event` has `[code]` +### 1.13 `jmap/client.py:576` / `jmap/async_client.py:461` — `create_task` missing the guard `create_event` has `[code]` ✅ FIXED `create_event` handles an empty `created` dict with a descriptive `JMAPMethodError` (`client.py:294–298`); `create_task` does `created["new-0"]["id"]` unguarded → bare KeyError, bypassing the JMAP error @@ -148,7 +148,7 @@ hierarchy callers are told to catch. Copy-paste gap in both clients. ## 2. Silent wrong results / data corruption -### 2.1 `lib/vcal.py:80` — COMPLETED fixup merges the next line into the property ⚠ data corruption `[repro]` +### 2.1 `lib/vcal.py:80` — COMPLETED fixup merges the next line into the property ⚠ data corruption `[repro]` ✅ FIXED (commit 22b9cc66) `fix()` normalizes CRLF→LF first, then the COMPLETED date-to-datetime regex `(\d+)\s` *consumes the newline without restoring it*: `COMPLETED:20240101\nSUMMARY:hello` becomes @@ -156,24 +156,24 @@ hierarchy callers are told to catch. Copy-paste gap in both clients. destroyed and the object parses with corrupted data. This runs on every inbound object. -### 2.2 `lib/vcal.py:242` — `create_ical(ical_fragment=...)` injects the fragment inside VALARM `[repro]` +### 2.2 `lib/vcal.py:242` — `create_ical(ical_fragment=...)` injects the fragment inside VALARM `[repro]` ✅ FIXED The fragment is re-inserted before the first `^END:V` line — which is `END:VALARM` when any `alarm_*` props were given. `ical_fragment='RRULE:...'` plus an alarm produces an event *without* recurrence and with an invalid RRULE inside the alarm. Should target `END:VEVENT|VTODO|VJOURNAL`. -### 2.3 `lib/vcal.py:88` — trailing-whitespace fixup is dead code `[repro]` +### 2.3 `lib/vcal.py:88` — trailing-whitespace fixup is dead code `[repro]` ✅ FIXED `re.sub(" *$", "", fixed)` without `re.MULTILINE` only touches the document end, never the per-line trailing spaces (iCloud X-APPLE-STRUCTURED-EVENT) that docstring fix #4 targets. The vobject traceback it was written to prevent still occurs. -### 2.4 `lib/vcal.py:85` — backslash-unescape regex is a no-op `[repro]` +### 2.4 `lib/vcal.py:85` — backslash-unescape regex is a no-op `[repro]` ✅ FIXED `re.sub(r"\\+('\")", r"\1", fixed)` matches only the literal two-character sequence `'"`; the group should be a character class `['\"]`. Harmless for compliant data, but the fix does nothing. -### 2.5 `lib/url.py:143` + `:159` — `canonical()` keeps credentials and mutates self `[repro]` +### 2.5 `lib/url.py:143` + `:159` — `canonical()` keeps credentials and mutates self `[repro]` ✅ FIXED Two related bugs: (a) `canonical()` builds its result from `self.url_parsed` instead of the `unauth()`'ed URL, so `URL('https://user:pass@example.com/cal/').canonical()` **retains the @@ -184,7 +184,7 @@ then overwrites `url_raw`/`url_parsed` **in place** — a mere `==` comparison silently rewrites the URL (port added, path re-quoted; a literal `+` becomes `%2B`), so subsequent requests can go to a different resource. -### 2.6 `search.py:648` — `combined-is-logical-and` workaround silently drops property filters `[code]` +### 2.6 `search.py:648` — `combined-is-logical-and` workaround silently drops property filters `[code]` ✅ FIXED The workaround strips property filters from the server query but passes the *ambient* `post_filter` (still `None` on otherwise-capable servers — e.g. Nextcloud, whose only relevant flag is `search.combined-is-logical-and: @@ -194,58 +194,58 @@ range. The sibling workarounds at 597–604 and 625–632 correctly force `post_filter=True`; this branch also uniquely lacks the `post_filter is not False` guard. -### 2.7 `search.py:193` — `undef` operator misses the category→CATEGORIES alias `[code]` +### 2.7 `search.py:193` — `undef` operator misses the category→CATEGORIES alias `[code]` ✅ FIXED The `undef` branch emits `PropFilter(property.upper())` without the alias mapping the non-undef branch applies, so `add_property_filter('category', '', operator='undef')` queries the nonexistent property `CATEGORY` — `is-not-defined` on it matches *every* object, returning events that do have categories. -### 2.8 `search.py:362`/`:506` — documented `'=='` exact-match is never enforced `[code]` +### 2.8 `search.py:362`/`:506` — documented `'=='` exact-match is never enforced `[code]` ✅ FIXED The docstring promises "`==` — exact match required, enforced client-side", but no code path inspects the `==` operator (only `'contains'` is checked at line 617) and the post-filter default block ignores it. On a fully-capable server, RFC 4791 substring `text-match` semantics leak through: `'=='` `'rain'` matches "Training". -### 2.9 `calendarobjectresource.py:1570` — `_set_data` leaves a stale `DataState` cache `[repro]` +### 2.9 `calendarobjectresource.py:1570` — `_set_data` leaves a stale `DataState` cache `[repro]` ✅ FIXED The raw-string branch clears the legacy instance attributes but never resets `self._state`. Sequence: fetch event → touch `event.id` / `is_loaded()` (caches state v1) → `event.load()` assigns `self.data = r.raw` → afterwards `get_data()` / `get_icalendar_instance()` / `id` still serve the **pre-reload content** while `.data` returns the new content. -### 2.10 `datastate.py:152` (+ `:67`, `:78`) — `BEGIN:FREEBUSY` never matches `VFREEBUSY` `[repro]` +### 2.10 `datastate.py:152` (+ `:67`, `:78`) — `BEGIN:FREEBUSY` never matches `VFREEBUSY` `[repro]` ✅ FIXED The component-type sniffing tests for `BEGIN:FREEBUSY`; real data says `BEGIN:VFREEBUSY`. A `FreeBusy` object holding raw data gets `get_component_type() → None`, so `is_loaded()`/`has_component()` are False, `save()` **silently no-ops** at the early return, and `load(only_if_unloaded=True)` reloads spuriously. -### 2.11 `calendarobjectresource.py:1943` — `_get_duration` isinstance check on the wrapper, not `.dt` `[repro]` +### 2.11 `calendarobjectresource.py:1943` — `_get_duration` isinstance check on the wrapper, not `.dt` `[repro]` ✅ FIXED `isinstance(i["DTSTART"], datetime)` tests the icalendar `vDDDTypes` wrapper (never a datetime), so the date-vs-datetime branch always takes the date path: a VTODO with a timed DTSTART and no DUE/DURATION gets duration **1 day instead of 0**. Completing a recurring task then sets the next DUE a full day late, and `Todo._next` shifts the recurrence. -### 2.12 `calendarobjectresource.py:2140` — sync safe-mode completion ignores `completion_timestamp` `[code]` +### 2.12 `calendarobjectresource.py:2140` — sync safe-mode completion ignores `completion_timestamp` `[code]` ✅ FIXED `_complete_recurring_safe` calls `completed.complete()` (defaults to *now*) while the async twin passes the caller's timestamp through. Sync/async divergence with user-visible effect on the recorded COMPLETED time. -### 2.13 `base_client.py:689` — calendar with displayname `""` dropped from results `[code]` +### 2.13 `base_client.py:689` — calendar with displayname `""` dropped from results `[code]` ✅ FIXED `if _try(calendar.get_display_name, ...)` is a truthiness check, so a calendar explicitly requested by URL whose displayname is the empty string is silently omitted. The async counterpart (`async_davclient.py:1262`) correctly uses `is not None`. -### 2.14 `async_davclient.py:957` — async `get_calendars()` lacks the GMX principal-URL fallback `[code]` +### 2.14 `async_davclient.py:957` — async `get_calendars()` lacks the GMX principal-URL fallback `[code]` ✅ FIXED Sync `get_calendars()` (`davclient.py:486–489`) falls back to the principal URL when `calendar-home-set` is missing; async returns `[]` for the same server. Parity gap. -### 2.15 `async_davclient.py:487` — issue-#158 workaround can return the probe response as the real one `[code]` +### 2.15 `async_davclient.py:487` — issue-#158 workaround can return the probe response as the real one `[code]` ✅ FIXED When the original request dies with a connection abort, the workaround sends a probe GET; if that GET is *not* 401+WWW-Authenticate (e.g. 200 with a login page), the code falls through and returns the **probe GET's response as the @@ -253,25 +253,25 @@ original request's response** — the caller sees status 200 for a PUT that never happened, and the real connection error is lost. Also: the sync client has no #158 workaround at all (parity gap in the other direction). -### 2.16 `async_davclient.py:435` — HTML-on-401 hint checks the wrong headers `[code]` +### 2.16 `async_davclient.py:435` — HTML-on-401 hint checks the wrong headers `[code]` ✅ FIXED The diagnostic checks `self.headers` (the client's own request headers) for `text/html` instead of `r.headers`, so the intended "server returned HTML, maybe set auth_type" hint can never fire. -### 2.17 `config.py:50` — `disable: true` ignored for named sections `[repro]` +### 2.17 `config.py:50` — `disable: true` ignored for named sections `[repro]` ✅ FIXED `expand_config_section` checks `config.get("section", ...)` with the string literal `"section"` instead of the variable. `disable` only works under `section='*'`; sections pulled in via a meta-section's `contains` list (or by name) connect to servers the user explicitly disabled. -### 2.18 `config.py:265` — explicit params without url/features silently discarded `[code]` +### 2.18 `config.py:265` — explicit params without url/features silently discarded `[code]` ✅ FIXED `get_connection_params` honors `explicit_params` only when `url` or `features` is present, and never merges them with the env/file source that wins: `get_davclient(password='secret')` with `CALDAV_URL`/`CALDAV_USERNAME` in env returns a config **without the password**, contradicting the docstring's "explicit parameters take highest priority". -### 2.19 `config.py:180` + `testing.py:127`/`:263` — shared module-level hint dicts get mutated `[repro]` +### 2.19 `config.py:180` + `testing.py:127`/`:263` — shared module-level hint dicts get mutated `[repro]` ✅ FIXED `resolve_features` with a string name returns the module-level `compatibility_hints` dict itself (the `base` branch deepcopies; this branch doesn't). `XandikosServer`/`RadicaleServer` then do a *shallow* `.copy()` and @@ -285,7 +285,7 @@ permanently polluted for the whole process, redirecting any later ## 3. Security -### 3.1 `discovery.py:329` — `require_tls=True` not enforced on well-known redirect target `[code]` +### 3.1 `discovery.py:329` — `require_tls=True` not enforced on well-known redirect target `[code]` ✅ FIXED `_well_known_lookup` never receives `require_tls`; a same-domain `Location: http://...` passes the `_is_subdomain_or_same` check and is returned as `ServiceInfo(tls=False)`, which `discover_service` returns unchecked. A @@ -294,7 +294,7 @@ guarantee to plaintext, and credentials follow. (Otherwise the discovery module's security posture is good: require_tls defaults True, same-domain redirect validation, single manual redirect hop.) -### 3.2 `response.py:277` — XML parser for untrusted server data lacks entity hardening `[code]` +### 3.2 `response.py:277` — XML parser for untrusted server data lacks entity hardening `[code]` ✅ FIXED `etree.XMLParser(remove_blank_text=True, huge_tree=self.huge_tree)` relies on libxml2 defaults for entity resolution. Current libxml2 blocks the classic XXE paths, but the library makes no guarantee across the unpinned dependency @@ -303,12 +303,14 @@ of server data in the package — one line fixes it: add `resolve_entities=False` (and consider `no_network=True`, `dtd_validation=False` explicitly). -### 3.3 `lib/error.py:51` — `PYTHON_CALDAV_COMMDUMP` persists bodies/headers in /tmp (low) `[code]` +**Fixed**: added `resolve_entities=False, no_network=True` to the `etree.XMLParser` call in `response.py:277`. `dtd_validation=False` is lxml's default so was not added explicitly. + +### 3.3 `lib/error.py:51` — `PYTHON_CALDAV_COMMDUMP` persists bodies/headers in /tmp (low) `[code]` ✅ FIXED `NamedTemporaryFile(delete=False)` dumps full request/response headers and bodies (calendar PII, custom auth headers) to files that accumulate indefinitely. Files are 0600, and the niquests-applied Authorization header is added after the dump point, so exposure is limited — but a cleanup policy -or a documented warning would be appropriate. +or a documented warning would be appropriate. **Human notes:** This is in /tmp, so we should expect some kind of cleanup on the OS level. There does exist some security notes in the CHANGELOG for the revision adding the feature, but the CHANGELOG has been pruned, so it's needed to consult git history to find it - it should definitively be lifted up to a more visible place. **Ruled out** (checked, found safe): SSRF via server-returned hrefs (`_normalize_href` reduces absolute URLs to path-only); credential leak on @@ -319,43 +321,43 @@ cross-host redirects (auth applied via auth callable, stripped by ## 4. JMAP backend -### 4.1 `jmap/convert/jscal_to_ical.py:384` — override child VEVENT gets the master's DTSTART `[repro]` +### 4.1 `jmap/convert/jscal_to_ical.py:384` — override child VEVENT gets the master's DTSTART `[repro]` ✅ FIXED `child_start = patch.get("start", start_str)` defaults to the master start. An override that doesn't move the occurrence (e.g. title-only change — the common case) renders a child VEVENT with RECURRENCE-ID at the occurrence but DTSTART at the *master's* start, relocating the occurrence. Default must be the override key (`rid_dt`). -### 4.2 `jmap/convert/jscal_to_ical.py:375` — EXDATE/RECURRENCE-ID value-type mismatch `[repro]` +### 4.2 `jmap/convert/jscal_to_ical.py:375` — EXDATE/RECURRENCE-ID value-type mismatch `[repro]` ✅ FIXED Override keys are rendered as naive floating DATE-TIMEs regardless of the event's `timeZone`/`showWithoutTime`: a TZID-anchored event gets `EXDATE:20260620T100000` (floating — per RFC 5545 it does not match the instance, so the **excluded occurrence reappears**), and an all-day event gets a DATETIME EXDATE against a `VALUE=DATE` DTSTART. -### 4.3 `jmap/convert/ical_to_jscal.py:100` (via `_utils.py:129`) — `Z`-suffix in LocalDateTime slots `[repro]` +### 4.3 `jmap/convert/ical_to_jscal.py:100` (via `_utils.py:129`) — `Z`-suffix in LocalDateTime slots `[repro]` ✅ FIXED UTC inputs produce `...Z` strings for RRULE `until` and recurrenceOverrides keys; RFC 8984 requires LocalDateTime there. Strict servers reject with `invalidArguments`; lenient ones mis-set the boundary, and a `Z`-suffixed override key can never match a LocalDateTime occurrence key. -### 4.4 `jmap/convert/*` — STATUS dropped in both directions `[code]` +### 4.4 `jmap/convert/*` — STATUS dropped in both directions `[code]` ✅ FIXED Neither converter maps `STATUS` ↔ `status` (only participationStatus/freeBusyStatus exist). `STATUS:CANCELLED` round-trips to the JSCalendar default `confirmed`; cancelled meetings come back as active. -### 4.5 `jmap/client.py:346` / `async_client.py:233` — `update_event` patch never clears removed properties `[code]` +### 4.5 `jmap/client.py:346` / `async_client.py:233` — `update_event` patch never clears removed properties `[code]` ✅ FIXED The full converted object is sent as the RFC 8620 PatchObject; the converter only includes keys conditionally, so a property deleted client-side (e.g. LOCATION, VALARM) is simply *absent* from the patch and **persists on the server**. Clearing requires explicit `null` entries. -### 4.6 `jmap/objects/calendar.py:113` — search `after`/`before` are not UTCDate `[code]` +### 4.6 `jmap/objects/calendar.py:113` — search `after`/`before` are not UTCDate `[code]` ✅ FIXED `datetime.isoformat()` is passed straight through (naive → no `Z`, aware → `+02:00` offset, plus microseconds); JMAP requires `...Z` UTCDate. Strict servers reject the query; lenient ones interpret the window inconsistently. -### 4.7 `jmap/client.py:462` / `async_client.py:347` — `newState` from `/changes` discarded `[code]` +### 4.7 `jmap/client.py:462` / `async_client.py:347` — `newState` from `/changes` discarded `[code]` ✅ FIXED `get_objects_by_sync_token` unpacks `new_state` into `_`. Callers' only option for a new baseline is a separate `get_sync_token()` call — changes landing in between are silently skipped on the next sync. @@ -373,50 +375,99 @@ producing drift bugs. modulo `await`. The build-side is already shared via `_JMAPClientBase` / `jmap/_methods`; moving the response-parsing glue into shared pure methods would shrink each sync/async method to ~3 lines. (The §1.13 and - §4.5 bugs are duplicated exactly because of this.) + §4.5 bugs are duplicated exactly because of this.) ✅ FIXED (commit ed3c47b0) 2. **`calendarobjectresource.py:1166–1205` — `_post_put` block pasted twice in sequence**; the second `elif r.status not in (204, 201)` is unreachable. Also factor the Etag/Schedule-Tag header→props snippet repeated in `load`/`_async_load` (the code itself carries a "consider - refactoring - this is repeated many places now" comment). + refactoring - this is repeated many places now" comment). ✅ FIXED — the + dead second copy in `_post_put` was removed and the Etag/Schedule-Tag + capture extracted into a shared `_update_tag_props()` helper now used by + `_post_put`, `load`, and `_async_load`. 3. **`async_davclient.py` re-implements ~200 lines of `DAVClient`** (init tail, get_calendars, rate-limit retry loop — byte-identical except `time.sleep` vs `asyncio.sleep`). The §2.14 GMX gap and §1.3 retry bug are direct drift products. Move into `BaseDAVClient` / `lib/error.py`. + ✅ FIXED — the byte-identical pure logic is now shared via + `BaseDAVClient`: `_init_rate_limit_config()` (the rate-limit init tail), + `_rate_limit_sleep_seconds()` (the retry sleep-decision — where §1.3 + lived), and `_calendar_home_url()` / `_build_calendars_from_propfind()` + (the get_calendars post-processing — where §2.14 lived). Only the + irreducible per-twin parts remain duplicated: the actual `time.sleep` vs + `await asyncio.sleep`, the awaited PROPFIND/principal I/O, and the + library-specific session/header setup in `__init__`. 4. **`search.py:869` — post-processing loads unloaded results one GET at a time**; `Calendar._multiget` can fetch them in a single REPORT. On the issue-#201 workaround path a 200-event search costs ~200 extra - round-trips. + round-trips. ✅ FIXED 5. **JMAP clients open a fresh HTTP connection per request** (async: `async with AsyncSession()` per `_request`; sync: module-level `requests.post`). `__exit__`/`__aexit__` already exist but do nothing — - hold one session in `_JMAPClientBase` and close it there. + hold one session in `_JMAPClientBase` and close it there. ✅ FIXED 6. **`Todo._async_complete_recurring_thisandfuture` copies ~60 lines of its sync twin** (the file says "TERRIBLY much code duplication here"), and the async safe-variant has drifted: it PUTs the completed copy twice. Extract a pure icalendar-mutation helper; keep 5-line sync/async - wrappers. + wrappers. ✅ FIXED — the icalendar mutation now lives once in the pure + (no-I/O) `_prepare_recurring_thisandfuture()` and + `_build_recurring_safe_completed()`; each sync/async twin is reduced to + a thin wrapper that does only the `await`-able save(s). The double-PUT of + the completed copy is gone (the copy is now completed in memory and PUT + once). New offline unit tests in `TestRecurringCompleteHelpers` cover the + mutation and the single-PUT invariant. 7. **`response.py` carries two parallel multistatus-parsing stacks** — legacy `_find_objects_and_props`/`expand_simple_props` (still load-bearing for `_multiget`, report-result building, `search_principals`) vs the newer dataclass parsers. Every parsing quirk (Confluence %2540, purelymail 404) must be maintained twice; the TODO at line 577 already - acknowledges this. + acknowledges this. ✅ FIXED (the duplicated structural parsing) — the + *named* quirks were already shared: Confluence `%2540` and absolute-URL + normalization live in `_normalize_href`, and the purelymail/stalwart 404 + response shapes in `_parse_response`, both of which both stacks call. The + remaining genuinely-duplicated piece — the propstat iteration plus the + "a 404 propstat means the property is absent" skip — is now in the single + `_collect_prop_elements()` helper, used by both `_extract_properties` + (dataclass stack) and `_find_objects_and_props` (legacy stack). As a side + effect the legacy path dropped its over-strict per-propstat asserts + (status-present / `cnt == len(propstat)` / non-404-status validation) that + the dataclass path never had, so the two stacks now treat odd-shaped + propstats identically. `TestParserStackEquivalence` guards the agreement. + What is *not* collapsed (and was not, to avoid touching the slow + integration-tested `_multiget`/report/`search_principals` paths): the two + value-conversion APIs — `_element_to_value` (pre-parsed values for the + dataclass results) vs `_expand_simple_prop` (caller-directed text + expansion). Those are two output formats, not a duplicated quirk; fully + migrating the `expand_simple_props` consumers onto the dataclass results + remains a larger follow-on. 8. **`search.py` sync/async driver loops duplicated** (~80 lines including - the Phase-1/Phase-2 exception-rethrow protocol and + the hase-1/Phase-2 exception-rethrow protocol and `_search_with_comptypes`). A small executor object with sync/async - implementations would leave one driver. + implementations would leave one driver. ✅ FIXED — the drift-prone + Phase-2 generator protocol (`gen.throw`/`gen.send`/`StopIteration`) now + lives once in the shared module-level `_advance_search_gen()`; each + driver loop is reduced to priming + a `while` that defers Phase-1 to a + `_dispatch_search_action` / `_async_dispatch_search_action` method. Only + the irreducible `await` and the per-action one-liners remain duplicated. --- ## 6. Altitude / design notes 1. **`response.py:464` — server fingerprints hardcoded in the generic - parser.** purelymail's `{https://purelymail.com}does-not-exist` tag, - Stalwart's "No resources found" string, and SOGo status notes live in the - core multistatus path instead of going through the compatibility-hints - feature matrix. Adding the next server's 404 shape means editing generic + parser.** ✅ FIXED. purelymail's `{https://purelymail.com}does-not-exist` + tag, Stalwart's "No resources found" string, and SOGo status notes lived in + the core multistatus path instead of going through the compatibility-hints + feature matrix. Adding the next server's 404 shape meant editing generic parsing — the exact inversion the hints mechanism exists to avoid. + + **Resolution:** `` and `` are optional children + of `` per RFC 4918 with server-defined content, so no per-server + config (nor content fingerprint) is warranted at all — `_parse_response` + now accepts either element generically. A genuinely novel tag still hits + `error.weirdness()`. The `check_404` debug guard was made `None`-safe (a + server may legally send these without a response-level status). Tests: + `test_parse_sync_collection_generic_responsedescription` / + `test_parse_sync_collection_generic_error` in `tests/test_protocol.py`. 2. **`vcal.fix()` is a regex-rewriting layer applied to every inbound object.** §2.1–§2.4 show the current fixups are individually broken in four different ways; the module's own TODOs flag the approach. Worth @@ -424,6 +475,8 @@ producing drift bugs. regression-testing each fixup against the exact server output it was written for. + **Human comment:** we've been discussing a bit moving this logic into the icalendar library - but it's hard to make good solutions, so we'll need to keep the stop-gap implementation as for now. + --- ## 7. Recommended priorities diff --git a/docs/source/http-libraries.rst b/docs/source/http-libraries.rst index dcc97b5f..f77ba8a7 100644 --- a/docs/source/http-libraries.rst +++ b/docs/source/http-libraries.rst @@ -11,47 +11,58 @@ There is also information in `GitHub issue #457 =2.0.0", "typing_extensions;python_version<'3.11'", "icalendar>6.0.0", @@ -87,17 +87,6 @@ test = [ #"caldav_server_tester" "deptry>=0.24.0; python_version >= '3.10'", ] -[tool.setuptools_scm] -write_to = "caldav/_version.py" - -[tool.setuptools] -py-modules = ["caldav"] -include-package-data = true - -[tool.setuptools.packages.find] -exclude = ["tests"] -namespaces = false - [tool.deptry] ignore = ["DEP002"] # Test dependencies (pytest, coverage, etc.) are not imported in main code diff --git a/tests/caldav_test_servers.yaml.example b/tests/caldav_test_servers.yaml.example index cb68b379..25f8241d 100644 --- a/tests/caldav_test_servers.yaml.example +++ b/tests/caldav_test_servers.yaml.example @@ -30,14 +30,14 @@ test-servers: # Docker servers (require docker-compose, see docker-test-servers/) # ========================================================================= # - # Set enabled to: - # - true: always enable - # - false: always disable - # - "auto": enable if docker is available (default for docker servers) + # Set enabled to true or false. Docker servers are skipped automatically + # when Docker is not available, and a running container is auto-detected + # even without a config entry, so listing them here is mainly to opt in or + # out explicitly and to override credentials/ports. baikal: type: docker - enabled: ${TEST_BAIKAL:-auto} + enabled: true host: ${BAIKAL_HOST:-localhost} port: ${BAIKAL_PORT:-8800} username: ${BAIKAL_USERNAME:-testuser} @@ -47,7 +47,7 @@ test-servers: nextcloud: type: docker - enabled: ${TEST_NEXTCLOUD:-false} + enabled: false host: ${NEXTCLOUD_HOST:-localhost} port: ${NEXTCLOUD_PORT:-8801} username: ${NEXTCLOUD_USERNAME:-testuser} @@ -56,7 +56,7 @@ test-servers: cyrus: type: docker - enabled: ${TEST_CYRUS:-false} + enabled: false host: ${CYRUS_HOST:-localhost} port: ${CYRUS_PORT:-8802} username: ${CYRUS_USERNAME:-user1} @@ -76,7 +76,7 @@ test-servers: sogo: type: docker - enabled: ${TEST_SOGO:-false} + enabled: false host: ${SOGO_HOST:-localhost} port: ${SOGO_PORT:-8803} username: ${SOGO_USERNAME:-testuser} @@ -84,7 +84,7 @@ test-servers: bedework: type: docker - enabled: ${TEST_BEDEWORK:-false} + enabled: false host: ${BEDEWORK_HOST:-localhost} port: ${BEDEWORK_PORT:-8804} username: ${BEDEWORK_USERNAME:-admin} @@ -92,7 +92,7 @@ test-servers: davical: type: docker - enabled: ${TEST_DAVICAL:-false} + enabled: false host: ${DAVICAL_HOST:-localhost} port: ${DAVICAL_PORT:-8805} username: ${DAVICAL_USERNAME:-testuser} @@ -101,7 +101,7 @@ test-servers: davis: type: docker - enabled: ${TEST_DAVIS:-false} + enabled: false host: ${DAVIS_HOST:-localhost} port: ${DAVIS_PORT:-8806} username: ${DAVIS_USERNAME:-testuser} @@ -109,7 +109,7 @@ test-servers: ccs: type: docker - enabled: ${TEST_CCS:-false} + enabled: false host: ${CCS_HOST:-localhost} port: ${CCS_PORT:-8807} username: ${CCS_USERNAME:-user01} @@ -117,7 +117,7 @@ test-servers: zimbra: type: docker - enabled: ${TEST_ZIMBRA:-false} + enabled: false host: ${ZIMBRA_HOST:-zimbra-docker.zimbra.io} port: ${ZIMBRA_PORT:-8808} username: ${ZIMBRA_USERNAME:-testuser@zimbra.io} @@ -126,7 +126,7 @@ test-servers: stalwart: type: docker - enabled: ${TEST_STALWART:-false} + enabled: false host: ${STALWART_HOST:-localhost} port: ${STALWART_PORT:-8809} # v0.16+: username is a full email address; password must not be in zxcvbn common-word list. @@ -136,7 +136,7 @@ test-servers: # OX App Suite requires a locally built Docker image — run build.sh first. ox: type: docker - enabled: ${TEST_OX:-false} + enabled: false host: ${OX_HOST:-localhost} port: ${OX_PORT:-8810} username: ${OX_USERNAME:-oxadmin} diff --git a/tests/docker-test-servers/baikal/README.md b/tests/docker-test-servers/baikal/README.md index 104cbcd3..3ecb778d 100644 --- a/tests/docker-test-servers/baikal/README.md +++ b/tests/docker-test-servers/baikal/README.md @@ -62,8 +62,6 @@ baikal: enabled: false ``` -Or use the environment variable: `TEST_BAIKAL=false`. - Or simply don't install Docker - the tests will automatically skip Baikal if Docker is not available. ## GitHub Actions (CI/CD) @@ -99,7 +97,7 @@ You can add more secrets in GitHub Actions settings for credentials. The test suite will automatically detect and use Baikal if configured. Configuration is in `tests/caldav_test_servers.yaml` (copy from `tests/caldav_test_servers.yaml.example` and customize). -To enable Baikal testing, set `enabled: true` (or `enabled: auto` to auto-detect Docker availability) in the YAML config: +To enable Baikal testing, set `enabled: true` in the YAML config: ```yaml baikal: @@ -107,7 +105,9 @@ baikal: enabled: true ``` -Or use the environment variable: `TEST_BAIKAL=true`. +Docker servers are also auto-detected: a running container is picked up by the +test suite even without an explicit config entry, and is skipped automatically +when Docker is not available. ## Troubleshooting diff --git a/tests/docker-test-servers/ccs/start.sh b/tests/docker-test-servers/ccs/start.sh index 02ea1447..658c9028 100755 --- a/tests/docker-test-servers/ccs/start.sh +++ b/tests/docker-test-servers/ccs/start.sh @@ -42,7 +42,7 @@ echo " Users: user01/user01, user02/user02, admin/admin" echo "" echo "Run tests from project root:" echo " cd ../../.." -echo " TEST_CCS=true pytest" +echo " pytest" echo "" echo "To stop CCS: ./stop.sh" echo "To view logs: docker-compose logs -f ccs" diff --git a/tests/docker-test-servers/cyrus/README.md b/tests/docker-test-servers/cyrus/README.md index 9c4dfdd2..02a15df1 100644 --- a/tests/docker-test-servers/cyrus/README.md +++ b/tests/docker-test-servers/cyrus/README.md @@ -67,8 +67,6 @@ cyrus: enabled: false ``` -Or use the environment variable: `TEST_CYRUS=false`. - Or simply don't install Docker - the tests will automatically skip Cyrus if Docker is not available. ## Troubleshooting diff --git a/tests/docker-test-servers/davical/start.sh b/tests/docker-test-servers/davical/start.sh index 5b9edb54..32add709 100755 --- a/tests/docker-test-servers/davical/start.sh +++ b/tests/docker-test-servers/davical/start.sh @@ -26,7 +26,7 @@ bash "$SCRIPT_DIR/setup_davical.sh" echo "" echo "Run tests from project root:" echo " cd ../../.." -echo " TEST_DAVICAL=true pytest" +echo " pytest" echo "" echo "To stop DAViCal: ./stop.sh" echo "To view logs: docker-compose logs -f" diff --git a/tests/docker-test-servers/davis/start.sh b/tests/docker-test-servers/davis/start.sh index 3e0235b0..34d52b22 100755 --- a/tests/docker-test-servers/davis/start.sh +++ b/tests/docker-test-servers/davis/start.sh @@ -23,7 +23,7 @@ bash "$SCRIPT_DIR/setup_davis.sh" echo "" echo "Run tests from project root:" echo " cd ../../.." -echo " TEST_DAVIS=true pytest" +echo " pytest" echo "" echo "To stop Davis: ./stop.sh" echo "To view logs: docker-compose logs -f davis" diff --git a/tests/docker-test-servers/nextcloud/README.md b/tests/docker-test-servers/nextcloud/README.md index 44e696b8..b6d53f11 100644 --- a/tests/docker-test-servers/nextcloud/README.md +++ b/tests/docker-test-servers/nextcloud/README.md @@ -44,7 +44,8 @@ This will: This Nextcloud instance comes **pre-configured** with: - Admin user: `admin` / `admin` -- Test user: `testuser` / `TestPassword123!` +- Test user: `testuser` / `testpass` +- Scheduling test users: `user1` / `testpass1`, `user2` / `testpass2`, `user3` / `testpass3` - Calendar and Contacts apps enabled - CalDAV URL: `http://localhost:8801/remote.php/dav` @@ -56,7 +57,7 @@ This Nextcloud instance comes **pre-configured** with: - `NEXTCLOUD_URL`: URL of the Nextcloud server (default: `http://localhost:8801`) - `NEXTCLOUD_USERNAME`: Test user username (default: `testuser`) -- `NEXTCLOUD_PASSWORD`: Test user password (default: `TestPassword123!`) +- `NEXTCLOUD_PASSWORD`: Test user password (default: `testpass`) ## Disabling Nextcloud Tests @@ -68,8 +69,6 @@ nextcloud: enabled: false ``` -Or use the environment variable: `TEST_NEXTCLOUD=false`. - Or simply don't install Docker - the tests will automatically skip Nextcloud if Docker is not available. ## Troubleshooting diff --git a/tests/docker-test-servers/ox/README.md b/tests/docker-test-servers/ox/README.md index eedf2695..ff5a6347 100644 --- a/tests/docker-test-servers/ox/README.md +++ b/tests/docker-test-servers/ox/README.md @@ -1,6 +1,6 @@ # OX App Suite CalDAV Test Server -[OX App Suite](https://www.open-xchange.com/) is a commercial groupware platform with CalDAV/CardDAV support. +[OX App Suite](https://ox.io/) is a commercial groupware platform with CalDAV/CardDAV support. ## Prerequisites @@ -42,7 +42,7 @@ are used so the container always starts clean). ```bash cd ../../.. -TEST_OX=true pytest tests/test_caldav.py -k OX -v +pytest tests/test_caldav.py -k OX -v ``` ## Notes diff --git a/tests/docker-test-servers/ox/start.sh b/tests/docker-test-servers/ox/start.sh index 38de5c98..6c923f84 100755 --- a/tests/docker-test-servers/ox/start.sh +++ b/tests/docker-test-servers/ox/start.sh @@ -60,7 +60,7 @@ echo " User: oxadmin / oxadmin" echo "" echo "Run tests from project root:" echo " cd ../../.." -echo " TEST_OX=true pytest tests/test_caldav.py -k OX -v" +echo " pytest tests/test_caldav.py -k OX -v" echo "" echo "To stop: ./stop.sh" echo "To view logs: docker-compose logs -f ox" diff --git a/tests/docker-test-servers/sogo/README.md b/tests/docker-test-servers/sogo/README.md index a9fce51e..230157fc 100644 --- a/tests/docker-test-servers/sogo/README.md +++ b/tests/docker-test-servers/sogo/README.md @@ -66,8 +66,6 @@ sogo: enabled: false ``` -Or use the environment variable: `TEST_SOGO=false`. - Or simply don't install Docker - the tests will automatically skip SOGo if Docker is not available. ## Troubleshooting diff --git a/tests/docker-test-servers/stalwart/start.sh b/tests/docker-test-servers/stalwart/start.sh index b9e7d0fc..39bc91a8 100755 --- a/tests/docker-test-servers/stalwart/start.sh +++ b/tests/docker-test-servers/stalwart/start.sh @@ -17,7 +17,7 @@ bash "$SCRIPT_DIR/setup_stalwart.sh" echo "" echo "Run tests from project root:" echo " cd ../../.." -echo " TEST_STALWART=true pytest" +echo " pytest" echo "" echo "To stop Stalwart: ./stop.sh" echo "To view logs: docker-compose logs -f stalwart" diff --git a/tests/docker-test-servers/zimbra/README.md b/tests/docker-test-servers/zimbra/README.md index 05e34cfc..213b3d3a 100644 --- a/tests/docker-test-servers/zimbra/README.md +++ b/tests/docker-test-servers/zimbra/README.md @@ -45,7 +45,7 @@ The start script will: ```bash cd ../../.. -TEST_ZIMBRA=true pytest tests/test_caldav.py -k Zimbra -v +pytest tests/test_caldav.py -k Zimbra -v ``` ## Notes diff --git a/tests/docker-test-servers/zimbra/start.sh b/tests/docker-test-servers/zimbra/start.sh index 2e0f267c..81142b8e 100755 --- a/tests/docker-test-servers/zimbra/start.sh +++ b/tests/docker-test-servers/zimbra/start.sh @@ -74,7 +74,7 @@ echo " testuser3@$ZIMBRA_DOMAIN / testpass" echo "" echo "Run tests from project root:" echo " cd ../../.." -echo " TEST_ZIMBRA=true pytest tests/test_caldav.py -k Zimbra -v" +echo " pytest tests/test_caldav.py -k Zimbra -v" echo "" echo "To stop Zimbra: ./stop.sh" echo "To view logs: docker-compose logs -f zimbra" diff --git a/tests/fixture_helpers.py b/tests/fixture_helpers.py index b20eb45a..1d7cf675 100644 --- a/tests/fixture_helpers.py +++ b/tests/fixture_helpers.py @@ -240,3 +240,25 @@ async def cleanup_calendar_objects(calendar: Any) -> None: pass except Exception: pass + + +async def adelete_calendar_if_present(principal: Any, cal_id: str) -> None: + """Best-effort removal of a leftover test calendar from a previous run. + + A test that recreates a calendar with a fixed ``cal_id`` must first clear + any leftover, or the recreate MKCALENDAR 405s ("resource already exists"). + + Only ``NotFoundError`` (the calendar isn't there) is swallowed - everything + else propagates. A previous incarnation wrapped this in a bare + ``except Exception: pass``, which silently hid a real bug (async + ``principal.calendar()`` raising ``TypeError``), so the cleanup never ran + and calendars leaked. Keep the catch narrow so that can't recur. + """ + from caldav.lib import error + + calendar = await _maybe_await(principal.calendar(cal_id=cal_id)) + await cleanup_calendar_objects(calendar) + try: + await _maybe_await(calendar.delete()) + except error.NotFoundError: + pass diff --git a/tests/test_async_davclient.py b/tests/test_async_davclient.py index 7d327563..0a77adb3 100644 --- a/tests/test_async_davclient.py +++ b/tests/test_async_davclient.py @@ -6,6 +6,7 @@ communication. We use Mock/MagicMock to emulate server communication. """ +import inspect import os from unittest.mock import AsyncMock, MagicMock, patch @@ -1058,3 +1059,55 @@ async def test_rate_limit_max_sleep_stops_adaptive_retries(self): with patch("caldav.async_davclient.asyncio.sleep", new_callable=AsyncMock): with pytest.raises(error.RateLimitError): await client.request("/") + + +class TestAsyncPrincipalCalendar: + """``principal.calendar()`` must work with async clients. + + Regression test: ``principal.calendar(cal_id=)`` used to raise + ``TypeError: argument of type 'coroutine' is not a container or iterable`` + for async clients, because the synchronous ``calendar_home_set`` property + evaluated ``"@" in `` without awaiting the async ``get_property``. + The cleanup blocks in the integration tests wrapped the call in a bare + ``except``, so calendars leaked silently and a later MKCALENDAR 405'd. + """ + + @pytest.mark.asyncio + async def test_calendar_by_cal_id_returns_awaitable(self) -> None: + """A plain cal_id needs the home set, so async returns a coroutine.""" + from caldav.collection import Calendar, Principal + + client = AsyncDAVClient(url="https://caldav.example.com/dav/") + principal = Principal(client=client, url="https://caldav.example.com/dav/principals/user/") + + ## The calendar-home-set discovery is the only would-be round-trip; mock + ## the async get_property so the test stays offline. + with patch.object( + Principal, + "get_property", + new=AsyncMock(return_value="https://caldav.example.com/dav/calendars/user/"), + ): + result = principal.calendar(cal_id="testcal") + assert inspect.iscoroutine(result), "async calendar() must return a coroutine" + calendar = await result + + assert isinstance(calendar, Calendar) + assert str(calendar.url).endswith("/calendars/user/testcal/") + + @pytest.mark.asyncio + async def test_calendar_by_full_url_stays_synchronous(self) -> None: + """A full-URL cal_id needs no home set, so it must NOT become a coroutine. + + ``test_calendar_by_full_url`` calls this without ``await`` and reads + ``.url`` directly, so the sync short-circuit must be preserved. + """ + from caldav.collection import Calendar, Principal + + client = AsyncDAVClient(url="https://caldav.example.com/dav/") + principal = Principal(client=client, url="https://caldav.example.com/dav/principals/user/") + + calendar = principal.calendar( + cal_id="https://caldav.example.com/dav/calendars/user/testcal/" + ) + assert isinstance(calendar, Calendar) + assert str(calendar.url).endswith("/calendars/user/testcal/") diff --git a/tests/test_async_integration.py b/tests/test_async_integration.py index ce1c10aa..1e4fc257 100644 --- a/tests/test_async_integration.py +++ b/tests/test_async_integration.py @@ -28,6 +28,10 @@ from .test_caldav import evr as evr_static # recurring annual event (1997) from .test_caldav import evr2 as evr2_static # bi-weekly with exception (2024) from .test_caldav import journal as journal_static +from .test_caldav import ( + near_now_ics, # shift an ical event's DTSTART/DTEND to ~now (sliding-window servers) + next_anniversary_windows, # near-future search windows for a FREQ=YEARLY event +) from .test_caldav import todo as todo_static # avoids clash with local var in add_todo() from .test_caldav import todo2 as todo2_static # avoids clash with todo2() generator from .test_caldav import todo3 as todo3_static @@ -54,6 +58,27 @@ async def wrapper(*args, **kwargs): return wrapper +## HTTP methods that change server state; a "write-delay" server settles each of +## these asynchronously, so we sleep AFTER every such request (the write-side +## counterpart of the search-cache delay, which only delays searches). +_WRITE_HTTP_METHODS = frozenset( + {"PUT", "DELETE", "MKCALENDAR", "MKCOL", "PROPPATCH", "MOVE", "COPY", "POST"} +) + + +def _async_write_delay_decorator(f, t=10): + """Sleep after every write request, to let an asynchronous server settle.""" + + @wraps(f) + async def wrapper(url, method="GET", *args, **kwargs): + response = await f(url, method, *args, **kwargs) + if str(method).upper() in _WRITE_HTTP_METHODS: + await asyncio.sleep(t) + return response + + return wrapper + + # Dynamic test data generators - use near-future dates to avoid # min-date-time restrictions on servers like CCS. _base_date = None @@ -219,6 +244,17 @@ async def async_client(self, test_server: TestServer, monkeypatch: Any) -> Any: _async_delay_decorator(AsyncCalendar.search, t=delay), ) + ## Apply write-delay (sleep after every write) for asynchronous servers. + ## Wrapped on the client instance, so monkeypatch reverts it after the test. + write_delay_config = client.features.is_supported("write-delay", dict) + if write_delay_config.get("behaviour") == "delay": + delay = write_delay_config.get("delay", 10) + monkeypatch.setattr( + client, + "request", + _async_write_delay_decorator(client.request, t=delay), + ) + yield client await client.close() @@ -472,21 +508,25 @@ async def test_principal_make_calendar(self, async_client: Any) -> None: from caldav.aio import AsyncCalendarSet, AsyncPrincipal from caldav.lib.error import AuthorizationError, MkcalendarError, NotFoundError - from .fixture_helpers import cleanup_calendar_objects + from .fixture_helpers import adelete_calendar_if_present, cleanup_calendar_objects cal_id = "pythoncaldav-async-test" calendar = None principal = None - # Try principal-based calendar creation (most servers) + # Try principal-based calendar creation (most servers). Clear any + # leftover with this cal_id first, so we exercise real creation and + # don't accumulate calendars (some servers enforce a quota). try: principal = await AsyncPrincipal.create(async_client) + await adelete_calendar_if_present(principal, cal_id) calendar = await principal.make_calendar(name="Async Test", cal_id=cal_id) except (MkcalendarError, AuthorizationError): - # Calendar already exists from a previous run - reuse it - # (mirrors sync _fixCalendar_ pattern) + # Calendar exists and can't be (re)created (e.g. no delete support + # to clear it first) - reuse it. Note: principal.calendar() returns + # a coroutine for async clients, so it must be awaited. if principal is not None: - calendar = principal.calendar(cal_id=cal_id) + calendar = await principal.calendar(cal_id=cal_id) except NotFoundError: # Principal discovery failed pass @@ -497,17 +537,23 @@ async def test_principal_make_calendar(self, async_client: Any) -> None: try: calendar = await calendar_home.make_calendar(name="Async Test", cal_id=cal_id) except (MkcalendarError, AuthorizationError): + # client.calendar() builds a Calendar by URL with no I/O, so it + # is not a coroutine and must not be awaited. calendar = async_client.calendar(cal_id=cal_id) assert calendar is not None - assert calendar.url is not None - - # Clean up based on server capabilities - if self.is_supported("delete-calendar"): - await calendar.delete() - else: - # Can't delete the calendar, just wipe its objects - await cleanup_calendar_objects(calendar) + try: + assert calendar.url is not None + finally: + # Always clean up so calendars don't accumulate (quota safety). + try: + if self.is_supported("delete-calendar"): + await calendar.delete() + else: + # Can't delete the calendar, just wipe its objects + await cleanup_calendar_objects(calendar) + except NotFoundError: + pass @pytest.mark.asyncio async def test_search_events(self, async_calendar: Any) -> None: @@ -541,6 +587,112 @@ async def test_search_events_by_date_range(self, async_calendar: Any) -> None: assert len(events) >= 1 assert "Async Test Event" in events[0].data + @pytest.mark.asyncio + async def test_search_without_comptype_with_date_range(self, async_calendar: Any) -> None: + """Async mirror of testSearchWithoutCompTypeWithDateRange. + + Test for https://github.com/python-caldav/caldav/issues/681 + + A time-range search that does NOT specify a component type must work + even on SabreDAV-based servers (Baikal, Nextcloud, ...) which - correctly + per RFC4791 section 9.7 - reject a CALDAV:time-range placed directly under + VCALENDAR with HTTP 400. The library works around this by splitting the + search into one query per component type. + + The search is run twice: once with the server's real feature + configuration, and once with search.time-range.comp-type-optional forced + to "supported", exercising the reactive HTTP-400 fallback. + """ + self.skip_unless_support("search.time-range.event") + base = _get_base_date() + uid = f"issue681-async-{uuid.uuid4()}@example.com" + await add_event( + async_calendar, + make_event( + uid, + "issue 681 async comp-type-less time-range", + base, + base + timedelta(hours=1), + ), + ) + + start = base - timedelta(hours=1) + end = base + timedelta(days=1) + + async def _assert_event_found(): + ## must not raise (the crux of issue #681) and must find the event + objects = await async_calendar.search(start=start, end=end) + assert [o for o in objects if uid in o.data], ( + "comp-type-less time-range search did not return the event" + ) + + ## Run 1: the server's real feature configuration (proactive comp-type split) + await _assert_event_found() + + ## Determine how this server reacts to the raw comp-type-less time-range + ## query. Only SabreDAV-style servers reject it with a ReportError (HTTP + ## 400) - the case the reactive fallback (issue #681 item 4) recovers from. + ## Others return nothing or a different error (e.g. Cyrus may answer 403), + ## where forcing the feature on is an unrecoverable misconfiguration. + from caldav.lib import error + + try: + await async_calendar.search(start=start, end=end, compatibility_workarounds=False) + raw_report_error = False + except error.ReportError: + raw_report_error = True + except error.DAVError: + raw_report_error = False + + ## Run 2 (only meaningful where the raw query raises a ReportError): force + ## the feature ON and verify the reactive fallback recovers and finds the event. + if raw_report_error: + features = async_calendar.client.features + key = "search.time-range.comp-type-optional" + had_key = key in features._server_features + saved = features._server_features.get(key) + features.set_feature(key, {"support": "full"}) + try: + objects = await async_calendar.search(start=start, end=end) + assert [o for o in objects if uid in o.data], ( + "reactive fallback did not recover the comp-type-less time-range search" + ) + finally: + if had_key: + features._server_features[key] = saved + else: + features._server_features.pop(key, None) + + @pytest.mark.asyncio + async def test_search_without_comptype_with_category(self, async_calendar: Any) -> None: + """Async mirror of testSearchWithoutCompTypeWithCategory. + + Test for https://github.com/python-caldav/caldav/issues/681 + + A property filter (CATEGORIES) without a component type must work. Under + the VCALENDAR comp-filter it targets VCALENDAR's own properties (no + CATEGORIES), so servers match nothing; the library splits the search into + one query per component type (search.text.comp-type-optional unsupported). + """ + self.skip_unless_support("search.text.category") + base = _get_base_date() + category = "issue681cat" + uuid.uuid4().hex[:8] + uid = f"issue681cat-async-{uuid.uuid4()}@example.com" + data = make_event( + uid, + "issue 681 async comp-type-less category search", + base, + base + timedelta(hours=1), + ).replace("END:VEVENT", f"CATEGORIES:{category}\nEND:VEVENT") + await add_event(async_calendar, data) + + ## Only the proactive split is testable here: servers silently return + ## nothing for a prop-filter under VCALENDAR (no error to recover from). + objects = await async_calendar.search(category=category) + assert [o for o in objects if uid in o.data], ( + "comp-type-less category search did not return the event" + ) + @pytest.mark.asyncio async def test_search_todos_pending(self, async_task_list: Any) -> None: """Test searching for pending todos.""" @@ -649,8 +801,9 @@ async def test_lookup_event(self, async_calendar: Any) -> None: self.skip_unless_support("save-load.event") c = async_calendar - # create the event - e1 = await c.add_event(ev1_static) + # create the event (near-now date so it stays visible to REPORT-based + # lookups on sliding-window servers; see near_now_ics) + e1 = await c.add_event(near_now_ics(ev1_static)) assert e1.url is not None # Verify that we can look it up from calendar by url @@ -661,7 +814,10 @@ async def test_lookup_event(self, async_calendar: Any) -> None: # look up by UID e3 = await c.get_event_by_uid("20010712T182145Z-123401@example.com") assert str(e3.icalendar_component["uid"]) == "20010712T182145Z-123401@example.com" - assert e3.url == e1.url + ## get_event_by_uid may return a different (canonical) URL than the PUT + ## URL on servers that don't preserve it (e.g. OX); see save-load.stable-url + if self.is_supported("save-load.stable-url"): + assert e3.url == e1.url # load directly from URL without going through the calendar object e4 = Event(client=c.client, url=e1.url) @@ -679,23 +835,31 @@ async def test_create_overwrite_delete_event(self, async_calendar: Any) -> None: self.skip_unless_support("save-load.event") c = async_calendar + ## near-now date so the event stays visible to REPORT-based lookups on + ## sliding-window servers (e.g. OX); see near_now_ics + ev1_now = near_now_ics(ev1_static) + # attempting to update a non-existing event must raise ConsistencyError with pytest.raises(error.ConsistencyError): - await c.add_event(ev1_static, no_create=True) + await c.add_event(ev1_now, no_create=True) # no_create + no_overwrite is always an error with pytest.raises(error.ConsistencyError): - await c.add_event(ev1_static, no_create=True, no_overwrite=True) + await c.add_event(ev1_now, no_create=True, no_overwrite=True) - e1 = await c.add_event(ev1_static) + e1 = await c.add_event(ev1_now) assert e1.url is not None - # same UID again → overwrite (unless server forbids it) - if not self.is_supported("save-load.mutable"): - e2 = await c.add_event(ev1_static) + # same UID again → overwrite (unless server forbids it). Overwriting via + # a fresh PUT without an If-Match etag is gated on save-load.mutable.if-match-optional: + # OX enforces optimistic concurrency and rejects such a PUT with 409. + if self.is_supported("save-load.mutable") and self.is_supported( + "save-load.mutable.if-match-optional" + ): + await c.add_event(ev1_now) # no_create on an existing event must succeed - e2 = await c.add_event(ev1_static, no_create=True) + e2 = await c.add_event(ev1_now, no_create=True) # modify and save with no_create e2.icalendar_component["summary"] = "Bastille Day Party!" @@ -706,7 +870,7 @@ async def test_create_overwrite_delete_event(self, async_calendar: Any) -> None: # no_overwrite on an existing event must raise ConsistencyError with pytest.raises(error.ConsistencyError): - await c.add_event(ev1_static, no_overwrite=True) + await c.add_event(ev1_now, no_overwrite=True) await e1.delete() @@ -766,14 +930,18 @@ async def test_load_event(self, async_calendar: Any, async_calendar2: Any) -> No c1 = async_calendar - e1_ = await c1.add_event(ev1_static) + e1_ = await c1.add_event(near_now_ics(ev1_static)) await e1_.load() # load the object returned by add_event events = await c1.get_events() assert len(events) >= 1 e1 = events[0] await e1.load() # load a freshly fetched handle - assert e1.url == e1_.url + ## e1 came from a search and may carry a different (canonical) URL than + ## the PUT URL on servers that don't preserve it (e.g. OX); see + ## save-load.stable-url + if self.is_supported("save-load.stable-url"): + assert e1.url == e1_.url @pytest.mark.asyncio async def test_copy_event(self, async_calendar: Any, async_calendar2: Any) -> None: @@ -784,14 +952,15 @@ async def test_copy_event(self, async_calendar: Any, async_calendar2: Any) -> No c1 = async_calendar c2 = async_calendar2 - e1_ = await c1.add_event(ev1_static) + await c1.add_event(near_now_ics(ev1_static)) events = await c1.get_events() e1 = events[0] # duplicate in same calendar with a new UID - e1_dup = e1.copy() - await e1_dup.save() - assert len(await c1.get_events()) == 2 + if self.is_supported("save.duplicate-event"): + e1_dup = e1.copy() + await e1_dup.save() + assert len(await c1.get_events()) == 2 # copy cross-calendar keeping the same UID if self.is_supported("save.duplicate-uid.cross-calendar"): @@ -808,8 +977,12 @@ async def test_copy_event(self, async_calendar: Any, async_calendar2: Any) -> No # copy in same calendar keeping UID — same-UID PUT is a no-op / overwrite e1_dup2 = e1.copy(keep_uid=True) await e1_dup2.save() - # count should still be 2 (not 3) because same UID overwrites - assert len(await c1.get_events()) == 2 + # same UID overwrites, so the count is unchanged: 2 where a new-UID + # duplicate was created above, 1 where duplicates are not allowed + if self.is_supported("save.duplicate-event"): + assert len(await c1.get_events()) == 2 + else: + assert len(await c1.get_events()) == 1 @pytest.mark.asyncio async def test_multi_get(self, async_calendar: Any) -> None: @@ -861,7 +1034,7 @@ async def test_object_by_sync_token(self, async_calendar: Any) -> None: objcnt += len(await c.get_todos()) objcnt += len(await c.get_events()) - obj = await c.add_event(ev1_static) + obj = await c.add_event(near_now_ics(ev1_static)) objcnt += 1 if self.is_supported("save-load.event.recurrences"): await c.add_event(evr_static) @@ -921,7 +1094,7 @@ async def test_object_by_sync_token(self, async_calendar: Any) -> None: if is_time_based: await asyncio.sleep(1) - obj3 = await c.add_event(ev3_static) + await c.add_event(near_now_ics(ev3_static)) if is_time_based: await asyncio.sleep(1) my_changed_objects = await c.get_objects_by_sync_token( @@ -983,7 +1156,7 @@ async def test_sync(self, async_calendar: Any) -> None: objcnt += len(await c.get_todos()) objcnt += len(await c.get_events()) - obj = await c.add_event(ev1_static) + obj = await c.add_event(near_now_ics(ev1_static)) objcnt += 1 if self.is_supported("save-load.event.recurrences"): await c.add_event(evr_static) @@ -1001,6 +1174,25 @@ async def test_sync(self, async_calendar: Any) -> None: assert my_objects.sync_token != "" assert len(list(my_objects)) == objcnt + stable_url = self.is_supported("save-load.stable-url") + + def synced_match(o): + """Return the synced object corresponding to o, or None. + + objects_by_url() is keyed by the server-reported URL, which on + servers that don't preserve the PUT URL (e.g. OX; see + save-load.stable-url) differs from o.url - so fall back to matching + by UID there. + """ + synced = my_objects.objects_by_url() + if stable_url: + return synced.get(o.url) + uid = o.icalendar_component["uid"] + return next( + (cand for cand in synced.values() if cand.icalendar_component["uid"] == uid), + None, + ) + if is_time_based: await asyncio.sleep(1) @@ -1024,12 +1216,12 @@ async def test_sync(self, async_calendar: Any) -> None: if not is_fragile: assert len(list(updated)) == 1 assert len(list(deleted)) == 0 - assert "foobar" in my_objects.objects_by_url()[obj.url].data + assert "foobar" in synced_match(obj).data if is_time_based: await asyncio.sleep(1) - obj3 = await c.add_event(ev3_static) + obj3 = await c.add_event(near_now_ics(ev3_static)) if is_time_based: await asyncio.sleep(1) @@ -1038,7 +1230,7 @@ async def test_sync(self, async_calendar: Any) -> None: if not is_fragile: assert len(list(updated)) == 1 assert len(list(deleted)) == 0 - assert obj3.url in my_objects.objects_by_url() + assert synced_match(obj3) is not None self.skip_unless_support("sync-token.delete") @@ -1052,7 +1244,7 @@ async def test_sync(self, async_calendar: Any) -> None: if not is_fragile: assert len(list(updated)) == 0 assert len(list(deleted)) == 1 - assert obj.url not in my_objects.objects_by_url() + assert synced_match(obj) is None if is_time_based: await asyncio.sleep(1) @@ -1304,30 +1496,35 @@ async def test_recurring_date_search(self, async_calendar: Any) -> None: self.skip_unless_support("search.recurrences.includes-implicit.event") c = async_calendar + # evr is a yearly event starting at 1997-11-02. Search the next future + # Nov-2 anniversary rather than a fixed historic year, so sliding-window + # servers (e.g. OX) can serve the time range. + year, narrow_start, narrow_end, wide_end = next_anniversary_windows() + await c.add_event(evr_static) r = await c.search( event=True, - start=datetime(2008, 11, 1, 17, 0, 0), - end=datetime(2008, 11, 3, 17, 0, 0), + start=narrow_start, + end=narrow_end, expand=False, ) assert len(r) == 1 r = await c.search( event=True, - start=datetime(2008, 11, 1, 17, 0, 0), - end=datetime(2008, 11, 3, 17, 0, 0), + start=narrow_start, + end=narrow_end, expand=True, ) assert len(r) == 1 assert r[0].data.count("END:VEVENT") == 1 - assert r[0].data.count("DTSTART;VALUE=DATE:2008") == 1 + assert r[0].data.count(f"DTSTART;VALUE=DATE:{year}") == 1 r2 = await c.search( event=True, - start=datetime(2008, 11, 1, 17, 0, 0), - end=datetime(2009, 11, 3, 17, 0, 0), + start=narrow_start, + end=wide_end, expand=True, ) assert len(r2) == 2 @@ -1591,8 +1788,8 @@ async def test_todo_datesearch(self, async_task_list: Any) -> None: foo = 5 if not self.is_supported("search.recurrences.includes-implicit.todo"): foo -= 1 - if self.check_compatibility_flag( - "vtodo_datesearch_nodtstart_task_is_skipped" + if not self.is_supported( + "search.time-range.todo.no-dtstart" ) or self.check_compatibility_flag( "vtodo_datesearch_nodtstart_task_is_skipped_in_closed_date_range" ): @@ -1664,7 +1861,9 @@ async def test_propfind(self, async_client: Any) -> None: """Raw XML propfind returns a multistatus response.""" from caldav.lib.python_utilities import to_local - self._skip_on_compatibility_flag("propfind_allprop_failure") + ## This only asserts a multistatus is returned, so (unlike the sync + ## testPropfind, which checks for DAV:resourcetype) it needs no + ## propfind.allprop.resourcetype gate. principal = await async_client.principal() foo = await async_client.propfind( principal.url, @@ -1740,7 +1939,7 @@ async def test_create_delete_calendar(self, async_client: Any) -> None: from caldav.aio import AsyncPrincipal from caldav.lib.error import AuthorizationError, NotFoundError - from .fixture_helpers import cleanup_calendar_objects + from .fixture_helpers import adelete_calendar_if_present principal = None try: @@ -1749,18 +1948,15 @@ async def test_create_delete_calendar(self, async_client: Any) -> None: pytest.skip("Cannot discover principal") cal_id = "pythoncaldav-async-createdelete-test" - try: - existing = principal.calendar(cal_id=cal_id) - await cleanup_calendar_objects(existing) - await existing.delete() - except Exception: - pass + await adelete_calendar_if_present(principal, cal_id) c = await principal.make_calendar(name="Yep", cal_id=cal_id) - assert c.url is not None - events = await c.get_events() - assert len(events) == 0 - await c.delete() + try: + assert c.url is not None + events = await c.get_events() + assert len(events) == 0 + finally: + await c.delete() @pytest.mark.asyncio async def test_calendar_by_full_url(self, async_calendar: Any, async_client: Any) -> None: @@ -1793,9 +1989,12 @@ async def test_set_calendar_properties(self, async_client: Any) -> None: from caldav.elements import dav from caldav.lib.error import AuthorizationError, NotFoundError - from .fixture_helpers import cleanup_calendar_objects + from .fixture_helpers import adelete_calendar_if_present self.skip_unless_support("create-calendar.set-displayname") + ## This test expects the display name to round-trip at a stable URL; + ## servers that relocate the calendar when a name is set (Zimbra) can't. + self.skip_unless_support("create-calendar.stable-url") self.skip_unless_support("delete-calendar") self.skip_unless_support("create-calendar") @@ -1806,26 +2005,23 @@ async def test_set_calendar_properties(self, async_client: Any) -> None: pytest.skip("Cannot discover principal") cal_id = "pythoncaldav-async-props-test" - try: - existing = principal.calendar(cal_id=cal_id) - await cleanup_calendar_objects(existing) - await existing.delete() - except Exception: - pass - - c = await principal.make_calendar(name="Yep", cal_id=cal_id) + await adelete_calendar_if_present(principal, cal_id) + + ## Use a distinct display name (not the sync fixture's "Yep") so that an + ## interrupted run of this test can never leave behind a second calendar + ## named "Yep" that would make the sync suite's principal.calendar(name="Yep") + ## lookup ambiguous. This test only checks that the display name round-trips, + ## so the actual name is irrelevant. + c = await principal.make_calendar(name="AsyncYep", cal_id=cal_id) try: props = await c.get_properties([dav.DisplayName()]) - assert "Yep" == props[dav.DisplayName.tag] + assert "AsyncYep" == props[dav.DisplayName.tag] - await c.set_properties([dav.DisplayName("hooray")]) + await c.set_properties([dav.DisplayName("hooray-async")]) props = await c.get_properties([dav.DisplayName()]) - assert props[dav.DisplayName.tag] == "hooray" + assert props[dav.DisplayName.tag] == "hooray-async" finally: - try: - await c.delete() - except Exception: - pass + await c.delete() # ==================== Group F – Regressions ==================== @@ -1916,6 +2112,9 @@ async def test_change_attendee_status_with_email_given( ) -> None: """change_attendee_status(attendee=email) updates PARTSTAT correctly.""" self.skip_unless_support("save-load.event") + ## Some servers (e.g. OX) forbid changing an attendee's PARTSTAT via a + ## direct PUT (403 Forbidden) and require iTIP scheduling instead. + self.skip_unless_support("save-load.mutable.attendee-partstat") c = async_calendar event = await c.add_event( uid="test1", @@ -1963,55 +2162,69 @@ async def test_edit_single_recurrence(self, async_calendar: Any) -> None: self.skip_unless_support("search.text") cal = async_calendar + ## Anchor the daily recurring event a few days in the future so servers + ## with a sliding REPORT window / no old-date support (e.g. CCS, ref + ## search.time-range.event.old-dates) can still serve the time ranges. + ## The integer passed to search()/summary_on() is a day offset from this + ## anchor day; the values just need to be distinct future days. + base = (datetime.now() + timedelta(days=2)).replace( + hour=8, minute=7, second=6, microsecond=0 + ) + await cal.add_event( uid="test1", summary="daily test", - dtstart=datetime(2015, 1, 1, 8, 7, 6), - dtend=datetime(2015, 1, 1, 9, 7, 6), + dtstart=base, + dtend=base + timedelta(hours=1), rrule={"FREQ": "DAILY"}, ) - async def search(month): + def day_start(offset): + return (base + timedelta(days=offset)).replace( + hour=0, minute=0, second=0, microsecond=0 + ) + + async def search(offset): recurrence = await cal.search( event=True, - start=datetime(2015, month, 1), - end=datetime(2015, month, 2), + start=day_start(offset), + end=day_start(offset) + timedelta(days=1), expand=True, ) assert len(recurrence) == 1 return recurrence[0] - async def summary_by_month(month): - return (await search(month)).icalendar_component["summary"] + async def summary_on(offset): + return (await search(offset)).icalendar_component["summary"] recurrence = await search(7) recurrence.icalendar_component["summary"] = "half a year of daily testing" await recurrence.save() - assert await summary_by_month(6) == "daily test" - assert await summary_by_month(7) == "half a year of daily testing" - assert await summary_by_month(8) == "daily test" + assert await summary_on(6) == "daily test" + assert await summary_on(7) == "half a year of daily testing" + assert await summary_on(8) == "daily test" recurrence = await search(2) recurrence.icalendar_component["summary"] = "one month of daily testing" await recurrence.save() - assert await summary_by_month(1) == "daily test" - assert await summary_by_month(2) == "one month of daily testing" - assert await summary_by_month(7) == "half a year of daily testing" + assert await summary_on(1) == "daily test" + assert await summary_on(2) == "one month of daily testing" + assert await summary_on(7) == "half a year of daily testing" recurrence = await search(7) recurrence.icalendar_component["summary"] = "six months of daily testing" await recurrence.save() - assert await summary_by_month(7) == "six months of daily testing" + assert await summary_on(7) == "six months of daily testing" recurrence = await search(9) recurrence.icalendar_component["summary"] = "daily testing" await recurrence.save(all_recurrences=True) - assert await summary_by_month(1) == "daily testing" - assert await summary_by_month(2) == "one month of daily testing" - assert await summary_by_month(3) == "daily testing" - assert await summary_by_month(7) == "six months of daily testing" + assert await summary_on(1) == "daily testing" + assert await summary_on(2) == "one month of daily testing" + assert await summary_on(3) == "daily testing" + assert await summary_on(7) == "six months of daily testing" # ==================== Group G – Auth errors & misc ==================== @@ -2123,7 +2336,7 @@ async def test_utf8_event(self, async_client: Any) -> None: from caldav.aio import AsyncPrincipal from caldav.lib.error import AuthorizationError, NotFoundError - from .fixture_helpers import cleanup_calendar_objects + from .fixture_helpers import adelete_calendar_if_present principal = None try: @@ -2132,24 +2345,18 @@ async def test_utf8_event(self, async_client: Any) -> None: pytest.skip("Cannot discover principal") cal_id = "pythoncaldav-async-utf8-test" - try: - existing = await principal.calendar(cal_id=cal_id) - await cleanup_calendar_objects(existing) - await existing.delete() - except Exception: - pass + await adelete_calendar_if_present(principal, cal_id) c = await principal.make_calendar(name="Yølp", cal_id=cal_id) try: - await c.add_event(ev1_static.replace("Bastille Day Party", "Bringebærsyltetøyfestival")) + await c.add_event( + near_now_ics(ev1_static).replace("Bastille Day Party", "Bringebærsyltetøyfestival") + ) events = await c.get_events() if "zimbra" not in str(c.url): assert len(events) == 1 finally: - try: - await c.delete() - except Exception: - pass + await c.delete() @pytest.mark.asyncio async def test_create_calendar_and_event_from_vobject(self, async_calendar: Any) -> None: @@ -2158,7 +2365,7 @@ async def test_create_calendar_and_event_from_vobject(self, async_calendar: Any) self.skip_unless_support("save-load.event") c = async_calendar cnt = len(await c.get_events()) - ve1 = vobject.readOne(ev1_static) + ve1 = vobject.readOne(near_now_ics(ev1_static)) await c.add_event(ve1) cnt += 1 events = await c.get_events() @@ -2366,8 +2573,13 @@ async def test_invite_and_respond(self, scheduling_setup: Any) -> None: new_attendee_inbox_items: list[Any] = [] auto_scheduled = False for _ in range(30): + ## Correlate by UID: a late METHOD:CANCEL from another scheduling + ## test's teardown can otherwise land here as a stray "new" item + ## (see testAcceptInviteUsernameEmailFallback). new_attendee_inbox_items = [ - item for item in await inbox1.get_items() if item.url not in inbox_urls_before + item + for item in await inbox1.get_items() + if item.url not in inbox_urls_before and item.id == event_uid ] ## Check whether the server auto-scheduled the event directly into ## the attendee's calendar. The event may land in any calendar, @@ -2441,6 +2653,16 @@ async def test_freebusy(self, scheduling_setup: Any) -> None: ## Just verify it completes without raising; response format varies per server. await coro + ## §1.5 regression: a Principal object (not a pre-resolved vCalAddress) + ## must also work as an attendee. Both the sync and async freebusy + ## tests above only ever passed a resolved address, so the + ## _async_freebusy_request branch that awaits Principal.get_vcal_address() + ## went uncovered — before the fix add_attendee() received an un-awaited + ## coroutine and crashed on attendee_obj.params[...]. + coro = principals[0].freebusy_request(dtstart, dtend, [principals[0]]) + assert asyncio.iscoroutine(coro) + await coro + # ------------------------------------------------------------------ # # Schedule-Tag tests (RFC 6638 section 3.2–3.3) # # These are async counterparts of the sync tests in # diff --git a/tests/test_caldav.py b/tests/test_caldav.py index 2df72fbf..36c5e08e 100644 --- a/tests/test_caldav.py +++ b/tests/test_caldav.py @@ -12,6 +12,7 @@ import logging import os import random +import re import sys import tempfile import time @@ -138,6 +139,44 @@ def _make_client( END:VEVENT """ + +def near_now_ics(ics, days=30, hours=11): + """Return a copy of an ical event string with DTSTART/DTEND shifted to ~now. + + Some servers (e.g. OX App Suite) only return objects within a sliding ~±1 + year window from REPORT-based lookups (ref search.unlimited-time-range), so + an event with a static historic date (the year-2006 dates in ev1/broken_ev1) + is invisible to get_events()/get_event_by_uid() even though it was stored + correctly. Using a near-now date keeps these save-then-search tests + meaningful on such servers. Only plain "DTSTART:"/"DTEND:" properties are + shifted; recurring all-day templates (DTSTART;VALUE=DATE:) are left alone. + """ + start = datetime.now() + timedelta(days=days) + end = start + timedelta(hours=hours) + ics = re.sub(r"DTSTART:[0-9T]+Z?", start.strftime("DTSTART:%Y%m%dT%H%M%SZ"), ics) + ics = re.sub(r"DTEND:[0-9T]+Z?", end.strftime("DTEND:%Y%m%dT%H%M%SZ"), ics) + return ics + + +def next_anniversary_windows(month=11, day=2, hour=17): + """Search windows around the next future anniversary of (month, day). + + A FREQ=YEARLY event (e.g. evr, anchored at 1997-11-02) recurs forever, so + historic search windows like 2008-11 are arbitrary. Servers with a sliding + REPORT window (e.g. OX App Suite, ref search.time-range.event.old-dates) can + only serve time ranges near now, so we search the next future occurrence + instead. Returns (year, narrow_start, narrow_end, wide_end): a ±1-day window + catching one occurrence in `year`, plus a wide_end one year later so that + [narrow_start, wide_end] catches two consecutive occurrences. + """ + now = datetime.now() + year = now.year if (now.month, now.day) <= (month, day) else now.year + 1 + narrow_start = datetime(year, month, day - 1, hour, 0, 0) + narrow_end = datetime(year, month, day + 1, hour, 0, 0) + wide_end = datetime(year + 1, month, day + 1, hour, 0, 0) + return year, narrow_start, narrow_end, wide_end + + ev2 = """BEGIN:VCALENDAR VERSION:2.0 PRODID:-//Example Corp.//CalDAV Client//EN @@ -796,10 +835,13 @@ def testInviteAndRespond(self): new_attendee_inbox_items = [] auto_scheduled = False for _ in range(30): + ## Correlate by UID: a late METHOD:CANCEL from another scheduling + ## test's teardown can otherwise land here as a stray "new" item + ## (see testAcceptInviteUsernameEmailFallback). new_attendee_inbox_items = [ item for item in self.principals[1].schedule_inbox().get_items() - if item.url not in inbox_items + if item.url not in inbox_items and item.id == event_uid ] ## Check whether the server auto-scheduled the event directly into ## the attendee's calendar (server-side automatic scheduling). @@ -904,12 +946,19 @@ def testAcceptInviteUsernameEmailFallback(self): ) self._auto_scheduled_event_uids.append(saved_event.id) + ## Correlate the inbox item to THIS invite by UID. Picking the first + ## arbitrary "new" item is flaky: deleting an organizer event (e.g. a + ## previous scheduling test's teardown) makes Zimbra deliver a late + ## METHOD:CANCEL for the old UID, which can land in this test's poll + ## window before our own REQUEST arrives. We match on UID (not method) + ## so that a wrongly-delivered non-REQUEST for our own UID still fails + ## the is_invite_request() assertion below rather than being hidden. new_attendee_inbox_items = [] for _ in range(30): new_attendee_inbox_items = [ item for item in self.principals[1].schedule_inbox().get_items() - if item.url not in inbox_items + if item.url not in inbox_items and item.id == saved_event.id ] if new_attendee_inbox_items: break @@ -1258,6 +1307,25 @@ def foo(*a, **kwa): return foo +## HTTP methods that change server state. A "write-delay" server settles each of +## these asynchronously, so we sleep AFTER every such request to let the change +## become visible before the test reads it back (the general, write-side +## counterpart of the search-cache delay, which only delays searches). +_WRITE_HTTP_METHODS = frozenset( + {"PUT", "DELETE", "MKCALENDAR", "MKCOL", "PROPPATCH", "MOVE", "COPY", "POST"} +) + + +def _write_delay_decorator(f, t=10): + def foo(url, method="GET", *a, **kwa): + response = f(url, method, *a, **kwa) + if str(method).upper() in _WRITE_HTTP_METHODS: + time.sleep(t) + return response + + return foo + + class RepeatedFunctionalTestsBaseClass: """This is a class with functional tests (tests that goes through basic functionality and actively communicates with third parties) @@ -1333,6 +1401,12 @@ def setup_method(self): if foo.get("behaviour") == "delay": Calendar._search = Calendar.search Calendar.search = _delay_decorator(Calendar.search, t=foo["delay"]) + foo = self.is_supported("write-delay", dict) + if foo.get("behaviour") == "delay": + ## Every write goes through the client request(); sleep after the + ## write verbs so the asynchronous change has settled before read-back. + ## Instance-level wrap (like rate-limit), torn down with the client. + self.caldav.request = _write_delay_decorator(self.caldav.request, t=foo["delay"]) if False and self.check_compatibility_flag("no-current-user-principal"): self.principal = Principal(client=self.caldav, url=self.server_params["principal_url"]) @@ -1458,10 +1532,30 @@ def _fixCalendar_(self, **kwargs): return self._default_calendar # Pre-processing: set up defaults for name and cal_id + comp_set = kwargs.get("supported_calendar_component_set", []) + # A component-restricted fixture (VTODO-only / VJOURNAL-only) is always + # looked up by cal_id, never by display name. + restricted = bool(comp_set) and "VEVENT" not in comp_set if "name" not in kwargs: if self.cleanup_regime in ("light", "pre"): self._teardownCalendar(cal_id=self.testcal_id) - if not self.is_supported("create-calendar.set-displayname"): + # Only give a display name when the server accepts one and keeps the + # calendar at the requested cal_id URL. On servers that assign a + # different canonical URL when a name is set (create-calendar.stable-url + # unsupported: Zimbra relocates, OX uses an opaque id) the library + # re-points to the canonical URL, so a named fixture would still work - + # but we keep the fixture nameless there so the bulk of the suite keeps + # addressing the fixture by its cal_id (simpler, and avoids the + # name-ambiguity issues below). Component-restricted fixtures also stay + # nameless: they are only ever found by cal_id, and giving them the same + # "Yep" name as the primary fixture would make principal.calendar( + # name="Yep") ambiguous and, on servers enforcing per-principal unique + # calendar names (SOGo), block the primary calendar from being (re)named. + if ( + restricted + or not self.is_supported("create-calendar.set-displayname") + or not self.is_supported("create-calendar.stable-url") + ): kwargs["name"] = None else: kwargs["name"] = "Yep" @@ -1470,10 +1564,9 @@ def _fixCalendar_(self, **kwargs): # that a VTODO-only calendar and a VJOURNAL-only calendar don't share the # same slot and cause MKCALENDAR failures (and wrong-type PUT errors) when # the calendar persists across tests under wipe-calendar cleanup regime. - comp_set = kwargs.get("supported_calendar_component_set", []) if comp_set and "VJOURNAL" in comp_set and "VEVENT" not in comp_set: kwargs["cal_id"] = self.testcal_id + "-journals" - elif comp_set and "VEVENT" not in comp_set: + elif restricted: kwargs["cal_id"] = self.testcal_id + "-tasks" else: kwargs["cal_id"] = self.testcal_id @@ -1543,37 +1636,11 @@ def testCheckCompatibility(self, request) -> None: fo = checker.features_checked fe = self.caldav.features - ## dotted list expected and observed - ## Snapshot checked features before compact=True calls collapse(), which - ## mutates _server_features by removing subfeatures that collapse into - ## their parent — making tested features look like untested ones. - checked_features = set(fo._server_features.keys()) - observed = fo.dotted_feature_set_list(compact=True) - expected = fe.dotted_feature_set_list(compact=True) - - for feature in set(observed.keys()).union(set(expected.keys())): - observation = fo.is_supported(feature, str) - expectation = fe.is_supported(feature, str) - if "fragile" in (observation, expectation): - continue - if "unknown" in (observation, expectation): - continue - ## Skip features the checker never explicitly tested - - ## the observation would just be a default, not a real result - if feature not in observed and feature not in checked_features: - continue - type_ = fo.find_feature(feature).get("type", "server-feature") - if type_ in ( - "client-feature", - "server-observation", - "tests-behaviour", - "client-hints", - "server-peculiarity", - ): - continue - assert expectation == observation, ( - f"expectation is {expectation}, observation is {observation} for {feature}" - ) + mismatches = fe.compare(fo) + assert not mismatches, "compatibility mismatches:\n" + "\n".join( + f" {m['feature']}: declared {m['expected']!r}, observed {m['observed']!r}" + for m in mismatches + ) def testSupport(self): """ @@ -1777,9 +1844,9 @@ def testPropfind(self): this is implicitly run by the setup) """ # ResourceType MUST be defined, and SHOULD be returned on a propfind - # for "allprop" if I have the permission to see it. - # So, no ResourceType returned seems like a bug in bedework - self.skip_on_compatibility_flag("propfind_allprop_failure") + # for "allprop" if I have the permission to see it (RFC4918 section 9.1). + # A few servers (bedework, CCS) omit it. + self.skip_unless_support("propfind.allprop.resourcetype") # first a raw xml propfind to the root URL foo = self.caldav.propfind( @@ -1792,12 +1859,15 @@ def testPropfind(self): assert "resourcetype" in to_local(foo.raw) # next, the internal _query_properties, returning an xml tree ... + # (DAV:status is a response-only element, not a queryable property - + # asking for it makes some servers, e.g. CCS, answer 400 - so we query a + # real live property instead and assert on this response, not the first.) foo2 = self.principal._query_properties( [ - dav.Status(), + dav.ResourceType(), ] ) - assert "resourcetype" in to_local(foo.raw) + assert "resourcetype" in to_local(foo2.raw) # TODO: more advanced asserts def testGetCalendarHomeSet(self): @@ -1848,10 +1918,12 @@ def testGetCalendar(self): assert str(c.url) in repr(c) def _notFound(self): - if self.check_compatibility_flag("non_existing_raises_other"): - return error.DAVError - else: + if self.is_supported("non-existing-raises-not-found"): return error.NotFoundError + else: + ## Some servers answer 403 instead of 404 (e.g. Robur); accept any + ## DAVError in that case. + return error.DAVError def testPrincipal(self): collections = self.principal.get_calendars() @@ -1890,13 +1962,22 @@ def testCreateDeleteCalendar(self): assert len(events) == 0 c.delete() - if self.is_supported("create-calendar.auto"): + if not self.is_supported("create-calendar.auto"): + # Two separate pytest.raises blocks: with both probes in a single + # block, the first one to raise would short-circuit the second, + # leaving it untested (and on auto-create servers get_events() + # doesn't raise at all, so the block passed only because + # get_display_name() happened to 404). with pytest.raises(self._notFound()): self.principal.calendar(cal_id="shouldnotexist").get_events() + with pytest.raises(self._notFound()): self.principal.calendar(cal_id="shouldnotexist").get_display_name() def testChangeAttendeeStatusWithEmailGiven(self): self.skip_unless_support("save-load.event") + ## Some servers (e.g. OX) forbid changing an attendee's PARTSTAT via a + ## direct PUT (403 Forbidden) and require iTIP scheduling instead. + self.skip_unless_support("save-load.mutable.attendee-partstat") c = self._fixCalendar() event = c.add_event( @@ -1950,8 +2031,9 @@ def cleanse(events): ## we're supposed to be working towards a brand new calendar assert len(existing_events) == 0 - # add event - c.add_event(broken_ev1) + # add event (near-now date so it stays visible to REPORT-based lookups + # on sliding-window servers; see near_now_ics) + c.add_event(near_now_ics(broken_ev1)) # c.get_events() should give a full list of events events = cleanse(c.get_events()) @@ -1963,8 +2045,14 @@ def cleanse(events): assert len(events2) == 1 assert events2[0].url == events[0].url - if self.is_supported("create-calendar") and self.is_supported( - "create-calendar.set-displayname" + if ( + self.is_supported("create-calendar") + and self.is_supported("create-calendar.set-displayname") + ## _fixCalendar only gives the calendar a display name ("Yep") when + ## the server also keeps the URL stable; on servers that assign a + ## different canonical URL when a name is set (Zimbra, OX) the fixture + ## is created nameless. + and self.is_supported("create-calendar.stable-url") ): ## We should be able to access the calender through the name c2 = self.principal.calendar(name="Yep") @@ -1973,22 +2061,81 @@ def cleanse(events): self.is_supported("delete-calendar") or self.is_supported("delete-calendar", str) == "fragile" ): - assert c2.url == c.url + ## A name lookup may return a different (canonical) calendar URL + ## than the one we created it at on servers that don't preserve + ## the URL (e.g. OX exposes the calendar under an internal + ## cal://0/NNN id); see save-load.stable-url. + if self.is_supported("save-load.stable-url"): + assert c2.url == c.url events2 = cleanse(c2.get_events()) assert len(events2) == 1 assert events2[0].url == events[0].url # add another event, it should be doable without having premade ICS + _dt = datetime.now() + timedelta(days=31) ev2 = c.add_event( - dtstart=datetime(2015, 10, 10, 8, 7, 6), + dtstart=_dt, summary="This is a test event", - dtend=datetime(2016, 10, 10, 9, 8, 7), + dtend=_dt + timedelta(hours=1), uid="ctuid1", ) events = c.get_events() assert len(events) == len(existing_events) + 2 ev2.delete() + def testNamedCalendarUrlIsUsable(self): + """A calendar created WITH a display name must be fully usable by URL. + + Exercises create-calendar.stable-url: on servers that assign a different + canonical URL when a name is set (unsupported - Zimbra relocates the + collection to a display-name-derived path, OX uses an opaque cal://0/NNN), + Calendar._create() must discover and adopt the canonical URL so that + object operations addressed via the returned calendar's .url resolve. + Regression guard for the Zimbra "event 404s on the cal_id URL even though + the collection is reachable there" quirk. On stable servers the calendar + simply stays at the requested cal_id and the test still passes. + """ + self.skip_unless_support("create-calendar") + self.skip_unless_support("create-calendar.set-displayname") + self.skip_unless_support("save-load.event") + + cal_id = self.testcal_id + "-named-url" + name = "csc-repoint-" + str(uuid.uuid4()) + self._teardownCalendar(cal_id=cal_id) + cal = self.principal.make_calendar(cal_id=cal_id, name=name) + try: + ## the display name stuck (set-displayname is supported) + assert cal.get_display_name() == name + + ## store an event and look it up BY URL through the returned calendar; + ## cal.url must point at the address that actually resolves. + uid = "csc-repoint-" + str(uuid.uuid4()) + _dt = datetime.now() + timedelta(days=20) + stored = cal.add_event( + dtstart=_dt, + dtend=_dt + timedelta(hours=1), + summary="re-point url test", + uid=uid, + ) + + ## REPORT-based lookup may lag on indexing servers (e.g. OX); retry. + fetched = None + for _ in range(10): + try: + fetched = cal.event_by_url(stored.url) + break + except error.NotFoundError: + time.sleep(1) + assert fetched is not None, "event not retrievable by URL on the created calendar" + assert fetched.url == stored.url + assert fetched.icalendar_component["uid"] == uid + finally: + try: + cal.delete() + except Exception: + pass + self._teardownCalendar(cal_id=cal_id) + @pytest.mark.parametrize("klass", ["Calendar", "Event"]) def testCreateEventFromiCal(self, klass): c = self._fixCalendar() @@ -2096,7 +2243,7 @@ def testObjectBySyncToken(self): if self.is_supported("save-load.todo.mixed-calendar"): objcnt += len(c.get_todos()) objcnt += len(c.get_events()) - obj = c.add_event(ev1) + obj = c.add_event(near_now_ics(ev1)) objcnt += 1 if self.is_supported("save-load.event.recurrences"): c.add_event(evr) @@ -2178,7 +2325,7 @@ def testObjectBySyncToken(self): ## ADDING yet another object ... and it should also be reported if is_time_based: time.sleep(1) - obj3 = c.add_event(ev3) + c.add_event(near_now_ics(ev3)) if is_time_based: time.sleep(1) my_changed_objects = c.get_objects_by_sync_token(sync_token=my_changed_objects.sync_token) @@ -2242,7 +2389,7 @@ def testSync(self): if self.is_supported("save-load.todo.mixed-calendar"): objcnt += len(c.get_todos()) objcnt += len(c.get_events()) - obj = c.add_event(ev1) + obj = c.add_event(near_now_ics(ev1)) objcnt += 1 if self.is_supported("save-load.event.recurrences"): c.add_event(evr) @@ -2261,6 +2408,25 @@ def testSync(self): assert my_objects.sync_token != "" assert len(list(my_objects)) == objcnt + stable_url = self.is_supported("save-load.stable-url") + + def synced_match(o): + """Return the synced object corresponding to o, or None. + + objects_by_url() is keyed by the server-reported URL, which on + servers that don't preserve the PUT URL (e.g. OX; see + save-load.stable-url) differs from o.url - so fall back to matching + by UID there. + """ + synced = my_objects.objects_by_url() + if stable_url: + return synced.get(o.url) + uid = o.icalendar_component["uid"] + return next( + (cand for cand in synced.values() if cand.icalendar_component["uid"] == uid), + None, + ) + if is_time_based: time.sleep(1) @@ -2287,13 +2453,13 @@ def testSync(self): if not is_fragile: assert len(list(updated)) == 1 assert len(list(deleted)) == 0 - assert "foobar" in my_objects.objects_by_url()[obj.url].data + assert "foobar" in synced_match(obj).data if is_time_based: time.sleep(1) ## ADDING yet another object ... and it should also be reported - obj3 = c.add_event(ev3) + obj3 = c.add_event(near_now_ics(ev3)) if is_time_based: time.sleep(1) @@ -2302,7 +2468,7 @@ def testSync(self): if not is_fragile: assert len(list(updated)) == 1 assert len(list(deleted)) == 0 - assert obj3.url in my_objects.objects_by_url() + assert synced_match(obj3) is not None self.skip_unless_support("sync-token.delete") @@ -2317,7 +2483,7 @@ def testSync(self): if not is_fragile: assert len(list(updated)) == 0 assert len(list(deleted)) == 1 - assert obj.url not in my_objects.objects_by_url() + assert synced_match(obj) is None if is_time_based: time.sleep(1) @@ -2337,12 +2503,16 @@ def testLoadEvent(self): c1 = self._fixCalendar(name="Yep", cal_id=self.testcal_id) c2 = self._fixCalendar(name="Yapp", cal_id=self.testcal_id2) - e1_ = c1.add_event(ev1) + e1_ = c1.add_event(near_now_ics(ev1)) if not self.check_compatibility_flag("event_by_url_is_broken"): e1_.load() e1 = c1.get_events()[0] if not self.check_compatibility_flag("event_by_url_is_broken"): - assert e1.url == e1_.url + ## e1 came from a search and may carry a different (canonical) URL + ## than the PUT URL on servers that don't preserve it (e.g. OX); see + ## save-load.stable-url. + if self.is_supported("save-load.stable-url"): + assert e1.url == e1_.url e1.load() if self.cleanup_regime == "post": self._teardownCalendar(cal_id=self.testcal_id) @@ -2361,10 +2531,10 @@ def testCopyEvent(self): assert not len(c1.get_events()) assert not len(c2.get_events()) - e1_ = c1.add_event(ev1) + e1_ = c1.add_event(near_now_ics(ev1)) e1 = c1.get_events()[0] - if not self.check_compatibility_flag("duplicates_not_allowed"): + if self.is_supported("save.duplicate-event"): ## Duplicate the event in the same calendar, with new uid e1_dup = e1.copy() e1_dup.save() @@ -2392,10 +2562,10 @@ def testCopyEvent(self): ## this makes no sense, there won't be any duplication e1_dup2 = e1.copy(keep_uid=True) e1_dup2.save() - if self.check_compatibility_flag("duplicates_not_allowed"): - assert len(c1.get_events()) == 1 - else: + if self.is_supported("save.duplicate-event"): assert len(c1.get_events()) == 2 + else: + assert len(c1.get_events()) == 1 if self.cleanup_regime == "post": self._teardownCalendar(cal_id=self.testcal_id) @@ -2408,8 +2578,9 @@ def testCreateCalendarAndEventFromVobject(self): ## in case the calendar is reused cnt = len(c.get_events()) - # add event from vobject data - ve1 = vobject.readOne(ev1) + # add event from vobject data (near-now date so it stays visible to + # REPORT-based lookups on sliding-window servers; see near_now_ics) + ve1 = vobject.readOne(near_now_ics(ev1)) c.add_event(ve1) cnt += 1 @@ -3335,8 +3506,8 @@ def testTodoDatesearch(self): ) if not self.is_supported("search.recurrences.includes-implicit.todo"): foo -= 1 ## t6 will not be returned - if self.check_compatibility_flag( - "vtodo_datesearch_nodtstart_task_is_skipped" + if not self.is_supported( + "search.time-range.todo.no-dtstart" ) or self.check_compatibility_flag( "vtodo_datesearch_nodtstart_task_is_skipped_in_closed_date_range" ): @@ -3369,10 +3540,20 @@ def testTodoDatesearch(self): todos2 = c.search(start=datetime(2025, 4, 14), todo=True, include_completed=True) todos3 = c.search(start=datetime(2025, 4, 14), todo=True) + ## On a compliant server t1/t4/t6 are returned by an open-ended future + ## search, so we get Todo objects back. Some servers legitimately return + ## nothing here: they skip no-dtstart todos (t1/t4) and don't carry the + ## recurring todo (t6) into the future - e.g. Stalwart, which skips + ## no-dtstart todos and only marks implicit-recurrence todos "fragile". + ## The presence/absence of each todo is verified by the urls_found logic + ## below; here we only type-check whatever did come back. if self.is_supported("search.time-range.open.end"): - assert isinstance(todos1[0], Todo) - assert isinstance(todos2[0], Todo) - assert isinstance(todos3[0], Todo) + if todos1: + assert isinstance(todos1[0], Todo) + if todos2: + assert isinstance(todos2[0], Todo) + if todos3: + assert isinstance(todos3[0], Todo) ## * t6 should be returned, as it's a yearly task spanning over 2025 ## * t1 should probably be returned, as it has no due date set and hence @@ -3385,8 +3566,8 @@ def testTodoDatesearch(self): urls_found = set(urls_found) if self.is_supported("search.recurrences.includes-implicit.todo", accept_fragile=True): urls_found.discard(t6.url) - if not self.check_compatibility_flag( - "vtodo_datesearch_nodtstart_task_is_skipped" + if self.is_supported( + "search.time-range.todo.no-dtstart" ) and not self.check_compatibility_flag("vtodo_datesearch_notime_task_is_skipped"): urls_found.discard(t4.url) if self.check_compatibility_flag("vtodo_no_due_infinite_duration"): @@ -3412,6 +3593,135 @@ def testSearchWithoutCompType(self): assert len(objects) == 2 assert set([type(x).__name__ for x in objects]) == {"Todo", "Event"} + def testSearchWithoutCompTypeWithDateRange(self): + """Test for https://github.com/python-caldav/caldav/issues/681 + + A time-range search that does NOT specify a component type must work + even on SabreDAV-based servers (Baikal, Nextcloud, ...) which - correctly + per RFC4791 section 9.7 - reject a CALDAV:time-range placed directly under + the VCALENDAR comp-filter with HTTP 400. The library works around this by + splitting the search into one query per component type + (search.time-range.comp-type-optional being unsupported). + + The search is run twice: once with the server's real feature + configuration, and once with search.time-range.comp-type-optional forced + to "supported". The forced run makes the library optimistically send the + comp-type-less time-range query that SabreDAV rejects, exercising the + reactive 400-fallback. Without that fallback the forced run fails on + Baikal. + """ + self.skip_unless_support("search.time-range.event") + cal = self._fixCalendar() + + ## Near-future dates, to steer clear of servers that restrict old-date + ## time-range searches. + now = datetime.now(timezone.utc) + dtstart = now + timedelta(days=1) + dtend = dtstart + timedelta(hours=1) + uid = "issue681-" + uuid.uuid4().hex + ical = ( + "BEGIN:VCALENDAR\r\n" + "VERSION:2.0\r\n" + "PRODID:-//python-caldav//issue681 test//EN\r\n" + "BEGIN:VEVENT\r\n" + f"UID:{uid}\r\n" + f"DTSTAMP:{now.strftime('%Y%m%dT%H%M%SZ')}\r\n" + f"DTSTART:{dtstart.strftime('%Y%m%dT%H%M%SZ')}\r\n" + f"DTEND:{dtend.strftime('%Y%m%dT%H%M%SZ')}\r\n" + "SUMMARY:issue 681 comp-type-less time-range search\r\n" + "END:VEVENT\r\n" + "END:VCALENDAR\r\n" + ) + cal.save_event(ical) + + start = now + end = now + timedelta(days=2) + + def _assert_event_found(): + ## must not raise (this is the crux of issue #681) and must find the event + objects = cal.search(start=start, end=end) + assert [o for o in objects if uid in o.data], ( + "comp-type-less time-range search did not return the event" + ) + + ## Run 1: the server's real feature configuration (proactive comp-type split) + _assert_event_found() + + ## Determine how this server reacts to the raw comp-type-less time-range + ## query. Only SabreDAV-style servers (Baikal, Nextcloud) reject it with a + ## ReportError (HTTP 400) - that is the case the reactive fallback (issue + ## #681 item 4) is designed to recover from. Others return nothing, or a + ## different error (e.g. Cyrus may answer 403), where forcing the feature on + ## is an unrecoverable misconfiguration not worth asserting on. + try: + cal.search(start=start, end=end, compatibility_workarounds=False) + raw_report_error = False + except error.ReportError: + raw_report_error = True + except error.DAVError: + raw_report_error = False + + ## Run 2 (only meaningful where the raw query raises a ReportError): force + ## search.time-range.comp-type-optional ON and verify the reactive fallback + ## recovers and still finds the event. + if raw_report_error: + features = self.caldav.features + key = "search.time-range.comp-type-optional" + had_key = key in features._server_features + saved = features._server_features.get(key) + features.set_feature(key, {"support": "full"}) + try: + objects = cal.search(start=start, end=end) + assert [o for o in objects if uid in o.data], ( + "reactive fallback did not recover the comp-type-less time-range search" + ) + finally: + if had_key: + features._server_features[key] = saved + else: + features._server_features.pop(key, None) + + def testSearchWithoutCompTypeWithCategory(self): + """Test for https://github.com/python-caldav/caldav/issues/681 + + A property filter (here CATEGORIES) without a component type must work. + Placed directly under the VCALENDAR comp-filter the prop-filter targets + VCALENDAR's own properties, which lack component properties like + CATEGORIES, so servers (Xandikos, SabreDAV, ...) match nothing. The + library works around this by splitting the search into one query per + component type (search.text.comp-type-optional being unsupported). + """ + self.skip_unless_support("search.text.category") + cal = self._fixCalendar() + + category = "issue681cat" + uuid.uuid4().hex[:8] + uid = "issue681cat-" + uuid.uuid4().hex + ical = ( + "BEGIN:VCALENDAR\r\n" + "VERSION:2.0\r\n" + "PRODID:-//python-caldav//issue681 test//EN\r\n" + "BEGIN:VEVENT\r\n" + f"UID:{uid}\r\n" + f"DTSTAMP:{datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')}\r\n" + f"DTSTART:{(datetime.now(timezone.utc) + timedelta(days=1)).strftime('%Y%m%dT%H%M%SZ')}\r\n" + f"DTEND:{(datetime.now(timezone.utc) + timedelta(days=1, hours=1)).strftime('%Y%m%dT%H%M%SZ')}\r\n" + "SUMMARY:issue 681 comp-type-less category search\r\n" + f"CATEGORIES:{category}\r\n" + "END:VEVENT\r\n" + "END:VCALENDAR\r\n" + ) + cal.save_event(ical) + + ## The proactive per-component-type split is the only safe fix here: unlike + ## the time-range case (where SabreDAV returns HTTP 400, which the reactive + ## fallback can catch), servers silently return nothing for a prop-filter + ## under VCALENDAR, so there is no error to recover from. Hence we only + ## verify the default (proactive) behaviour. + objects = cal.search(category=category) + assert [o for o in objects if uid in o.data], ( + "comp-type-less category search did not return the event" + ) + def testTodoCompletion(self): """ Will check that todo-items can be completed and deleted @@ -3530,7 +3840,9 @@ def testUtf8Event(self): c = self._fixCalendar(name="Yølp", cal_id=self.testcal_id) # add event - e1 = c.add_event(ev1.replace("Bastille Day Party", "Bringebærsyltetøyfestival")) + e1 = c.add_event( + near_now_ics(ev1).replace("Bastille Day Party", "Bringebærsyltetøyfestival") + ) # fetch it back events = c.get_events() @@ -3555,7 +3867,9 @@ def testUnicodeEvent(self): c = self._fixCalendar(name="Yølp", cal_id=self.testcal_id) # add event - e1 = c.add_event(to_str(ev1.replace("Bastille Day Party", "Bringebærsyltetøyfestival"))) + e1 = c.add_event( + to_str(near_now_ics(ev1).replace("Bastille Day Party", "Bringebærsyltetøyfestival")) + ) # c.get_events() should give a full list of events events = c.get_events() @@ -3566,6 +3880,9 @@ def testUnicodeEvent(self): def testSetCalendarProperties(self): self.skip_unless_support("create-calendar.set-displayname") + ## This test expects the fixture's display name ("Yep") and renames the + ## calendar in place; both require the URL to stay put when a name is set. + self.skip_unless_support("create-calendar.stable-url") self.skip_unless_support("delete-calendar") c = self._fixCalendar() @@ -3595,56 +3912,48 @@ def testSetCalendarProperties(self): if not self.is_supported("delete-calendar"): raise - c.set_properties( - [ - dav.DisplayName("hooray"), - ] - ) - props = c.get_properties( - [ - dav.DisplayName(), - ] - ) - assert props[dav.DisplayName.tag] == "hooray" - - ## calendar color and calendar order are extra properties not - ## described by RFC5545, but anyway supported by quite some - ## server implementations - if self.check_compatibility_flag("calendar_color"): - props = c.get_properties( - [ - ical.CalendarColor(), - ] - ) - assert props[ical.CalendarColor.tag] != "sort of blueish" - c.set_properties( - [ - ical.CalendarColor("blue"), - ] - ) - props = c.get_properties( - [ - ical.CalendarColor(), - ] - ) - assert props[ical.CalendarColor.tag] == "blue" - if self.check_compatibility_flag("calendar_order"): - props = c.get_properties( - [ - ical.CalendarOrder(), - ] - ) - assert props[ical.CalendarOrder.tag] != "-434" + try: c.set_properties( [ - ical.CalendarOrder("12"), + dav.DisplayName("hooray"), ] ) props = c.get_properties( [ - ical.CalendarOrder(), + dav.DisplayName(), ] ) + assert props[dav.DisplayName.tag] == "hooray" + finally: + ## Restore the fixture's canonical display name. Under the + ## wipe-calendar cleanup regime the calendar is reused (never + ## deleted) between tests, so a lingering "hooray" would make this + ## test non-idempotent (the next run reads "hooray", not "Yep") and, + ## on servers enforcing per-principal unique calendar names (SOGo), + ## block other calendars from taking the "hooray" name. + try: + c.set_properties([dav.DisplayName("Yep")]) + except error.PropsetError: + ## Best-effort cleanup only: the assertion of interest has + ## already run above. Some servers reject setting the display + ## name (PropsetError); if so there's nothing to restore and + ## nothing actionable to do here, so swallow it silently. + pass + + ## calendar color and calendar order are extra properties not + ## described by RFC5545, but anyway supported by quite some server + ## implementations. How they behave is probed in detail by the + ## server-tester (calendar-color / calendar-order); here we just + ## smoke-test that a supported property can be set. Some servers + ## normalise the colour name (e.g. "blue" -> a hex value), so for the + ## colour we only assert that *something* was stored, not the exact value. + if self.is_supported("calendar-color"): + c.set_properties([ical.CalendarColor("blue")]) + props = c.get_properties([ical.CalendarColor()]) + assert props[ical.CalendarColor.tag] + if self.is_supported("calendar-order"): + c.set_properties([ical.CalendarOrder("12")]) + props = c.get_properties([ical.CalendarOrder()]) assert props[ical.CalendarOrder.tag] == "12" def testLookupEvent(self): @@ -3656,8 +3965,9 @@ def testLookupEvent(self): c = self._fixCalendar() assert c.url is not None - # add event - e1 = c.add_event(ev1) + # add event, with a near-now date so it stays visible to REPORT-based + # lookups on sliding-window servers (see near_now_ics()). + e1 = c.add_event(near_now_ics(ev1)) assert e1.url is not None # Verify that we can look it up, both by URL and by ID @@ -3668,7 +3978,15 @@ def testLookupEvent(self): # look up by UID e3 = c.get_event_by_uid("20010712T182145Z-123401@example.com") assert e3.vobject_instance.vevent.uid == e1.vobject_instance.vevent.uid - assert e3.url == e1.url + if self.is_supported("save-load.stable-url"): + assert e3.url == e1.url + else: + ## The server reports the object under a different (canonical) URL + ## than the one we stored it at (e.g. OX App Suite). We can't compare + ## URLs, but we can confirm the looked-up URL is a real, fetchable + ## resource holding the same event. + e3.load() + assert e3.icalendar_component["uid"] == e1.icalendar_component["uid"] e4 = Event(client=self.caldav, url=e1.url) e4.load() @@ -3686,17 +4004,21 @@ def testCreateOverwriteDeleteEvent(self): c = self._fixCalendar() assert c.url is not None + ## near-now date so the event stays visible to REPORT-based lookups on + ## sliding-window servers (e.g. OX); see near_now_ics + ev1_now = near_now_ics(ev1) + # attempts on updating/overwriting a non-existing event should fail: with pytest.raises(error.ConsistencyError): - c.add_event(ev1, no_create=True) + c.add_event(ev1_now, no_create=True) # no_create and no_overwrite is mutually exclusive, this will always # raise an error (unless the ical given is blank) with pytest.raises(error.ConsistencyError): - c.add_event(ev1, no_create=True, no_overwrite=True) + c.add_event(ev1_now, no_create=True, no_overwrite=True) # add event - e1 = c.add_event(ev1) + e1 = c.add_event(ev1_now) todo_ok = self.is_supported("save-load.todo.mixed-calendar") if todo_ok: @@ -3706,19 +4028,30 @@ def testCreateOverwriteDeleteEvent(self): assert t1.url is not None if not self.check_compatibility_flag("event_by_url_is_broken"): assert c.event_by_url(e1.url).url == e1.url - assert c.get_event_by_uid(e1.id).url == e1.url + ## get_event_by_uid may return a different (canonical) URL than the PUT + ## URL on servers that don't preserve it (e.g. OX); see save-load.stable-url + e_by_uid = c.get_event_by_uid(e1.id) + if self.is_supported("save-load.stable-url"): + assert e_by_uid.url == e1.url + else: + assert e_by_uid.icalendar_component["uid"] == e1.icalendar_component["uid"] no_create = True ## add same event again. As it has same uid, it should be overwritten - ## (but some calendars may throw a "409 Conflict") - if self.is_supported("save-load.mutable"): - e2 = c.add_event(ev1) + ## (but some calendars may throw a "409 Conflict"). Overwriting via a + ## fresh PUT without an If-Match etag is gated on save-load.mutable.if-match-optional: + ## OX enforces optimistic concurrency and rejects such a PUT with 409 + ## (etag-conditional save() still works, so save-load.mutable stays full). + if self.is_supported("save-load.mutable") and self.is_supported( + "save-load.mutable.if-match-optional" + ): + e2 = c.add_event(ev1_now) if todo_ok: t2 = c.add_todo(todo) ## add same event with "no_create". Should work like a charm. - e2 = c.add_event(ev1, no_create=no_create) + e2 = c.add_event(ev1_now, no_create=no_create) if todo_ok: t2 = c.add_todo(todo, no_create=no_create) @@ -3740,7 +4073,7 @@ def testCreateOverwriteDeleteEvent(self): ## "no_overwrite" should throw a ConsistencyError. with pytest.raises(error.ConsistencyError): - c.add_event(ev1, no_overwrite=True) + c.add_event(ev1_now, no_overwrite=True) if todo_ok: with pytest.raises(error.ConsistencyError): c.add_todo(todo, no_overwrite=True) @@ -3750,15 +4083,13 @@ def testCreateOverwriteDeleteEvent(self): if todo_ok: t1.delete() - if self.check_compatibility_flag("non_existing_raises_other"): - expected_error = error.DAVError - else: - expected_error = error.NotFoundError - # Verify that we can't look it up, both by URL and by ID with pytest.raises(self._notFound()): c.event_by_url(e1.url) - if self.is_supported("save-load.mutable"): + ## e2 only exists if the put-overwrite block above ran + if self.is_supported("save-load.mutable") and self.is_supported( + "save-load.mutable.if-match-optional" + ): with pytest.raises(self._notFound()): c.event_by_url(e2.url) if not self.check_compatibility_flag("event_by_url_is_broken"): @@ -3876,20 +4207,23 @@ def testRecurringDateSearch(self): self.skip_unless_support("search.recurrences.includes-implicit.event") c = self._fixCalendar() - # evr is a yearly event starting at 1997-11-02 + # evr is a yearly event starting at 1997-11-02. We search the next + # future Nov-2 anniversary rather than a fixed historic year, so that + # sliding-window servers (e.g. OX) can serve the time range. + year, narrow_start, narrow_end, wide_end = next_anniversary_windows() e = c.add_event(evr) - ## Without "expand", we should still find it when searching over 2008 ... + ## Without "expand", we should still find it when searching the anniversary with pytest.deprecated_call(): r = c.date_search( - datetime(2008, 11, 1, 17, 00, 00), - datetime(2008, 11, 3, 17, 00, 00), + narrow_start, + narrow_end, expand=False, ) r2 = c.search( event=True, - start=datetime(2008, 11, 1, 17, 00, 00), - end=datetime(2008, 11, 3, 17, 00, 00), + start=narrow_start, + end=narrow_end, expand=False, ) assert len(r) == 1 @@ -3899,46 +4233,46 @@ def testRecurringDateSearch(self): ## legacy method name with pytest.deprecated_call(): r1 = c.date_search( - datetime(2008, 11, 1, 17, 00, 00), - datetime(2008, 11, 3, 17, 00, 00), + narrow_start, + narrow_end, expand=True, ) ## server expansion, with client side fallback r2 = c.search( event=True, - start=datetime(2008, 11, 1, 17, 00, 00), - end=datetime(2008, 11, 3, 17, 00, 00), + start=narrow_start, + end=narrow_end, expand=True, ) ## r3 was client-side expansion, but this is the default now ## server side expansion r4 = c.search( event=True, - start=datetime(2008, 11, 1, 17, 00, 00), - end=datetime(2008, 11, 3, 17, 00, 00), + start=narrow_start, + end=narrow_end, server_expand=True, ) assert len(r1) == 1 assert len(r2) == 1 assert r1[0].data.count("END:VEVENT") == 1 assert r2[0].data.count("END:VEVENT") == 1 - ## due to expandation, the DTSTART should be in 2008 - assert r1[0].data.count("DTSTART;VALUE=DATE:2008") == 1 - assert r2[0].data.count("DTSTART;VALUE=DATE:2008") == 1 + ## due to expandation, the DTSTART should be in the anniversary year + assert r1[0].data.count(f"DTSTART;VALUE=DATE:{year}") == 1 + assert r2[0].data.count(f"DTSTART;VALUE=DATE:{year}") == 1 if self.is_supported("search.recurrences.expanded.event"): - assert r4[0].data.count("DTSTART;VALUE=DATE:2008") == 1 + assert r4[0].data.count(f"DTSTART;VALUE=DATE:{year}") == 1 ## With expand=True and searching over two recurrences ... with pytest.deprecated_call(): r1 = c.date_search( - datetime(2008, 11, 1, 17, 00, 00), - datetime(2009, 11, 3, 17, 00, 00), + narrow_start, + wide_end, expand=True, ) r2 = c.search( event=True, - start=datetime(2008, 11, 1, 17, 00, 00), - end=datetime(2009, 11, 3, 17, 00, 00), + start=narrow_start, + end=wide_end, expand=True, ) @@ -4052,30 +4386,45 @@ def testEditSingleRecurrence(self): cal = self._fixCalendar() + ## Anchor the daily recurring event a few days in the future so servers + ## with a sliding REPORT window / no old-date support (e.g. CCS, ref + ## search.time-range.event.old-dates) can still serve the time ranges. + ## The integer passed to search()/summary_on() is a day offset from this + ## anchor day; the values just need to be distinct future days. + base = (datetime.now() + timedelta(days=2)).replace( + hour=8, minute=7, second=6, microsecond=0 + ) + ## Create a daily recurring event cal.add_event( uid="test1", summary="daily test", - dtstart=datetime(2015, 1, 1, 8, 7, 6), - dtend=datetime(2015, 1, 1, 9, 7, 6), + dtstart=base, + dtend=base + timedelta(hours=1), rrule={"FREQ": "DAILY"}, ) - def search(month): + def day_start(offset): + return (base + timedelta(days=offset)).replace( + hour=0, minute=0, second=0, microsecond=0 + ) + + def search(offset): """ - Internal function to find one recurrence object + Internal function to find one recurrence object - the occurrence on + the day `offset` days after the event's anchor day. """ recurrence = cal.search( event=True, - start=datetime(2015, month, 1), - end=datetime(2015, month, 2), + start=day_start(offset), + end=day_start(offset) + timedelta(days=1), expand=True, ) assert len(recurrence) == 1 return recurrence[0] - def summary_by_month(month): - return search(month).icalendar_component["summary"] + def summary_on(offset): + return search(offset).icalendar_component["summary"] ## Search for a recurrence recurrence = search(7) @@ -4085,52 +4434,56 @@ def summary_by_month(month): recurrence.save() ## Only one day should be affected - assert summary_by_month(6) == "daily test" - assert summary_by_month(7) == "half a year of daily testing" - assert summary_by_month(8) == "daily test" + assert summary_on(6) == "daily test" + assert summary_on(7) == "half a year of daily testing" + assert summary_on(8) == "daily test" ## let's try to set several recurrence exceptions recurrence = search(2) recurrence.icalendar_component["summary"] = "one month of daily testing" recurrence.save() - assert summary_by_month(1) == "daily test" - assert summary_by_month(2) == "one month of daily testing" - assert summary_by_month(7) == "half a year of daily testing" + assert summary_on(1) == "daily test" + assert summary_on(2) == "one month of daily testing" + assert summary_on(7) == "half a year of daily testing" ## Changing any of the exceptions should also work recurrence = search(7) recurrence.icalendar_component["summary"] = "six months of daily testing" recurrence.save() - assert summary_by_month(7) == "six months of daily testing" + assert summary_on(7) == "six months of daily testing" ## parameter all_recurrences should change all recurrences - - ## except February and July + ## except the two edited exceptions (offsets 2 and 7) recurrence = search(9) recurrence.icalendar_component["summary"] = "daily testing" recurrence.save(all_recurrences=True) - assert summary_by_month(1) == "daily testing" - assert summary_by_month(2) == "one month of daily testing" - assert summary_by_month(3) == "daily testing" - assert summary_by_month(7) == "six months of daily testing" - - ## Last ... let's change the dtend and dtstart of the recurrence - recurrence = search(9) - recurrence.icalendar_component.pop("dtstart") - recurrence.icalendar_component.add("dtstart", datetime(2015, 9, 1, 8, 0, 0)) - recurrence.icalendar_component.pop("dtend") - recurrence.icalendar_component.add("dtend", datetime(2015, 9, 1, 10, 0, 0)) - recurrence.save(all_recurrences=True) - - recurrence = search(8) - assert ( - recurrence.icalendar_component.start.astimezone() - == datetime(2015, 8, 1, 8, 0, 0).astimezone() - ) - assert ( - recurrence.icalendar_component.end.astimezone() - == datetime(2015, 8, 1, 10, 0, 0).astimezone() - ) + assert summary_on(1) == "daily testing" + assert summary_on(2) == "one month of daily testing" + assert summary_on(3) == "daily testing" + assert summary_on(7) == "six months of daily testing" + + ## Last ... let's change the dtend and dtstart of the recurrence. + ## This reschedules the whole series (moves the master DTSTART) while the + ## two exceptions above are still attached - some servers (e.g. OX) reject + ## that re-anchoring with a 409 Conflict, so it is gated on its own flag. + if self.is_supported("save-load.event.recurrences.exception.reschedule"): + recurrence = search(9) + recurrence.icalendar_component.pop("dtstart") + recurrence.icalendar_component.add("dtstart", day_start(9).replace(hour=8)) + recurrence.icalendar_component.pop("dtend") + recurrence.icalendar_component.add("dtend", day_start(9).replace(hour=10)) + recurrence.save(all_recurrences=True) + + recurrence = search(8) + assert ( + recurrence.icalendar_component.start.astimezone() + == day_start(8).replace(hour=8).astimezone() + ) + assert ( + recurrence.icalendar_component.end.astimezone() + == day_start(8).replace(hour=10).astimezone() + ) def testOffsetURL(self): """ diff --git a/tests/test_caldav_unit.py b/tests/test_caldav_unit.py index 8ae1c68b..880659d4 100755 --- a/tests/test_caldav_unit.py +++ b/tests/test_caldav_unit.py @@ -8,6 +8,7 @@ import pickle from datetime import date, datetime, timedelta, timezone +from typing import Any from unittest import mock from urllib.parse import urlparse @@ -656,6 +657,14 @@ def testAbsoluteURL(self): def _load(self, only_if_unloaded=True): self.data = todo6 + def _batch_load(self, objects): + ## Search results are batch-loaded via a single calendar-multiget REPORT + ## (Calendar._batch_load_objects); the mocked server returns no + ## calendar-data, so inject todo6 here the same way _load does per object. + for obj in objects: + obj.data = todo6 + + @mock.patch("caldav.collection.Calendar._batch_load_objects", new=_batch_load) @mock.patch("caldav.calendarobjectresource.CalendarObjectResource.load", new=_load) def testDateSearch(self): """ @@ -708,6 +717,8 @@ def testDateSearch(self): """ client = MockedDAVClient(xml) calendar = Calendar(client, url="/principals/calendar/home@petroski.example.com/963/") + ## expand=False does no client-side time-range filtering, so all three + ## server-returned hrefs are returned regardless of the search window. with pytest.deprecated_call(): results = calendar.date_search(datetime(2021, 2, 1), datetime(2021, 2, 7), expand=False) assert len(results) == 3 @@ -1503,6 +1514,85 @@ def testDataAPINoDataState(self): assert event._get_component_type_cheap() is None assert event._has_data() is False + def test_set_data_updates_state_cache(self) -> None: + """§2.9: _set_data (raw string branch) must reset _state so that + get_data()/get_icalendar_instance()/id return the new content. + + Bug: _set_data cleared _data/_vobject_instance/_icalendar_instance + but never updated self._state. Once _state was cached by an earlier + call to _ensure_state() (e.g. via event.id or is_loaded()), all + subsequent reads through the new API served stale content. + """ + from caldav.datastate import RawDataState + + client = DAVClient(url="http://cal.example.com/") + ev2 = """BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Example Corp.//CalDAV Client//EN +BEGIN:VEVENT +UID:updated-uid@example.com +DTSTAMP:20260101T000000Z +DTSTART:20260601T100000Z +DTEND:20260601T110000Z +SUMMARY:Updated Event +END:VEVENT +END:VCALENDAR +""" + event = Event(client, data=ev1) + + # Prime the state cache — simulates the common scenario where the + # object is accessed before a reload (e.g. event.id or is_loaded()) + assert event.id == "20010712T182145Z-123401@example.com" + assert isinstance(event._state, RawDataState) + + # Simulate what load() does: assign new raw data + event.data = ev2 + + # _state must now reflect the new data + assert isinstance(event._state, RawDataState) + assert event.get_data() == ev2, "get_data() returned stale pre-reload content" + assert event.id == "updated-uid@example.com", "id returned stale UID after reload" + assert "Updated Event" in event.get_data() + + def test_vfreebusy_component_type_detection(self) -> None: + """§2.10: RawDataState.get_component_type() tested for 'BEGIN:FREEBUSY' + but real iCalendar data uses 'BEGIN:VFREEBUSY', so FreeBusy objects + got component_type=None → is_loaded()/has_component() False → save() + silent no-op and load(only_if_unloaded=True) spuriously reloads. + Also fixes get_uid()/get_component_type() in DataState base class + which listed 'FREEBUSY' instead of 'VFREEBUSY' as comp.name. + """ + from caldav.datastate import RawDataState + + freebusy_data = """BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Test//Test//EN +BEGIN:VFREEBUSY +UID:freebusy@example.com +DTSTAMP:20240101T120000Z +DTSTART:20240601T090000Z +DTEND:20240601T110000Z +FREEBUSY:20240601T090000Z/20240601T100000Z +END:VFREEBUSY +END:VCALENDAR +""" + state = RawDataState(freebusy_data) + assert state.get_component_type() == "VFREEBUSY", ( + "RawDataState.get_component_type() returned None for VFREEBUSY data " + "(was checking for 'BEGIN:FREEBUSY' instead of 'BEGIN:VFREEBUSY')" + ) + assert state.get_uid() == "freebusy@example.com" + + # Also verify via the base class parsers (IcalendarState path) + import icalendar + + from caldav.datastate import IcalendarState + + ical = icalendar.Calendar.from_ical(freebusy_data) + istate = IcalendarState(ical) + assert istate.get_component_type() == "VFREEBUSY" + assert istate.get_uid() == "freebusy@example.com" + def testDataAPIEdgeCases(self): """Test edge cases in the data API (issue #613).""" cal_url = "http://me:hunter2@calendar.example:80/" @@ -1625,6 +1715,31 @@ def testTodoDuration(self): assert "DUE" not in my_todo4.component assert my_todo4.component["duration"].dt == timedelta(2) + def testTodoDurationTimedDtstart(self): + """§2.11: _get_duration must return timedelta(0) for a VTODO with a timed DTSTART + and no DUE/DURATION — not timedelta(days=1). + + isinstance(i["DTSTART"], datetime) tested the vDDDTypes wrapper (always False), + so the date-vs-datetime branch always took the 'is a date' path, returning 1 day. + Fix: test isinstance(i["DTSTART"].dt, datetime) instead. + """ + cal_url = "http://me:hunter2@calendar.example:80/" + client = DAVClient(url=cal_url) + todo_timed_dtstart = """BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Test//EN +BEGIN:VTODO +UID:timed-dtstart@example.com +DTSTAMP:20240101T000000Z +DTSTART:20240601T100000Z +SUMMARY:Todo with timed DTSTART only +END:VTODO +END:VCALENDAR""" + todo_item = Todo(client, data=todo_timed_dtstart) + assert todo_item.get_duration() == timedelta(0), ( + f"Expected timedelta(0) for timed DTSTART with no DUE, got {todo_item.get_duration()}" + ) + def testURL(self): """Exercising the URL class""" long_url = "http://foo:bar@www.example.com:8080/caldav.php/?foo=bar" @@ -1722,6 +1837,28 @@ def testURL(self): == URL("//www.example.com/bar/").canonical() ) + # 9b) canonical() must strip credentials (§2.5a) + cred_url = URL("https://user:pass@example.com/cal/") + canon = cred_url.canonical() + assert "user" not in str(canon), "canonical() leaked username" + assert "pass" not in str(canon), "canonical() leaked password" + + # 9c) canonical() must not mutate self (§2.5b): + # a URL with no auth — unauth() returns self, canonical() must + # still return a fresh object and leave self unchanged. + plain_url = URL("http://example.com/cal/path/") + before = str(plain_url) + _ = plain_url.canonical() + assert str(plain_url) == before, "canonical() mutated self" + + # 9d) __eq__ calls canonical() — must not mutate self (§2.5b) + # A literal '+' in the path would become '%2B' after quote(unquote()), + # silently changing what resource subsequent requests target. + plus_url = URL("http://example.com/cal/foo+bar/") + before_plus = str(plus_url) + _ = plus_url == URL("http://example.com/cal/foo+bar/") + assert str(plus_url) == before_plus, "__eq__ mutated URL containing '+'" + # 10) pickle assert pickle.loads(pickle.dumps(url1)) == url1 @@ -1781,6 +1918,9 @@ def testExtractAuth(self): "basic", "digest", } + # §1.8: trailing comma (seen in the wild) must not raise IndexError + assert client.extract_auth_types("Basic,") == {"basic"} + assert client.extract_auth_types('Basic realm="x",') == {"basic"} def testAutoUrlEcloudWithEmailUsername(self) -> None: """ @@ -2095,8 +2235,8 @@ def test_add_object_orphan_does_not_raise_notfound(self): created = [] original_create = CalendarObjectResource._create - CalendarObjectResource._create = ( - lambda self_, id=None, path=None, retry_on_failure=True: created.append(True) + CalendarObjectResource._create = lambda self_, id=None, path=None, retry_on_failure=True: ( + created.append(True) ) try: calendar.add_object(Event, self._orphan_ical) @@ -2756,6 +2896,77 @@ def test_rate_limit_max_sleep_stops_adaptive_retries(self, mocked): client.request("/") +class TestAsyncProbeResponseNotReturnedAsReal: + """§2.15: async _async_request: when the probe GET for issue-#158 workaround does + not receive a 401+WWW-Authenticate response, the original exception must be re-raised. + + Before the fix, a probe response with status != 401 (e.g. 200 HTML login page) fell + through to response = DAVResponse(r, self), returning the probe GET response as if + it were the real request's response — status 200 for a PUT that never happened. + """ + + @pytest.mark.asyncio + async def test_probe_200_reraises_original_exception(self): + """If the probe GET returns 200 (not a 401 challenge), the original error must propagate.""" + from unittest.mock import AsyncMock, patch + + from caldav.async_davclient import AsyncDAVClient + + client = AsyncDAVClient(url="http://cal.example.com/", password="secret") + + probe_resp = mock.MagicMock() + probe_resp.status_code = 200 + probe_resp.reason = "OK" + probe_resp.headers = {"Content-Type": "text/html"} + probe_resp.reason_phrase = "OK" + + original_error = ConnectionError("server aborted connection") + + async def mock_request(*args, **kwargs): + if kwargs.get("method") == "GET" and not kwargs.get("auth"): + return probe_resp + raise original_error + + with patch.object(client.session, "request", side_effect=mock_request): + with pytest.raises((ConnectionError, Exception)): + await client._async_request("/some/resource", "PUT", "data", {}) + + +class TestRateLimitNoPlusNone: + """§1.3: rate-limit retry must not raise TypeError when second 429 has no usable Retry-After. + + sleep_seconds += rate_limit_time_slept / 2 executed before the is-None check, + so None += 2.5 raised TypeError instead of the documented RateLimitError. + """ + + def _make_response(self, status_code, headers=None): + r = mock.MagicMock() + r.status_code = status_code + r.headers = headers or {} + r.reason = "Too Many Requests" + return r + + @mock.patch("caldav.davclient.requests.Session.request") + def test_second_429_without_retry_after_raises_rate_limit_error(self, mocked): + """Second 429 with Retry-After: 0 (compute_sleep_seconds → None) must raise + RateLimitError, not TypeError.""" + ok = mock.MagicMock() + ok.status_code = 200 + ok.headers = {} + mocked.side_effect = [ + self._make_response(429, {"Retry-After": "5"}), + self._make_response(429, {"Retry-After": "0"}), # compute_sleep_seconds → None + ] + client = DAVClient( + url="http://cal.example.com/", + rate_limit_handle=True, + rate_limit_default_sleep=None, + ) + with mock.patch("caldav.davclient.time.sleep"): + with pytest.raises(error.RateLimitError): + client.request("/") + + class TestDateToUtcConversion: """ RFC 4791 §9.9: time-range start/end MUST be UTC datetime values. @@ -2897,6 +3108,26 @@ def test_recursive_meta_section(self): } assert set(expand_config_section(config, "all")) == {"a", "b", "c"} + def test_missing_section_returns_empty(self): + """§1.9: expand_config_section(config, "default") when "default" is absent must + return [] rather than raising KeyError.""" + from caldav.config import expand_config_section + + config = {"work": {"caldav_url": "https://work.example.com/"}} + # Requesting a section that doesn't exist should return [] (no match), not crash + assert expand_config_section(config, "default") == [] + + def test_disable_respected_for_named_sections(self): + """§2.17: disable:true must suppress named sections, not just glob '*' results. + + The old code used the literal string 'section' instead of the variable, + so disable was only effective under the '*' glob path. + """ + from caldav.config import expand_config_section + + config = {"work": {"caldav_url": "https://work.example.com/", "disable": True}} + assert expand_config_section(config, "work") == [] + class TestConfigSectionInheritance: """Unit tests for caldav.config.config_section (inherits key).""" @@ -3047,6 +3278,50 @@ def test_calendar_url_extracted_from_section(self, tmp_path): assert len(results) == 1 assert results[0]["calendar_url"] == "/dav/user/mycalendar/" + def test_section_with_features_but_no_url(self, tmp_path): + """A section without caldav_url is usable when it has features — + the client constructor resolves the URL from auto-connect.url hints.""" + import json + + from caldav.config import get_all_file_connection_params + + config = { + "ecloud": { + "caldav_username": "user@e.email", + "caldav_password": "pass", + "features": "ecloud", + } + } + config_file = tmp_path / "calendar.conf" + config_file.write_text(json.dumps(config)) + results = get_all_file_connection_params(str(config_file), "ecloud") + assert len(results) == 1 + assert results[0]["username"] == "user@e.email" + assert results[0]["features"] + + def test_get_connection_params_features_but_no_url(self, tmp_path): + """Same as above, but through get_connection_params — the code path + used by get_davclient(config_section=...).""" + import json + + from caldav.config import get_connection_params + + config = { + "ecloud": { + "caldav_username": "user@e.email", + "caldav_password": "pass", + "features": "ecloud", + } + } + config_file = tmp_path / "calendar.conf" + config_file.write_text(json.dumps(config)) + params = get_connection_params( + config_file=str(config_file), config_section="ecloud", environment=False + ) + assert params is not None + assert params["username"] == "user@e.email" + assert params["features"] + def test_meta_section_returns_multiple_dicts(self, tmp_path): import json @@ -3074,6 +3349,48 @@ def test_meta_section_returns_multiple_dicts(self, tmp_path): } +class TestExplicitParamsMerge: + """§2.18: get_connection_params explicit kwargs must be merged with env/file config. + + The old code only returned explicit_params when 'url' or 'features' was present; + params like password-only were silently discarded when an env/file source was found. + """ + + def test_explicit_password_merged_with_env_url(self, monkeypatch): + """get_connection_params(password='secret') with CALDAV_URL in env must include the password.""" + from caldav.config import get_connection_params + + monkeypatch.setenv("CALDAV_URL", "https://env.example.com/") + monkeypatch.setenv("CALDAV_USERNAME", "envuser") + # Unset file config to avoid config-file interference + monkeypatch.delenv("CALDAV_CONFIG_FILE", raising=False) + result = get_connection_params(password="secret", check_config_file=False) + assert result is not None + assert result.get("password") == "secret" + assert result.get("url") == "https://env.example.com/" + + +class TestResolveFeaturesMutation: + """§2.19: resolve_features and testing.py server classes must deepcopy hint dicts. + + Returning or shallow-copying a module-level dict then mutating a nested key + permanently corrupts the module-level dict for all subsequent users. + """ + + def test_resolve_features_string_returns_independent_copy(self): + """resolve_features('xandikos') must return a deep copy, not the module object.""" + from caldav import compatibility_hints as hints + from caldav.config import resolve_features + + original_domain = hints.xandikos.get("auto-connect.url", {}).get("domain", "") + result = resolve_features("xandikos") + # Mutate the returned copy + if "auto-connect.url" in result and isinstance(result["auto-connect.url"], dict): + result["auto-connect.url"]["domain"] = "MUTATED:9999" + # Original must be unchanged + assert hints.xandikos.get("auto-connect.url", {}).get("domain") == original_domain + + class TestResolveProperties: """Tests for _resolve_properties unbound variable bug (issue #647 / calendar-cli #114).""" @@ -3245,3 +3562,344 @@ def test_change_attendee_status_raises_when_username_not_email(self): ev = self._make_event_with_mock_client("just_a_username") with pytest.raises(caldav_error.NotFoundError): ev.change_attendee_status(partstat="ACCEPTED") + + +class TestAddAttendee: + """§1.6: add_attendee() crashes with UnboundLocalError on uppercase MAILTO: scheme. + + RFC 3986 §3.1 specifies URI schemes are case-insensitive, so "MAILTO:user@example.com" + is valid and common in real-world iCalendar data. The old code only matched lowercase + "mailto:" — uppercase fell through all string branches, leaving attendee_obj unassigned. + """ + + _base_event = """\ +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Test//Test//EN +BEGIN:VEVENT +UID:test-add-attendee@example.com +DTSTAMP:20240101T000000Z +DTSTART:20240601T100000Z +DTEND:20240601T110000Z +SUMMARY:Test event +END:VEVENT +END:VCALENDAR +""" + + def test_add_attendee_uppercase_mailto(self): + """add_attendee('MAILTO:user@example.com') must not raise UnboundLocalError.""" + ev = Event(data=self._base_event) + ev.add_attendee("MAILTO:user@example.com") + attendee = ev.icalendar_component["attendee"] + assert "user@example.com" in str(attendee).lower() + + def test_add_attendee_mixed_case_mailto(self): + """Mixed-case scheme variants like 'Mailto:' must also work.""" + ev = Event(data=self._base_event) + ev.add_attendee("Mailto:user@example.com") + attendee = ev.icalendar_component["attendee"] + assert "user@example.com" in str(attendee).lower() + + +class TestChangeAttendeeStatusNoAttendees: + """§1.7: change_attendee_status() raises bare KeyError when event has no ATTENDEE property. + + ical_obj["attendee"] raises KeyError when the key is absent; the NotFoundError-catching + loop in the Principal branch never sees it, so the "Principal is not invited" message + is unreachable and callers get an unexpected KeyError instead. + + Also: the not-found message contained a literal '%s' placeholder that was never + substituted. + """ + + _event_no_attendees = """\ +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Test//Test//EN +BEGIN:VEVENT +UID:test-no-attendees@example.com +DTSTAMP:20240101T000000Z +DTSTART:20240601T100000Z +DTEND:20240601T110000Z +SUMMARY:Event with no attendees +END:VEVENT +END:VCALENDAR +""" + + def test_change_attendee_status_no_attendees_raises_not_found(self): + """Calling change_attendee_status on an event with no ATTENDEE must raise + NotFoundError, not KeyError.""" + ev = Event(data=self._event_no_attendees) + with pytest.raises(error.NotFoundError): + ev.change_attendee_status("mailto:nobody@example.com", partstat="ACCEPTED") + + def test_change_attendee_status_error_message_contains_attendee(self): + """The not-found error message must contain the attendee address, not a literal '%s'.""" + ev = Event(data=self._event_no_attendees) + with pytest.raises(error.NotFoundError) as exc_info: + ev.change_attendee_status("mailto:nobody@example.com", partstat="ACCEPTED") + assert "%s" not in str(exc_info.value) + assert "nobody@example.com" in str(exc_info.value) + + +class TestFeatureSetCopyFeatureSet: + """§1.10 + §1.11: FeatureSet.copyFeatureSet() correctness bugs. + + §1.10: Merging a plain-string feature over an existing string-valued feature raised + bare AssertionError because the 'support' not in server_node guard prevented the + update branch from running. + + §1.11: An unknown feature name produced a UserWarning but was still stored in + _server_features; a later collapse()/is_supported() then hit a message-less + AssertionError far from the originating config. Unknown features must be skipped + (continue after warning) so bad keys never contaminate the feature set. + """ + + def test_string_feature_can_be_overridden(self): + """copyFeatureSet must accept a string value that overrides an existing string.""" + from caldav.compatibility_hints import FeatureSet + + fs = FeatureSet({"scheduling": "unsupported"}) + fs.copyFeatureSet({"scheduling": "fragile"}) + assert fs.is_supported("scheduling") is False # fragile → False per is_supported semantics + + def test_string_feature_full_override(self): + """Overriding 'unsupported' with 'full' must make is_supported return True.""" + from caldav.compatibility_hints import FeatureSet + + fs = FeatureSet({"scheduling": "unsupported"}) + fs.copyFeatureSet({"scheduling": "full"}) + assert fs.is_supported("scheduling") is True + + def test_unknown_feature_warns_and_does_not_store(self): + """An unknown feature name must emit UserWarning and must NOT be stored.""" + import warnings + + from caldav.compatibility_hints import FeatureSet + + fs = FeatureSet({}) + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + fs.copyFeatureSet({"totally_nonexistent_feature_xyz": "full"}) + + assert any("totally_nonexistent_feature_xyz" in str(warning.message) for warning in w) + # The bad key must NOT be in the internal feature dict + assert "totally_nonexistent_feature_xyz" not in fs._server_features + + +class TestXMLEntityHardening: + """§3.2: XML parser must not expand entity references from untrusted server data. + + etree.XMLParser without resolve_entities=False expands inline DOCTYPE + entities, allowing a malicious server to inject arbitrary text into + parsed values. With resolve_entities=False the entity reference is + left as-is (text becomes None for element content). + """ + + def test_xml_entity_not_expanded(self): + """Entity defined in DOCTYPE must NOT be expanded into element text.""" + xml = b""" +]> + + + / + + &xxe; + HTTP/1.1 200 OK + + +""" + resp = MockedDAVResponse(xml) + assert resp.tree is not None + displayname_el = resp.tree.find(".//{DAV:}displayname") + assert displayname_el is not None + assert displayname_el.text != "INJECTED", ( + "XML entity was expanded — resolve_entities=False is missing from the parser" + ) + + +class TestDAVClientCredentialPrecedence: + """§1.2: DAVClient credential handling bugs. + + - URL with username but no password (user@host) crashed with TypeError inside + urllib.parse.unquote(None). + - URL credentials had higher precedence than explicit kwargs; async client was + the opposite (explicit kwargs win). Now sync matches async: explicit kwargs win. + """ + + def test_url_with_user_but_no_password_does_not_crash(self): + """DAVClient(url='https://user@host/', password='p') must not raise TypeError.""" + client = DAVClient(url="https://user@cal.example.com/dav/", password="secret") + assert client.username == "user" + assert client.password == b"secret" + + def test_explicit_kwargs_take_precedence_over_url_credentials(self): + """Explicit username/password kwargs must override credentials embedded in the URL.""" + client = DAVClient( + url="https://urluser:urlpass@cal.example.com/dav/", + username="kwarguser", + password="kwargpass", + ) + assert client.username == "kwarguser" + assert client.password == b"kwargpass" + + +class TestPostPutRedirect: + """§1.1: 302 response to PUT must update event.url from Location header. + + Bug: `[x[1] for x in r.headers if x[0] == "location"]` iterates the + headers dict yielding key *strings*, so x[0] is the first character of + each header name — never "location". The list is always empty and any + 302 raises IndexError. + """ + + @mock.patch("caldav.davclient.requests.Session.request") + def test_302_put_updates_url_from_location_header(self, mocked): + try: + from niquests.structures import CaseInsensitiveDict + except ImportError: + from requests.structures import CaseInsensitiveDict + + new_url = "http://cal.example.com/cal/new-location.ics" + resp = mock.MagicMock() + resp.status_code = 302 + resp.headers = CaseInsensitiveDict({"Location": new_url}) + resp.reason = "Found" + resp.content = b"" + mocked.return_value = resp + + client = DAVClient(url="http://cal.example.com/") + cal = Calendar(client=client, url="http://cal.example.com/cal/") + event = Event( + client=client, + url="http://cal.example.com/cal/event.ics", + data=ev1, + parent=cal, + ) + event.save() + assert str(event.url) == new_url + + +class TestWarnUnreadableDisplayName: + """§5 (DRY): the shared helper backing the sync/async name-matching loops. + + Warn unless we positively know the server can't read DAV:displayname via + PROPFIND (propfind.displayname non-supported); warn when the feature is + supported or when there is no feature matrix to consult. + """ + + @staticmethod + def _client(features): + client = mock.MagicMock() + client.features = features + return client + + def test_warns_when_feature_supported(self, caplog): + from caldav.base_client import _warn_unreadable_display_name + from caldav.compatibility_hints import FeatureSet + + cal = mock.MagicMock() + cal.url = "http://cal.example.com/cal/" + with caplog.at_level("WARNING", logger="caldav"): + _warn_unreadable_display_name( + self._client(FeatureSet()), cal, "Work", Exception("boom") + ) + assert any("Could not read display name" in r.message for r in caplog.records) + + def test_silent_when_feature_unsupported(self, caplog): + from caldav.base_client import _warn_unreadable_display_name + from caldav.compatibility_hints import FeatureSet + + features = FeatureSet({"propfind.displayname": {"support": "unsupported"}}) + with caplog.at_level("WARNING", logger="caldav"): + _warn_unreadable_display_name( + self._client(features), mock.MagicMock(), "Work", Exception("boom") + ) + assert not caplog.records + + def test_silent_when_parent_propfind_unsupported(self, caplog): + from caldav.base_client import _warn_unreadable_display_name + from caldav.compatibility_hints import FeatureSet + + # propfind.displayname falls back to the propfind parent when not probed + features = FeatureSet({"propfind": {"support": "unsupported"}}) + with caplog.at_level("WARNING", logger="caldav"): + _warn_unreadable_display_name( + self._client(features), mock.MagicMock(), "Work", Exception("boom") + ) + assert not caplog.records + + def test_warns_when_no_feature_matrix(self, caplog): + from caldav.base_client import _warn_unreadable_display_name + + client = mock.MagicMock() + client.features = None + with caplog.at_level("WARNING", logger="caldav"): + _warn_unreadable_display_name(client, mock.MagicMock(), "Work", Exception("boom")) + assert any("Could not read display name" in r.message for r in caplog.records) + + +class TestRecurringCompleteHelpers: + """Pure (no-I/O) unit tests for the recurring-task completion helpers. + + These exercise the icalendar-mutation logic shared by the sync and async + ``complete()`` paths without touching a server. + ref https://github.com/python-caldav/caldav code-review §5.6 + """ + + def _make_todo(self, data: str = todo6) -> Todo: + client = MockedDAVClient("") + cal_url = "https://somwhere.in.the.universe.example/some/caldav/root/cal/" + calendar = Calendar(client, url=cal_url) + return Todo(client, data=data, parent=calendar, url=cal_url + "t.ics") + + def test_prepare_thisandfuture_advances_series(self) -> None: + todo = self._make_todo() + before = len(todo.icalendar_instance.subcomponents) + todo._prepare_recurring_thisandfuture(datetime(2026, 6, 14, tzinfo=timezone.utc)) + subs = todo.icalendar_instance.subcomponents + ## a completed recurrence + a new THISANDFUTURE instance were added + assert len(subs) == before + 2 + last = subs[-1] + assert last["RECURRENCE-ID"].params.get("RANGE") == "THISANDFUTURE" + ## one of the recurrences is now marked COMPLETED + assert any(str(s.get("STATUS")) == "COMPLETED" for s in subs) + + def test_build_safe_completed_returns_completed_copy(self) -> None: + todo = self._make_todo() + orig_dtstart = todo.icalendar_component["DTSTART"].dt + completed = todo._build_recurring_safe_completed(datetime(2026, 6, 14, tzinfo=timezone.utc)) + assert completed is not None + ## the standalone copy is completed and no longer recurring + assert str(completed.icalendar_component["STATUS"]) == "COMPLETED" + assert "RRULE" not in completed.icalendar_component + ## the master task advanced to its next occurrence and stays recurring + assert todo.icalendar_component["DTSTART"].dt > orig_dtstart + assert "RRULE" in todo.icalendar_component + + def test_build_safe_completed_none_when_count_one(self) -> None: + ## A recurring task with COUNT=1 is not really recurring + todo = self._make_todo() + todo.icalendar_component["RRULE"]["COUNT"] = [1] + assert ( + todo._build_recurring_safe_completed(datetime(2026, 6, 14, tzinfo=timezone.utc)) is None + ) + + def test_safe_completion_issues_two_puts(self, monkeypatch: Any) -> None: + """The standalone completed copy must not be PUT twice. + + The old async twin had drifted into saving the completed copy once + as still-pending and again as completed (3 PUTs total for the + operation). Completing in memory first means one PUT per object. + """ + saves: list[Todo] = [] + + def fake_save(self: Todo, *a: Any, **k: Any) -> Todo: + saves.append(self) + return self + + monkeypatch.setattr(Todo, "save", fake_save) + todo = self._make_todo() + todo._complete_recurring_safe(datetime(2026, 6, 14, tzinfo=timezone.utc)) + ## one PUT for the standalone completed copy, one for the advanced master + assert len(saves) == 2 diff --git a/tests/test_compatibility_hints.py b/tests/test_compatibility_hints.py index 3277871f..79dc0a9d 100644 --- a/tests/test_compatibility_hints.py +++ b/tests/test_compatibility_hints.py @@ -207,19 +207,27 @@ def test_collapse_parent_already_exists(self) -> None: assert fs._server_features["search.text"] == {"support": "fragile"} def test_collapse_parent_exists_same_value(self) -> None: - """When parent exists with same value as subfeatures, should still collapse""" + """When parent exists with same value as subfeatures, should still collapse. + + Uses a genuine *grouping* parent (principal-search.by-name has no + explicit default); independent parents such as sync-token are + intentionally never collapsed (see + test_collapse_independent_parent_not_collapsed). by-name's parent + principal-search has a second, unset child (list-all), so the collapse + does not cascade further up. + """ fs = FeatureSet() fs._server_features = { - "sync-token": {"support": "unsupported"}, - "sync-token.delete": {"support": "unsupported"}, + "principal-search.by-name": {"support": "unsupported"}, + "principal-search.by-name.self": {"support": "unsupported"}, } fs.collapse() # All have same value, so subfeature should be removed - assert "sync-token.delete" not in fs._server_features - assert fs._server_features["sync-token"] == {"support": "unsupported"} + assert "principal-search.by-name.self" not in fs._server_features + assert fs._server_features["principal-search.by-name"] == {"support": "unsupported"} def test_collapse_empty_featureset(self) -> None: """Collapse should handle empty featureset without errors""" @@ -243,19 +251,19 @@ def test_collapse_no_parent_features(self) -> None: assert fs._server_features == {"sync-token": {"support": "full"}} def test_collapse_single_subfeature(self) -> None: - """Single subfeature should collapse since parent derives from children""" + """Single subfeature should collapse since a grouping parent derives from children""" fs = FeatureSet() - # sync-token only has one subfeature: delete + # principal-search.by-name (a grouping node) only has one subfeature: self fs._server_features = { - "sync-token.delete": {"support": "unsupported"}, + "principal-search.by-name.self": {"support": "unsupported"}, } fs.collapse() # Parent status is derived from the single child, so collapse is valid - assert "sync-token" in fs._server_features - assert "sync-token.delete" not in fs._server_features + assert "principal-search.by-name" in fs._server_features + assert "principal-search.by-name.self" not in fs._server_features def test_collapse_with_complex_dict_values(self) -> None: """Collapse should handle complex dictionary values""" @@ -263,20 +271,20 @@ def test_collapse_with_complex_dict_values(self) -> None: complex_value = { "support": "fragile", - "behaviour": "time-based", + "behaviour": "inconsistent", "extra": "metadata", } fs._server_features = { - "sync-token": complex_value.copy(), - "sync-token.delete": complex_value.copy(), + "principal-search.by-name": complex_value.copy(), + "principal-search.by-name.self": complex_value.copy(), } fs.collapse() # Both have same value, should collapse - assert "sync-token.delete" not in fs._server_features - assert fs._server_features["sync-token"] == complex_value + assert "principal-search.by-name.self" not in fs._server_features + assert fs._server_features["principal-search.by-name"] == complex_value def test_collapse_principal_search_real_scenario(self) -> None: """Test user's real scenario: principal-search subfeatures with same value should collapse""" @@ -302,140 +310,66 @@ def test_collapse_principal_search_real_scenario(self) -> None: assert "principal-search.list-all" not in fs._server_features assert "principal-search" in fs._server_features - def test_independent_subfeature_not_derived(self) -> None: - """Test that independent subfeatures (with explicit defaults) don't affect parent derivation""" - fs = FeatureSet() - - # Scenario: create-calendar.auto is set to unsupported, but it's an independent - # feature (has explicit default) and should NOT cause create-calendar to be - # derived as unsupported - fs._server_features = { - "create-calendar.auto": {"support": "unsupported"}, - } - - # create-calendar should return its default (full), NOT derive from .auto - result = fs.is_supported("create-calendar", return_type=dict) - assert result == {"support": "full"}, ( - f"create-calendar should default to 'full' when only independent " - f"subfeature .auto is set, but got {result}" - ) - - # Verify that the independent subfeature itself is still accessible - auto_result = fs.is_supported("create-calendar.auto", return_type=dict) - assert auto_result == {"support": "unsupported"} - - def test_parent_default_not_overridden_by_subfeature_derivation(self) -> None: - """Test that a parent with an explicit default is not overridden by subfeature derivation. + def test_collapse_independent_parent_not_collapsed(self) -> None: + """An independent parent (one with its own explicit default) is never + folded away by its children. - Zimbra scenario: create-calendar.set-displayname is unsupported, but - create-calendar has an explicit default of 'full'. The parent feature - represents an independent capability (calendar creation works), so the - subfeature status should not override the default. + sync-token carries a default, so even when its only child + sync-token.delete is unsupported the parent keeps its own (separately + probed) status: the two represent distinct capabilities and must not be + conflated. """ fs = FeatureSet() fs._server_features = { - "create-calendar.set-displayname": {"support": "unsupported"}, + "sync-token": {"support": "full"}, + "sync-token.delete": {"support": "unsupported"}, } - # create-calendar should return its default (full), NOT derive unsupported - # from .set-displayname - result = fs.is_supported("create-calendar", return_type=dict) - assert result == {"support": "full"}, ( - f"create-calendar should default to 'full' even when " - f".set-displayname is unsupported, but got {result}" - ) - - def test_hierarchical_vs_independent_subfeatures(self) -> None: - """Test that hierarchical subfeatures derive parent, but independent ones don't""" - fs = FeatureSet() - - # Hierarchical subfeatures: principal-search.by-name and principal-search.list-all - # These should cause parent to derive to "unknown" when mixed - fs.set_feature("principal-search.by-name", {"support": "unknown"}) - fs.set_feature("principal-search.list-all", {"support": "unsupported"}) - - # Should derive to "unknown" due to mixed hierarchical subfeatures - result = fs.is_supported("principal-search", return_type=dict) - assert result == {"support": "unknown"}, ( - f"principal-search should derive to 'unknown' from mixed hierarchical " - f"subfeatures, but got {result}" - ) - - # Now test independent subfeature: create-calendar.auto - # This should NOT affect create-calendar parent - fs2 = FeatureSet() - fs2.set_feature("create-calendar.auto", {"support": "unsupported"}) - - # Should return default, NOT derive from independent subfeature - result2 = fs2.is_supported("create-calendar", return_type=dict) - assert result2 == {"support": "full"}, ( - f"create-calendar should default to 'full' ignoring independent " - f"subfeature .auto, but got {result2}" - ) + fs.collapse() - def test_intermediate_feature_derives_from_children(self) -> None: - """Test that intermediate features (e.g. search.text) derive status from their children""" - # search.text has 4 direct children: case-sensitive, case-insensitive, - # substring, category (none have explicit defaults) + assert fs._server_features["sync-token"] == {"support": "full"} + assert fs._server_features["sync-token.delete"] == {"support": "unsupported"} - # All children set with mixed statuses -> derive "unknown" - fs = FeatureSet( - { - "search.text.case-sensitive": {"support": "unsupported"}, - "search.text.case-insensitive": {"support": "unsupported"}, - "search.text.substring": {"support": "unsupported"}, - "search.text.category": {"support": "fragile"}, - } - ) - assert not fs.is_supported("search.text") - assert fs.is_supported("search.text", return_type=dict) == {"support": "unknown"} + def test_collapse_does_not_alter_independent_sibling(self) -> None: + """collapse() must be lossless w.r.t. is_supported() for *every* + subfeature, including independent siblings. - # Partial children set with mixed non-positive statuses -> inconclusive, - # falls back to default ("full") - fs1b = FeatureSet( - { - "search.text.case-sensitive": {"support": "unsupported"}, - "search.text.category.substring": {"support": "fragile"}, - } - ) - assert fs1b.is_supported("search.text") + Regression for the save.duplicate-event compatibility-test breakage: + only save.duplicate-uid.cross-calendar was declared (ungraceful). The + grouping chain duplicate-uid -> save would have collapsed into an + explicit save=ungraceful, which the independent sibling + save.duplicate-event (own default "full") then inherited - flipping it + from "full" to "ungraceful". collapse() must not fold up to a parent + that has an independent child whose resolution would change. + """ + fs = FeatureSet({"save.duplicate-uid.cross-calendar": {"support": "ungraceful"}}) - # All children unsupported -> parent derives as "unsupported" - fs2 = FeatureSet( - { - "search.text.case-sensitive": {"support": "unsupported"}, - "search.text.case-insensitive": {"support": "unsupported"}, - "search.text.substring": {"support": "unsupported"}, - "search.text.category": {"support": "unsupported"}, - } - ) - assert not fs2.is_supported("search.text") - assert fs2.is_supported("search.text", return_type=dict) == {"support": "unsupported"} + before = fs.is_supported("save.duplicate-event", return_type=str) + assert before == "full" - # No children set -> falls back to default ("full") - fs3 = FeatureSet({}) - assert fs3.is_supported("search.text") + fs.collapse() - # Explicit parent value takes precedence over children - fs4 = FeatureSet( - { - "search.text": {"support": "full"}, - "search.text.case-sensitive": {"support": "unsupported"}, - } + after = fs.is_supported("save.duplicate-event", return_type=str) + assert after == "full", ( + f"collapse() changed save.duplicate-event from {before!r} to {after!r}" ) - assert fs4.is_supported("search.text") - + # The genuine observation is preserved either way. + assert fs.is_supported("save.duplicate-uid.cross-calendar", return_type=str) == "ungraceful" -class TestDeriveFromSubfeatures: - """Test _derive_from_subfeatures with partial and complete subfeature configs. - Uses search.recurrences which has two relevant children without defaults: - - search.recurrences.expanded - - search.recurrences.includes-implicit +class TestImplicitDerivation: + """Test is_supported() implicit derivation: parent→child, child→parent, explicit defaults. - The default for search.recurrences (a server-feature) is {"support": "full"}. + Covers: + - Children without explicit defaults derive the parent value. + - Parent set explicitly propagates down to unset children. + - Features with explicit defaults ignore subfeature derivation. + - Partial/incomplete child sets fall through to the feature's default. """ + ## TODO: the tests covering "all children" may need to be + ## protected against future additions in compatibility_hints.py + @pytest.mark.parametrize( "scenario, config, query, expected_support", [ @@ -448,6 +382,22 @@ class TestDeriveFromSubfeatures: "search.recurrences", "unsupported", ), + ( + "parent_unsupported", + { + "save-load": {"support": "unsupported"}, + }, + "save-load.event", + "unsupported", + ), + ( + "parent_with_explicit_default_unsupported", + { + "create-calendar": {"support": "unsupported"}, + }, + "create-calendar.auto", + "unsupported", + ), ( "all_children_supported", { @@ -482,6 +432,16 @@ class TestDeriveFromSubfeatures: "search.recurrences", "full", # any positive support → derive as supported ), + ( + ## Earlier logic had it that if a node has only one child, the parent should not be affected by the child, but if there are more children and all are unsupported, the parent is automatically flipped to unsupported. However, this special case logic should have been rendered obsolete by the new logic that every node having an explicit default is considered independent + "independent_feature_always_trumps", + { + "save-load.mutable.attendee-partstat": {"support": "unsupported"}, + "save-load.mutable.if-match-optional": {"support": "unsupported"}, + }, + "save-load.mutable", + "full", + ), ( "gmx_partial_unsupported_query_unset_sibling_child", { @@ -507,6 +467,47 @@ class TestDeriveFromSubfeatures: "search.recurrences.includes-implicit.todo", "full", ), + ( + "mixed_children_incomplete_unset_sibling_falls_to_default", + { + "save-load.todo": {"support": "full"}, + "save-load.journal": {"support": "unsupported"}, + }, + "save-load.event", + "full", # incomplete set: cannot derive anything about unset siblings + ), + ( + "explicit_default_overrides_children", + { + "create-calendar.auto": {"support": "unsupported"}, + "create-calendar.set-displayname": {"support": "unsupported"}, + }, + "create-calendar", + "full", # this feature does not depend on the sub-features + ), + ( + "partial_mixed_children_query_parent_falls_to_default", + { + "search.text.case-sensitive": {"support": "unsupported"}, + "search.text.case-insensitive": {"support": "full"}, + }, + "search.text", + "full", # partial+mixed: cannot conclude unsupported; default applies + ), + ( + ## Regression: setting a sibling child (search.text=full) caused + ## _derive_from_subfeatures on the grandparent "search" to return + ## full, which then bled into independent sibling features that have + ## their own explicit default. search.time-range.comp-type-optional + ## has default=unsupported and must not be overridden by a derived + ## (not explicitly set) ancestor status. + "derived_parent_does_not_bleed_into_independent_sibling", + { + "search.text": {"support": "full"}, + }, + "search.time-range.comp-type-optional", + "unsupported", # own explicit default, must not inherit derived "full" from ancestor + ), ], ids=lambda x: x if isinstance(x, str) and "_" in x else "", ) @@ -535,13 +536,16 @@ def test_string_resolves_profile(self) -> None: import caldav.compatibility_hints as ch result = _resolve_features("synology") - assert result is ch.synology + assert result == ch.synology + # deepcopy ensures the caller cannot mutate the shared profile + assert result is not ch.synology def test_string_with_prefix(self) -> None: import caldav.compatibility_hints as ch result = _resolve_features("compatibility_hints.synology") - assert result is ch.synology + assert result == ch.synology + assert result is not ch.synology def test_dict_without_base_passes_through(self) -> None: features = {"search.text": {"support": "unsupported"}} @@ -579,3 +583,50 @@ def test_base_with_prefix(self) -> None: assert result["sync-token"] == "full" # Original base feature should be overridden assert result["sync-token"] != "fragile" + + +class TestFeatureSetCompare: + """Test FeatureSet.compare(): declared (expected) vs observed feature sets.""" + + def test_matching_sets_no_mismatch(self) -> None: + expected = FeatureSet({"search.comp-type": {"support": "unsupported"}}) + observed = FeatureSet() + observed.set_feature("search.comp-type", "unsupported") + assert expected.compare(observed) == [] + + def test_subfeature_observed_default_conflicts_with_inherited_unsupported( + self, + ) -> None: + """Regression for the Infomaniak ``search.comp-type.optional`` blind spot. + + The parent ``search.comp-type`` is declared ``unsupported``, so the child + ``search.comp-type.optional`` inherits ``unsupported``. The server is + observed to *support* the child (``full``) - but ``full`` happens to be + the child's implicit default, so it is dropped from the compacted + observed dict. The mismatch must still be reported (it previously + slipped through because the feature was in neither compacted dict). + """ + expected = FeatureSet({"search.comp-type": {"support": "unsupported"}}) + observed = FeatureSet() + observed.set_feature("search.comp-type", "unsupported") + observed.set_feature("search.comp-type.optional") # -> {"support": "full"} + + mismatches = expected.compare(observed) + + by_feature = {m["feature"]: m for m in mismatches} + assert "search.comp-type.optional" in by_feature + assert by_feature["search.comp-type.optional"]["expected"] == "unsupported" + assert by_feature["search.comp-type.optional"]["observed"] == "full" + + def test_unprobed_declared_feature_is_not_flagged(self) -> None: + """A feature declared unsupported but never probed by the tester must not + be reported - we have no observation to contradict it.""" + expected = FeatureSet({"search.comp-type": {"support": "unsupported"}}) + observed = FeatureSet() # tester probed nothing + assert expected.compare(observed) == [] + + def test_fragile_and_unknown_are_ignored(self) -> None: + expected = FeatureSet({"search.comp-type": {"support": "unsupported"}}) + observed = FeatureSet() + observed.set_feature("search.comp-type", "fragile") + assert expected.compare(observed) == [] diff --git a/tests/test_discovery.py b/tests/test_discovery.py new file mode 100644 index 00000000..84b7a8a1 --- /dev/null +++ b/tests/test_discovery.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +""" +Unit tests for caldav.discovery — RFC 6764 service discovery. + +No network communication; DNS and HTTP are mocked. +""" + +from unittest import mock + +from caldav.discovery import discover_service + + +def _make_redirect_response(location: str, status_code: int = 302): + """Return a minimal mock HTTP response that redirects to *location*.""" + resp = mock.MagicMock() + resp.status_code = status_code + resp.headers = {"Location": location} + return resp + + +def _make_ok_response(): + resp = mock.MagicMock() + resp.status_code = 200 + resp.headers = {} + return resp + + +class TestRequireTLSDowngradeBlocked: + """§3.1: require_tls=True must be enforced on the well-known redirect target. + + _well_known_lookup always probes https:// but never received require_tls, + so a same-domain redirect to http:// passed the domain-validation check and + was returned as ServiceInfo(tls=False). discover_service returned it + unchecked — a misconfigured or MITM server could silently downgrade TLS. + """ + + @mock.patch("caldav.discovery.requests.get") + @mock.patch("caldav.discovery._srv_lookup", return_value=[]) + def test_http_redirect_rejected_when_require_tls(self, _srv, mock_get): + """discover_service(require_tls=True) must return None when the + well-known URI redirects to a plain-HTTP URL.""" + mock_get.return_value = _make_redirect_response( + "http://example.com/caldav/" # same domain, but HTTP + ) + + result = discover_service("example.com", require_tls=True) + + assert result is None, f"Expected None (TLS downgrade rejected), got {result}" + + @mock.patch("caldav.discovery.requests.get") + @mock.patch("caldav.discovery._srv_lookup", return_value=[]) + def test_http_redirect_accepted_when_require_tls_false(self, _srv, mock_get): + """discover_service(require_tls=False) must accept an HTTP redirect.""" + mock_get.return_value = _make_redirect_response("http://example.com/caldav/") + + result = discover_service("example.com", require_tls=False) + + assert result is not None + assert result.tls is False + assert result.url == "http://example.com/caldav/" + + @mock.patch("caldav.discovery.requests.get") + @mock.patch("caldav.discovery._srv_lookup", return_value=[]) + def test_https_redirect_accepted_when_require_tls(self, _srv, mock_get): + """HTTPS redirect is always accepted regardless of require_tls.""" + mock_get.return_value = _make_redirect_response("https://caldav.example.com/dav/") + + result = discover_service("example.com", require_tls=True) + + assert result is not None + assert result.tls is True + assert "caldav.example.com" in result.url + + @mock.patch("caldav.discovery.requests.get") + @mock.patch("caldav.discovery._srv_lookup", return_value=[]) + def test_cross_domain_http_redirect_also_rejected(self, _srv, mock_get): + """A cross-domain HTTP redirect must be rejected (domain check fires first, + but require_tls must also be a backstop).""" + mock_get.return_value = _make_redirect_response("http://evil.attacker.com/caldav/") + + result = discover_service("example.com", require_tls=True) + + assert result is None diff --git a/tests/test_jmap_integration.py b/tests/test_jmap_integration.py index 02f04fd3..255d249e 100644 --- a/tests/test_jmap_integration.py +++ b/tests/test_jmap_integration.py @@ -220,7 +220,7 @@ def test_event_sync(self, client, calendar_id): token_before = client.get_sync_token() event_id = client.create_event(calendar_id, _minimal_ical("Sync Test Event")) try: - added, _modified, _deleted = client.get_objects_by_sync_token(token_before) + added, _modified, _deleted, _new_token = client.get_objects_by_sync_token(token_before) assert any("Sync Test Event" in jscal_to_ical(a.get_data()) for a in added) finally: client.delete_event(event_id) @@ -280,7 +280,9 @@ async def test_event_sync(self, async_client, async_calendar_id): async_calendar_id, _minimal_ical("Async Sync Test Event") ) try: - added, _modified, _deleted = await async_client.get_objects_by_sync_token(token_before) + added, _modified, _deleted, _new_token = await async_client.get_objects_by_sync_token( + token_before + ) assert any("Async Sync Test Event" in jscal_to_ical(a.get_data()) for a in added) finally: await async_client.delete_event(event_id) diff --git a/tests/test_jmap_unit.py b/tests/test_jmap_unit.py index 4b7a6dc6..3d12cdd8 100644 --- a/tests/test_jmap_unit.py +++ b/tests/test_jmap_unit.py @@ -207,6 +207,8 @@ def test_picks_first_calendar_capable_account(self): assert session.account_id == "user_calendar" +from datetime import datetime, timezone + from caldav.jmap.objects.calendar import JMAPCalendar from caldav.jmap.objects.calendar_object import JMAPCalendarObject @@ -341,7 +343,9 @@ def capturing_post(*args, **kwargs): mock_resp.raise_for_status = MagicMock() return mock_resp - monkeypatch.setattr("caldav.jmap.client.requests.post", capturing_post) + mock_http = MagicMock() + mock_http.post.side_effect = capturing_post + client._http_session = mock_http cal = JMAPCalendar(id=calendar_id, name="Test") cal._client = client cal._is_async = False @@ -372,6 +376,26 @@ def test_calendar_search_with_date_range(self, monkeypatch): assert query_args["filter"]["after"] == "2026-01-01T00:00:00" assert query_args["filter"]["before"] == "2026-12-31T23:59:59" + def test_calendar_search_datetime_converted_to_utcdate(self, monkeypatch): + """§4.6: datetime.isoformat() produced wrong format for JMAP UTCDate. + Naive datetimes produce no Z, aware non-UTC produce +HH:MM offset; + JMAP requires ...Z (UTC, no microseconds).""" + import datetime as _dt + + resp = self._query_get_response([self._RAW_EVENT]) + cal, captured = self._capturing_calendar(monkeypatch, resp) + tz_plus2 = _dt.timezone(_dt.timedelta(hours=2)) + start_aware = datetime(2026, 6, 1, 12, 0, 0, tzinfo=tz_plus2) # +02:00 noon → UTC 10:00 + end_utc = datetime(2026, 12, 31, 23, 59, 59, tzinfo=timezone.utc) + cal.search(start=start_aware, end=end_utc) + query_args = captured["json"]["methodCalls"][0][1] + assert query_args["filter"]["after"] == "2026-06-01T10:00:00Z", ( + f"Expected UTC Z-format, got {query_args['filter']['after']!r}" + ) + assert query_args["filter"]["before"] == "2026-12-31T23:59:59Z", ( + f"Expected UTC Z-format, got {query_args['filter']['before']!r}" + ) + def test_calendar_search_ignores_unknown_params(self, monkeypatch): """Verify that unknown search parameters are silently ignored.""" resp = self._query_get_response([self._RAW_EVENT]) @@ -567,7 +591,9 @@ def _make_client_with_mocked_session(monkeypatch, api_response_json): mock_resp.status_code = 200 mock_resp.json.return_value = api_response_json mock_resp.raise_for_status = MagicMock() - monkeypatch.setattr("caldav.jmap.client.requests.post", lambda *a, **kw: mock_resp) + mock_http = MagicMock() + mock_http.post.return_value = mock_resp + client._http_session = mock_http return client @@ -585,6 +611,33 @@ def test_context_manager(self): with JMAPClient(url="http://x", username="u", password="p") as client: assert isinstance(client, JMAPClient) + def test_context_manager_closes_http_session(self): + mock_close = MagicMock() + mock_http = MagicMock() + mock_http.close = mock_close + with patch("caldav.jmap.client.requests.Session", return_value=mock_http): + client = JMAPClient(url="http://x", username="u", password="p") + with client: + assert client._http_session is mock_http + mock_close.assert_called_once() + assert client._http_session is None + + def test_http_session_reused_across_requests(self, monkeypatch): + client = JMAPClient(url=_JMAP_URL, username=_USERNAME, password=_PASSWORD) + client._session_cache = Session(api_url=_API_URL, account_id=_USERNAME, state="s") + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.json.return_value = {"methodResponses": []} + mock_resp.raise_for_status = MagicMock() + with patch("caldav.jmap.client.requests.Session") as MockSession: + mock_sess = MagicMock() + mock_sess.post.return_value = mock_resp + MockSession.return_value = mock_sess + client._request([("Calendar/get", {}, "c0")]) + client._request([("Calendar/get", {}, "c1")]) + MockSession.assert_called_once() + assert mock_sess.post.call_count == 2 + def test_build_auth_basic_when_username_given(self): client = JMAPClient(url="http://x", username="u", password="p") assert isinstance(client._auth, HTTPBasicAuth) @@ -645,7 +698,9 @@ def test_request_raises_auth_error_on_401(self, monkeypatch): mock_resp = MagicMock() mock_resp.status_code = 401 mock_resp.raise_for_status = MagicMock() - monkeypatch.setattr("caldav.jmap.client.requests.post", lambda *a, **kw: mock_resp) + mock_http = MagicMock() + mock_http.post.return_value = mock_resp + client._http_session = mock_http with pytest.raises(JMAPAuthError): client._request([("Calendar/get", {"accountId": _USERNAME, "ids": None}, "c0")]) @@ -899,7 +954,7 @@ def test_parse_event_set_partial_failure(self): assert not_destroyed["ev-old"]["type"] == "notFound" -from datetime import date, datetime, timedelta, timezone +from datetime import date, timedelta import icalendar as _icalendar @@ -968,8 +1023,9 @@ def test_duration_round_trip(self): assert _duration_to_timedelta(_timedelta_to_duration(td)) == td def test_format_local_dt_utc(self): + # RFC 8984: LocalDateTime slots (override keys, RRULE until) must not carry Z suffix. dt = datetime(2024, 6, 15, 9, 0, 0, tzinfo=timezone.utc) - assert _format_local_dt(dt) == "2024-06-15T09:00:00Z" + assert _format_local_dt(dt) == "2024-06-15T09:00:00" def test_format_local_dt_naive(self): dt = datetime(2024, 6, 15, 9, 0, 0) @@ -1604,6 +1660,112 @@ def test_update_event_drops_uid_from_patch(self, monkeypatch): patch = update_args["update"]["ev1"] assert "uid" not in patch + def test_update_event_nulls_removed_optional_properties(self, monkeypatch): + # RFC 8620 §3.3: absent keys in a PatchObject preserve the server value. + # To actually delete a property the patch must set it to null. + # An ical → jscal conversion that omits LOCATION/DESCRIPTION must send + # {"locations": null, "description": null, ...} so the server removes them. + _ICAL_WITH_LOCATION = ( + "BEGIN:VCALENDAR\r\nVERSION:2.0\r\n" + "BEGIN:VEVENT\r\n" + "UID:loc-uid@example.com\r\n" + "DTSTART:20240615T090000Z\r\n" + "SUMMARY:Event with Location\r\n" + "LOCATION:Old Conference Room\r\n" + "END:VEVENT\r\nEND:VCALENDAR\r\n" + ) + _ICAL_WITHOUT_LOCATION = ( + "BEGIN:VCALENDAR\r\nVERSION:2.0\r\n" + "BEGIN:VEVENT\r\n" + "UID:loc-uid@example.com\r\n" + "DTSTART:20240615T090000Z\r\n" + "SUMMARY:Event without Location\r\n" + "END:VEVENT\r\nEND:VCALENDAR\r\n" + ) + resp = self._set_response(updated={"ev1": None}) + client, captured = self._capturing_client(monkeypatch, resp) + # First, pretend the event had a location (we don't need to call create; just update) + client.update_event("ev1", _ICAL_WITHOUT_LOCATION) + patch = captured["json"]["methodCalls"][0][1]["update"]["ev1"] + # The patch must contain explicit null for 'locations' to remove it from the server + assert "locations" in patch + assert patch["locations"] is None + + def _sequence_client(self, responses): + """Return (client, captured) replaying ``responses`` POST-by-POST. + + ``captured["patches"]`` collects the ``update`` patch dict sent on each + CalendarEvent/set POST, in order. + """ + captured: dict = {"patches": []} + client = JMAPClient(url=_JMAP_URL, username=_USERNAME, password=_PASSWORD) + client._session_cache = Session(api_url=_API_URL, account_id=_USERNAME, state="state-abc") + seq = iter(responses) + + def post(*args, **kwargs): + body = kwargs.get("json", {}) + update = body["methodCalls"][0][1].get("update") + if update: + captured["patches"].append(dict(update["ev1"])) # copy: caller mutates in place + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.json.return_value = next(seq) + mock_resp.raise_for_status = MagicMock() + return mock_resp + + mock_http = MagicMock() + mock_http.post.side_effect = post + client._http_session = mock_http + return client, captured + + def test_update_event_retries_dropping_server_rejected_null_keys(self, monkeypatch): + # A server (e.g. Stalwart) rejects null-clearing of recurrence properties + # it does not support, reporting one offending property per response. + # update_event must drop each reported null-cleanup key and retry until + # the update succeeds — never failing on harmless cleanup. + def reject(prop): + return self._set_response( + notUpdated={ + "ev1": { + "type": "invalidProperties", + "description": "Invalid property.", + "properties": [prop], + } + } + ) + + responses = [ + reject("recurrenceRules"), + reject("excludedRecurrenceRules"), + self._set_response(updated={"ev1": None}), + ] + client, captured = self._sequence_client(responses) + client.update_event("ev1", self._MINIMAL_ICAL) + + assert len(captured["patches"]) == 3 + # First attempt nulled both recurrence keys; the final accepted patch dropped them. + assert captured["patches"][0]["recurrenceRules"] is None + assert "recurrenceRules" not in captured["patches"][2] + assert "excludedRecurrenceRules" not in captured["patches"][2] + + def test_update_event_does_not_drop_explicitly_set_property(self, monkeypatch): + # If the rejected property was actually assigned a value by the client + # (not null-cleanup), the rejection is genuine and must surface — no retry. + resp = self._set_response( + notUpdated={ + "ev1": { + "type": "invalidProperties", + "description": "Invalid property.", + "properties": ["title"], + } + } + ) + client, captured = self._sequence_client([resp]) + with pytest.raises(JMAPMethodError) as exc_info: + client.update_event("ev1", self._MINIMAL_ICAL) + assert exc_info.value.error_type == "invalidProperties" + assert len(captured["patches"]) == 1 # no retry + def test_delete_event_success(self, monkeypatch): resp = self._set_response(destroyed=["ev1"]) client = _make_client_with_mocked_session(monkeypatch, resp) @@ -1630,7 +1792,9 @@ def capturing_post(*args, **kwargs): mock_resp.raise_for_status = MagicMock() return mock_resp - monkeypatch.setattr("caldav.jmap.client.requests.post", capturing_post) + mock_http = MagicMock() + mock_http.post.side_effect = capturing_post + client._http_session = mock_http return client, captured def _query_get_response(self, items): @@ -1753,14 +1917,22 @@ def _make_client(self): client._session_cache = Session(api_url=_API_URL, account_id=_USERNAME, state="state-abc") return client - def test_get_sync_token_returns_state(self, monkeypatch): + def _mock_http(self, client, response=None, side_effect=None): + mock_http = MagicMock() + if side_effect is not None: + mock_http.post.side_effect = side_effect + elif response is not None: + mock_http.post.return_value = response + client._http_session = mock_http + return mock_http + + def test_get_sync_token_returns_state(self): resp = self._get_resp_with_state([], state="tok-1") - monkeypatch.setattr( - "caldav.jmap.client.requests.post", lambda *a, **kw: self._make_mock(resp) - ) - assert self._make_client().get_sync_token() == "tok-1" + client = self._make_client() + self._mock_http(client, self._make_mock(resp)) + assert client.get_sync_token() == "tok-1" - def test_get_sync_token_sends_empty_ids(self, monkeypatch): + def test_get_sync_token_sends_empty_ids(self): captured = {} resp = self._get_resp_with_state([]) @@ -1768,61 +1940,74 @@ def capturing_post(*args, **kwargs): captured["json"] = kwargs.get("json", {}) return self._make_mock(resp) - monkeypatch.setattr("caldav.jmap.client.requests.post", capturing_post) - self._make_client().get_sync_token() + client = self._make_client() + self._mock_http(client, side_effect=capturing_post) + client.get_sync_token() assert captured["json"]["methodCalls"][0][1]["ids"] == [] - def test_get_objects_no_changes(self, monkeypatch): + def test_get_objects_no_changes(self): resp = self._changes_resp() - monkeypatch.setattr( - "caldav.jmap.client.requests.post", lambda *a, **kw: self._make_mock(resp) - ) - added, modified, deleted = self._make_client().get_objects_by_sync_token("state-1") + client = self._make_client() + self._mock_http(client, self._make_mock(resp)) + added, modified, deleted, _ = client.get_objects_by_sync_token("state-1") assert added == [] and modified == [] and deleted == [] - def test_get_objects_deleted_returns_ids(self, monkeypatch): + def test_get_objects_deleted_returns_ids(self): resp = self._changes_resp(destroyed=["ev1"]) - monkeypatch.setattr( - "caldav.jmap.client.requests.post", lambda *a, **kw: self._make_mock(resp) - ) - added, modified, deleted = self._make_client().get_objects_by_sync_token("state-1") + client = self._make_client() + self._mock_http(client, self._make_mock(resp)) + added, modified, deleted, _ = client.get_objects_by_sync_token("state-1") assert deleted == ["ev1"] and added == [] and modified == [] - def test_get_objects_added_returns_ical(self, monkeypatch): + def test_get_objects_added_returns_ical(self): changes_resp = self._changes_resp(created=["ev1"]) get_resp = self._get_resp_with_state([self._RAW_EVENT]) - mock_post = MagicMock( - side_effect=[self._make_mock(changes_resp), self._make_mock(get_resp)] + client = self._make_client() + self._mock_http( + client, + side_effect=[self._make_mock(changes_resp), self._make_mock(get_resp)], ) - monkeypatch.setattr("caldav.jmap.client.requests.post", mock_post) - added, modified, deleted = self._make_client().get_objects_by_sync_token("state-1") + added, modified, deleted, _ = client.get_objects_by_sync_token("state-1") assert len(added) == 1 assert isinstance(added[0], JMAPCalendarObject) assert added[0].id == "ev1" assert modified == [] and deleted == [] - def test_get_objects_modified_returns_ical(self, monkeypatch): + def test_get_objects_modified_returns_ical(self): changes_resp = self._changes_resp(updated=["ev1"]) get_resp = self._get_resp_with_state([self._RAW_EVENT]) - mock_post = MagicMock( - side_effect=[self._make_mock(changes_resp), self._make_mock(get_resp)] + client = self._make_client() + self._mock_http( + client, + side_effect=[self._make_mock(changes_resp), self._make_mock(get_resp)], ) - monkeypatch.setattr("caldav.jmap.client.requests.post", mock_post) - added, modified, deleted = self._make_client().get_objects_by_sync_token("state-1") + added, modified, deleted, _ = client.get_objects_by_sync_token("state-1") assert len(modified) == 1 assert isinstance(modified[0], JMAPCalendarObject) assert modified[0].id == "ev1" assert added == [] and deleted == [] - def test_get_objects_has_more_raises(self, monkeypatch): + def test_get_objects_has_more_raises(self): resp = self._changes_resp(created=["ev1"], has_more=True) - monkeypatch.setattr( - "caldav.jmap.client.requests.post", lambda *a, **kw: self._make_mock(resp) - ) + client = self._make_client() + self._mock_http(client, self._make_mock(resp)) with pytest.raises(JMAPMethodError) as exc_info: - self._make_client().get_objects_by_sync_token("state-1") + client.get_objects_by_sync_token("state-1") assert exc_info.value.error_type == "serverPartialFail" + def test_get_objects_returns_new_sync_token(self): + """§4.7: newState from /changes was discarded into _. Callers had no + way to chain sync calls without a separate get_sync_token() round-trip, + creating a race window where intervening changes would be silently missed.""" + resp = self._changes_resp(new_state="state-99") + client = self._make_client() + self._mock_http(client, self._make_mock(resp)) + result = client.get_objects_by_sync_token("state-1") + assert len(result) == 4, "expected 4-tuple (added, modified, deleted, new_sync_token)" + added, modified, deleted, new_token = result + assert new_token == "state-99" + assert added == [] and modified == [] and deleted == [] + def test_parse_event_changes_all_fields(self): resp_args = { "oldState": "s1", @@ -1969,25 +2154,32 @@ def _make_client(self): client._session_cache = Session(api_url=_API_URL, account_id=_USERNAME, state="state-abc") return client - def test_get_task_lists_returns_list(self, monkeypatch): + def _mock_http(self, client, response=None, side_effect=None): + mock_http = MagicMock() + if side_effect is not None: + mock_http.post.side_effect = side_effect + elif response is not None: + mock_http.post.return_value = response + client._http_session = mock_http + return mock_http + + def test_get_task_lists_returns_list(self): resp = self._tasklist_response([self._MINIMAL_TASKLIST]) - monkeypatch.setattr( - "caldav.jmap.client.requests.post", lambda *a, **kw: self._make_mock(resp) - ) - result = self._make_client().get_task_lists() + client = self._make_client() + self._mock_http(client, self._make_mock(resp)) + result = client.get_task_lists() assert len(result) == 1 assert isinstance(result[0], dict) assert result[0]["name"] == "My Tasks" - def test_create_task_returns_server_id(self, monkeypatch): + def test_create_task_returns_server_id(self): resp = self._set_response(created={"new-0": {"id": "sv-task-1"}}) - monkeypatch.setattr( - "caldav.jmap.client.requests.post", lambda *a, **kw: self._make_mock(resp) - ) - task_id = self._make_client().create_task("tl1", "Buy groceries") + client = self._make_client() + self._mock_http(client, self._make_mock(resp)) + task_id = client.create_task("tl1", "Buy groceries") assert task_id == "sv-task-1" - def test_create_task_passes_task_list_id(self, monkeypatch): + def test_create_task_passes_task_list_id(self): captured = {} resp = self._set_response(created={"new-0": {"id": "sv-task-1"}}) @@ -1995,71 +2187,74 @@ def capturing_post(*args, **kwargs): captured["json"] = kwargs.get("json", {}) return self._make_mock(resp) - monkeypatch.setattr("caldav.jmap.client.requests.post", capturing_post) - self._make_client().create_task("my-list", "Test Task") + client = self._make_client() + self._mock_http(client, side_effect=capturing_post) + client.create_task("my-list", "Test Task") create_args = captured["json"]["methodCalls"][0][1] assert create_args["create"]["new-0"]["taskListId"] == "my-list" - def test_create_task_raises_on_failure(self, monkeypatch): + def test_create_task_raises_on_failure(self): resp = self._set_response(notCreated={"new-0": {"type": "invalidArguments"}}) - monkeypatch.setattr( - "caldav.jmap.client.requests.post", lambda *a, **kw: self._make_mock(resp) - ) + client = self._make_client() + self._mock_http(client, self._make_mock(resp)) with pytest.raises(JMAPMethodError) as exc_info: - self._make_client().create_task("tl1", "Test") + client.create_task("tl1", "Test") assert exc_info.value.error_type == "invalidArguments" - def test_get_task_returns_task_object(self, monkeypatch): + def test_create_task_raises_jmap_error_when_created_is_empty(self): + """§1.13: create_task must raise JMAPMethodError (not KeyError) when the server + returns a Task/set response with an empty 'created' dict and no 'notCreated' entry.""" + resp = self._set_response(created={}, notCreated={}) + client = self._make_client() + self._mock_http(client, self._make_mock(resp)) + with pytest.raises(JMAPMethodError): + client.create_task("tl1", "Test") + + def test_get_task_returns_task_object(self): resp = self._get_response([self._MINIMAL_TASK]) - monkeypatch.setattr( - "caldav.jmap.client.requests.post", lambda *a, **kw: self._make_mock(resp) - ) - task = self._make_client().get_task("task1") + client = self._make_client() + self._mock_http(client, self._make_mock(resp)) + task = client.get_task("task1") assert isinstance(task, dict) assert task["id"] == "task1" - def test_get_task_raises_on_not_found(self, monkeypatch): + def test_get_task_raises_on_not_found(self): resp = self._get_response([]) - monkeypatch.setattr( - "caldav.jmap.client.requests.post", lambda *a, **kw: self._make_mock(resp) - ) + client = self._make_client() + self._mock_http(client, self._make_mock(resp)) with pytest.raises(JMAPMethodError) as exc_info: - self._make_client().get_task("missing") + client.get_task("missing") assert exc_info.value.error_type == "notFound" - def test_update_task_success(self, monkeypatch): + def test_update_task_success(self): resp = self._set_response(updated={"task1": None}) - monkeypatch.setattr( - "caldav.jmap.client.requests.post", lambda *a, **kw: self._make_mock(resp) - ) - self._make_client().update_task("task1", {"title": "Updated"}) + client = self._make_client() + self._mock_http(client, self._make_mock(resp)) + client.update_task("task1", {"title": "Updated"}) - def test_update_task_raises_on_failure(self, monkeypatch): + def test_update_task_raises_on_failure(self): resp = self._set_response(notUpdated={"task1": {"type": "notFound"}}) - monkeypatch.setattr( - "caldav.jmap.client.requests.post", lambda *a, **kw: self._make_mock(resp) - ) + client = self._make_client() + self._mock_http(client, self._make_mock(resp)) with pytest.raises(JMAPMethodError) as exc_info: - self._make_client().update_task("task1", {"title": "X"}) + client.update_task("task1", {"title": "X"}) assert exc_info.value.error_type == "notFound" - def test_delete_task_success(self, monkeypatch): + def test_delete_task_success(self): resp = self._set_response(destroyed=["task1"]) - monkeypatch.setattr( - "caldav.jmap.client.requests.post", lambda *a, **kw: self._make_mock(resp) - ) - self._make_client().delete_task("task1") + client = self._make_client() + self._mock_http(client, self._make_mock(resp)) + client.delete_task("task1") - def test_delete_task_raises_on_failure(self, monkeypatch): + def test_delete_task_raises_on_failure(self): resp = self._set_response(notDestroyed={"task1": {"type": "notFound"}}) - monkeypatch.setattr( - "caldav.jmap.client.requests.post", lambda *a, **kw: self._make_mock(resp) - ) + client = self._make_client() + self._mock_http(client, self._make_mock(resp)) with pytest.raises(JMAPMethodError) as exc_info: - self._make_client().delete_task("task1") + client.delete_task("task1") assert exc_info.value.error_type == "notFound" - def test_task_requests_use_task_capability(self, monkeypatch): + def test_task_requests_use_task_capability(self): captured = {} resp = self._tasklist_response([self._MINIMAL_TASKLIST]) @@ -2067,8 +2262,9 @@ def capturing_post(*args, **kwargs): captured["json"] = kwargs.get("json", {}) return self._make_mock(resp) - monkeypatch.setattr("caldav.jmap.client.requests.post", capturing_post) - self._make_client().get_task_lists() + client = self._make_client() + self._mock_http(client, side_effect=capturing_post) + client.get_task_lists() assert TASK_CAPABILITY in captured["json"]["using"] assert CALENDAR_CAPABILITY not in captured["json"]["using"] @@ -2223,6 +2419,33 @@ async def test_context_manager(self): async with AsyncJMAPClient(url=_JMAP_URL, username=_USERNAME, password=_PASSWORD) as client: assert isinstance(client, AsyncJMAPClient) + @pytest.mark.asyncio + async def test_context_manager_closes_http_session(self, monkeypatch): + mock_close = AsyncMock() + mock_http = MagicMock() + mock_http.close = mock_close + mock_http.headers = MagicMock() + monkeypatch.setattr("caldav.jmap.async_client.AsyncSession", lambda: mock_http) + client = AsyncJMAPClient(url=_JMAP_URL, username=_USERNAME, password=_PASSWORD) + async with client: + assert client._http_session is mock_http + mock_close.assert_called_once() + assert client._http_session is None + + @pytest.mark.asyncio + async def test_http_session_reused_across_requests(self, monkeypatch): + client = self._make_client() + mock_resp = self._make_mock_response({"methodResponses": []}) + mock_http = MagicMock() + mock_http.post = AsyncMock(return_value=mock_resp) + mock_http.headers = MagicMock() + with patch("caldav.jmap.async_client.AsyncSession") as MockAsyncSession: + MockAsyncSession.return_value = mock_http + await client._request([("Calendar/get", {}, "c0")]) + await client._request([("Calendar/get", {}, "c1")]) + MockAsyncSession.assert_called_once() + assert mock_http.post.call_count == 2 + @pytest.mark.asyncio async def test_get_calendars_returns_list(self, monkeypatch): cal = {"id": "cal1", "name": "Personal", "isSubscribed": True, "myRights": {}} @@ -2327,7 +2550,7 @@ async def test_get_objects_no_changes(self, monkeypatch): mock_http.__aexit__ = AsyncMock(return_value=None) mock_http.post = AsyncMock(return_value=self._make_mock_response(self._changes_resp())) monkeypatch.setattr("caldav.jmap.async_client.AsyncSession", lambda: mock_http) - added, modified, deleted = await self._make_client().get_objects_by_sync_token("state-1") + added, modified, deleted, _ = await self._make_client().get_objects_by_sync_token("state-1") assert added == [] and modified == [] and deleted == [] @pytest.mark.asyncio @@ -2339,7 +2562,7 @@ async def test_get_objects_deleted_returns_ids(self, monkeypatch): return_value=self._make_mock_response(self._changes_resp(destroyed=["ev1"])) ) monkeypatch.setattr("caldav.jmap.async_client.AsyncSession", lambda: mock_http) - added, modified, deleted = await self._make_client().get_objects_by_sync_token("state-1") + added, modified, deleted, _ = await self._make_client().get_objects_by_sync_token("state-1") assert deleted == ["ev1"] and added == [] and modified == [] @pytest.mark.asyncio @@ -2354,7 +2577,7 @@ async def test_get_objects_added_returns_ical(self, monkeypatch): ] ) monkeypatch.setattr("caldav.jmap.async_client.AsyncSession", lambda: mock_http) - added, modified, deleted = await self._make_client().get_objects_by_sync_token("state-1") + added, modified, deleted, _ = await self._make_client().get_objects_by_sync_token("state-1") assert len(added) == 1 assert isinstance(added[0], JMAPCalendarObject) assert added[0].id == "ev-async-1" @@ -2534,7 +2757,7 @@ async def test_get_objects_modified_returns_ical(self, monkeypatch): ] ) monkeypatch.setattr("caldav.jmap.async_client.AsyncSession", lambda: mock_http) - added, modified, deleted = await self._make_client().get_objects_by_sync_token("state-1") + added, modified, deleted, _ = await self._make_client().get_objects_by_sync_token("state-1") assert len(modified) == 1 assert isinstance(modified[0], JMAPCalendarObject) assert modified[0].id == "ev-async-1" @@ -2548,3 +2771,126 @@ async def test_create_task_passes_task_list_id(self, monkeypatch): create_args = captured["json"]["methodCalls"][0][1] new_task = create_args["create"]["new-0"] assert new_task["taskListId"] == "tl-target" + + +class TestOverrideWithoutStartUsesOccurrenceTime: + """§4.1: override child VEVENT must use occurrence time as DTSTART, not master start.""" + + def test_title_only_override_dtstart_equals_occurrence(self): + # Master: 2024-06-17T09:00:00Z (UTC), weekly recurrence. + # Override for 2024-06-24T09:00:00Z changes only title — no "start" in patch. + # Child DTSTART must be 20240624T090000Z, not 20240617T090000Z. + jscal = { + "uid": "override-dtstart@example.com", + "title": "Master Title", + "start": "2024-06-17T09:00:00Z", + "duration": "PT1H", + "recurrenceRules": [{"@type": "RecurrenceRule", "frequency": "weekly"}], + "recurrenceOverrides": { + "2024-06-24T09:00:00Z": {"title": "Changed Title"}, + }, + } + result = jscal_to_ical(jscal) + import icalendar as _ic + + cal = _ic.Calendar.from_ical(result) + events = [c for c in cal.subcomponents if isinstance(c, _ic.Event)] + assert len(events) == 2 + child = next(e for e in events if e.get("RECURRENCE-ID") is not None) + # DTSTART of the child must match its own occurrence, not the master start + child_dtstart = child["DTSTART"].dt + if hasattr(child_dtstart, "utctimetuple"): + import datetime as _dt + + assert child_dtstart == _dt.datetime(2024, 6, 24, 9, 0, 0, tzinfo=_dt.timezone.utc) + else: + assert str(child_dtstart) == "2024-06-24" + + +class TestExdateValueType: + """§4.2: EXDATE value type must match DTSTART (TZID or DATE, not floating).""" + + def test_exdate_for_tzid_event_has_tzid_param(self): + # A TZID-anchored event's excluded override must produce EXDATE with TZID, + # not a floating EXDATE (which per RFC 5545 won't match the instance). + jscal = _minimal_jscal( + start="2024-06-17T14:00:00", + timeZone="Europe/Berlin", + recurrenceRules=[{"@type": "RecurrenceRule", "frequency": "weekly"}], + recurrenceOverrides={"2024-06-24T14:00:00": {"excluded": True}}, + ) + result = jscal_to_ical(jscal) + # Must have TZID on EXDATE; a plain EXDATE:... without TZID is a floating datetime + assert "EXDATE;TZID=Europe/Berlin:" in result + + def test_exdate_for_allday_event_is_date_value(self): + jscal = { + "uid": "allday-exdate@example.com", + "title": "All Day Recurring", + "start": "2024-06-17T00:00:00", + "showWithoutTime": True, + "duration": "P1D", + "recurrenceRules": [{"@type": "RecurrenceRule", "frequency": "weekly"}], + "recurrenceOverrides": {"2024-06-24T00:00:00": {"excluded": True}}, + } + result = jscal_to_ical(jscal) + # All-day EXDATE must be a DATE value (8-digit YYYYMMDD, not YYYYMMDDTHHMMSS datetime). + # The icalendar library may or may not emit explicit VALUE=DATE — either form is acceptable. + assert "EXDATE" in result + assert "20240624" in result + assert "20240624T" not in result # must not be a datetime + + +class TestStatusMapping: + """§4.4: STATUS must be mapped in both ical→jscal and jscal→ical directions.""" + + def test_ical_status_cancelled_to_jscal(self): + ical = _make_ical( + "DTSTART:20240615T100000Z\r\nSUMMARY:Cancelled Meeting\r\nSTATUS:CANCELLED\r\n" + ) + result = ical_to_jscal(ical) + assert result.get("status") == "cancelled" + + def test_ical_status_tentative_to_jscal(self): + ical = _make_ical( + "DTSTART:20240615T100000Z\r\nSUMMARY:Tentative Meeting\r\nSTATUS:TENTATIVE\r\n" + ) + result = ical_to_jscal(ical) + assert result.get("status") == "tentative" + + def test_ical_status_confirmed_to_jscal(self): + ical = _make_ical( + "DTSTART:20240615T100000Z\r\nSUMMARY:Confirmed Meeting\r\nSTATUS:CONFIRMED\r\n" + ) + result = ical_to_jscal(ical) + assert result.get("status") == "confirmed" + + def test_ical_no_status_omits_jscal_status(self): + ical = _make_ical("DTSTART:20240615T100000Z\r\nSUMMARY:No Status\r\n") + result = ical_to_jscal(ical) + assert "status" not in result + + def test_jscal_status_cancelled_to_ical(self): + result = jscal_to_ical(_minimal_jscal(status="cancelled")) + assert "STATUS:CANCELLED" in result + + def test_jscal_status_tentative_to_ical(self): + result = jscal_to_ical(_minimal_jscal(status="tentative")) + assert "STATUS:TENTATIVE" in result + + def test_jscal_status_confirmed_to_ical(self): + result = jscal_to_ical(_minimal_jscal(status="confirmed")) + assert "STATUS:CONFIRMED" in result + + def test_jscal_no_status_omits_ical_status(self): + result = jscal_to_ical(_minimal_jscal()) + assert "STATUS:" not in result + + def test_status_cancelled_round_trips(self): + original = _make_ical( + "DTSTART:20240615T100000Z\r\nSUMMARY:Cancelled\r\nSTATUS:CANCELLED\r\n" + ) + jscal = ical_to_jscal(original) + assert jscal.get("status") == "cancelled" + round_tripped = jscal_to_ical(jscal) + assert "STATUS:CANCELLED" in round_tripped diff --git a/tests/test_protocol.py b/tests/test_protocol.py index c3a55375..9c8af001 100644 --- a/tests/test_protocol.py +++ b/tests/test_protocol.py @@ -208,6 +208,53 @@ def test_parse_sync_collection_response(self): assert result.deleted[0] == "/cal/deleted.ics" assert result.sync_token == "new-token" + def test_parse_sync_collection_generic_responsedescription(self): + """A 404 may carry an arbitrary . + + Per RFC 4918 is an optional child of + ; its text is server-defined. We must not hardcode + any particular server's wording (e.g. Stalwart's "No resources + found"). + """ + xml = b""" + + + /cal/gone.ics + HTTP/1.1 404 Not Found + The thing you asked for is not here anymore + + tok + """ + + result = DAVResponse.from_bytes(xml).parse_sync_collection() + + assert result.deleted == ["/cal/gone.ics"] + assert result.changed == [] + + def test_parse_sync_collection_generic_error(self): + """A 404 may carry an arbitrary element. + + Per RFC 4918 is an optional child of and its + children are server-defined. We must not hardcode any + particular server's error condition (e.g. purelymail's + {https://purelymail.com}does-not-exist). + """ + xml = b""" + + + /cal/gone.ics + HTTP/1.1 404 Not Found + + + tok + """ + + result = DAVResponse.from_bytes(xml).parse_sync_collection() + + assert result.deleted == ["/cal/gone.ics"] + assert result.changed == [] + def test_parse_complex_properties(self): """Parse complex properties like supported-calendar-component-set.""" xml = b""" @@ -252,3 +299,48 @@ def test_parse_complex_properties(self): # calendar-home-set - extracted href home_set = props["{urn:ietf:params:xml:ns:caldav}calendar-home-set"] assert home_set == "/calendars/user/" + + +class TestParserStackEquivalence: + """Guard the shared propstat-collection logic (code-review §5.7). + + The dataclass parsers (parse_propfind -> _extract_properties) and the + legacy _find_objects_and_props path must agree on the duplicated quirks + that used to be implemented twice: the "a 404 propstat means the property + is absent" skip and which prop elements get collected per href. + """ + + # one href with a found prop (200) and an absent prop (404 propstat), + # plus a second href that 404s entirely. + _xml = b""" + + + /cal/a/ + + A + HTTP/1.1 200 OK + + + + HTTP/1.1 404 Not Found + + + + /cal/missing/ + HTTP/1.1 404 Not Found + + """ + + def test_404_propstat_skipped_in_both_stacks(self): + dataclass_props = DAVResponse.from_bytes(self._xml).parse_propfind() + legacy = DAVResponse.from_bytes(self._xml)._find_objects_and_props() + + # dataclass stack: /cal/a/ keeps displayname, drops the 404 color prop + a_result = next(r for r in dataclass_props if r.href == "/cal/a/") + assert "{DAV:}displayname" in a_result.properties + assert "{http://apple.com/ns/ical/}calendar-color" not in a_result.properties + + # legacy stack: same set of collected prop tags for the same href + assert set(legacy["/cal/a/"].keys()) == set(a_result.properties.keys()) + # the entirely-404 href is present but carries no props in either stack + assert legacy["/cal/missing/"] == {} diff --git a/tests/test_schedule_tag.py b/tests/test_schedule_tag.py index c702b321..65526694 100644 --- a/tests/test_schedule_tag.py +++ b/tests/test_schedule_tag.py @@ -242,3 +242,30 @@ def test_schedule_tag_updated_in_props_after_successful_save(self, mocked): assert event.schedule_tag == new_tag, ( "schedule_tag prop not updated after successful conditional save" ) + + # ------------------------------------------------------------------ # + # 7. _post_put header handling (characterization for the dedup of # + # the block that used to be pasted twice) # + # ------------------------------------------------------------------ # + + @mock.patch("caldav.davclient.requests.Session.request") + def test_etag_captured_from_put_response(self, mocked): + """A PUT returning an Etag header must store it in props.""" + mocked.return_value = _make_put_response(201, {"Etag": '"etag-from-put"'}) + + event = _make_event_with_tag(None) + event.save() + + assert event.props[dav.GetEtag.tag] == '"etag-from-put"' + + @mock.patch("caldav.davclient.requests.Session.request") + def test_302_on_put_updates_url(self, mocked): + """A 302 in response to a PUT must follow the Location header.""" + mocked.return_value = _make_put_response( + 302, {"location": "http://cal.example.com/cal/moved.ics"} + ) + + event = _make_event_with_tag(None) + event.save() + + assert str(event.url) == "http://cal.example.com/cal/moved.ics" diff --git a/tests/test_search.py b/tests/test_search.py index 1dc7a3f2..b46a6b62 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -14,8 +14,9 @@ import icalendar import pytest -from caldav import Event, Journal, Todo +from caldav import Calendar, Event, Journal, Todo from caldav.davclient import DAVClient +from caldav.lib import error from caldav.lib.url import URL from caldav.search import CalDAVSearcher @@ -140,6 +141,31 @@ END:VEVENT END:VCALENDAR""" +# Two events for §2.6 combined-is-logical-and tests +SPECIAL_EVENT = """BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Test//Test//EN +BEGIN:VEVENT +UID:special-event@example.com +DTSTAMP:20240101T120000Z +DTSTART:20240615T140000Z +DTEND:20240615T150000Z +SUMMARY:My Special Event +END:VEVENT +END:VCALENDAR""" + +UNRELATED_EVENT = """BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Test//Test//EN +BEGIN:VEVENT +UID:unrelated-event@example.com +DTSTAMP:20240101T120000Z +DTSTART:20240615T160000Z +DTEND:20240615T170000Z +SUMMARY:Unrelated Meeting +END:VEVENT +END:VCALENDAR""" + @pytest.fixture def mock_client() -> DAVClient: @@ -867,3 +893,518 @@ def mock_is_supported(feat, type_=bool): assert result == [event] calendar._request_report_build_resultlist.assert_called_once_with(full_xml, None, None) + + +class TestCompTypeOptionalTimeRange: + """Regression tests for https://github.com/python-caldav/caldav/issues/681. + + A CALDAV:time-range filter is only valid inside a comp-filter for + VEVENT/VTODO/VJOURNAL/VFREEBUSY/VALARM (RFC 4791 section 9.7), never + directly under the VCALENDAR comp-filter. When no component type is + specified, the library must NOT emit a under VCALENDAR - + it must split the search into one query per component type instead. + + SabreDAV-based servers (Baikal, Nextcloud, ...) reject the illegal query + with HTTP 400 "You cannot add time-range filters on the VCALENDAR + component". + """ + + _NS = {"C": "urn:ietf:params:xml:ns:caldav"} + + def _vcalendar_timerange_children(self, xml): + """Return any elements that are direct children of the + VCALENDAR comp-filter (i.e. the RFC-illegal placement).""" + from lxml import etree + + x = xml.xmlelement() if hasattr(xml, "xmlelement") else None + if x is None: + return [] + return x.xpath('//C:comp-filter[@name="VCALENDAR"]/C:time-range', namespaces=self._NS) + + def test_untyped_timerange_search_splits_per_comptype( + self, mock_client: DAVClient, mock_url: str + ) -> None: + """Backward-compat mode: an untyped time-range search must split into + per-component queries rather than placing under VCALENDAR.""" + from caldav.compatibility_hints import FeatureSet + + mock_client.features = FeatureSet(None) # default / backward-compat (what end users get) + + calls = [] + calendar = mock.Mock() + calendar.client = mock_client + + def rep(xml, comp_cls, props=None): + calls.append(xml) + return (mock.Mock(), []) + + calendar._request_report_build_resultlist.side_effect = rep + + searcher = CalDAVSearcher( + start=datetime(2024, 1, 1, tzinfo=timezone.utc), + end=datetime(2024, 2, 1, tzinfo=timezone.utc), + ) + searcher.search(calendar) + + assert calls, "no REPORT was issued" + for xml in calls: + assert not self._vcalendar_timerange_children(xml), ( + "time-range must not be placed directly under VCALENDAR" + ) + ## split into one query per component type (VEVENT/VTODO/VJOURNAL) + assert len(calls) == 3 + + def test_reactive_workaround_on_vcalendar_timerange_rejection( + self, mock_client: DAVClient, mock_url: str + ) -> None: + """If the feature is (mis)configured as supported and the server rejects + the comp-type-less time-range query with a 400, the library must retry + by splitting into per-component queries.""" + from caldav.compatibility_hints import FeatureSet + + ## Feature explicitly configured as supported, so the library optimistically + ## sends the comp-type-less time-range query that SabreDAV rejects. + mock_client.features = FeatureSet( + {"search.time-range.comp-type-optional": {"support": "full"}} + ) + + event = Event(client=mock_client, url=mock_url, data=SIMPLE_EVENT) + calendar = mock.Mock() + calendar.client = mock_client + + def rep(xml, comp_cls, props=None): + if self._vcalendar_timerange_children(xml): + raise error.ReportError( + "400 Bad Request - You cannot add time-range filters on the VCALENDAR component" + ) + return (mock.Mock(), [event] if comp_cls is Event else []) + + calendar._request_report_build_resultlist.side_effect = rep + + searcher = CalDAVSearcher( + start=datetime(2024, 1, 1, tzinfo=timezone.utc), + end=datetime(2024, 2, 1, tzinfo=timezone.utc), + ) + result = searcher.search(calendar) + + assert result == [event] + + def test_compatibility_workarounds_false_sends_raw_query( + self, mock_client: DAVClient, mock_url: str + ) -> None: + """compatibility_workarounds=False must disable the comp-type split and send + the comp-type-less time-range query verbatim (single REPORT), so the + compatibility checker can observe the raw server behaviour.""" + from caldav.compatibility_hints import FeatureSet + + mock_client.features = FeatureSet(None) + + calls = [] + calendar = mock.Mock() + calendar.client = mock_client + + def rep(xml, comp_cls, props=None): + calls.append(xml) + return (mock.Mock(), []) + + calendar._request_report_build_resultlist.side_effect = rep + + searcher = CalDAVSearcher( + start=datetime(2024, 1, 1, tzinfo=timezone.utc), + end=datetime(2024, 2, 1, tzinfo=timezone.utc), + ) + searcher.search(calendar, post_filter=False, compatibility_workarounds=False) + + ## exactly one report, sent verbatim with the (RFC-questionable) time-range + ## directly under VCALENDAR - no splitting + assert len(calls) == 1 + assert self._vcalendar_timerange_children(calls[0]) + + +class TestCompTypeOptionalPropFilter: + """Regression tests for https://github.com/python-caldav/caldav/issues/681. + + A CALDAV:prop-filter (CATEGORIES, SUMMARY, ...) placed directly under the + VCALENDAR comp-filter filters on VCALENDAR's own properties, which do not + include component properties like CATEGORIES. Servers (e.g. Xandikos, and + SabreDAV-based servers) therefore match nothing. When no component type is + specified, the library must split the search into one query per component + type so the prop-filter lands inside a VEVENT/VTODO/VJOURNAL comp-filter. + """ + + _NS = {"C": "urn:ietf:params:xml:ns:caldav"} + + def _vcalendar_propfilter_children(self, xml): + """Return any elements that are direct children of the + VCALENDAR comp-filter (i.e. filtering on a non-existent VCALENDAR prop).""" + x = xml.xmlelement() if hasattr(xml, "xmlelement") else None + if x is None: + return [] + return x.xpath('//C:comp-filter[@name="VCALENDAR"]/C:prop-filter', namespaces=self._NS) + + def test_untyped_propfilter_search_splits_per_comptype( + self, mock_client: DAVClient, mock_url: str + ) -> None: + """Backward-compat mode: an untyped property-filter search must split into + per-component queries rather than placing under VCALENDAR.""" + from caldav.compatibility_hints import FeatureSet + + mock_client.features = FeatureSet(None) # default / backward-compat + + calls = [] + calendar = mock.Mock() + calendar.client = mock_client + + def rep(xml, comp_cls, props=None): + calls.append(xml) + return (mock.Mock(), []) + + calendar._request_report_build_resultlist.side_effect = rep + + searcher = CalDAVSearcher() + searcher.add_property_filter("SUMMARY", "meeting") + searcher.search(calendar) + + assert calls, "no REPORT was issued" + for xml in calls: + assert not self._vcalendar_propfilter_children(xml), ( + "prop-filter must not be placed directly under VCALENDAR" + ) + ## split into one query per component type (VEVENT/VTODO/VJOURNAL) + assert len(calls) == 3 + + +class TestSearchDriverExceptionHandling: + """The search() driver runs the generator's yielded actions and must feed any + exception raised by an action back INTO the generator (via gen.throw()) so the + search logic's own try/except blocks can act on it. Without this, the + generator's error-handling branches (issue #681 fallback, per-object load + error handling, ...) would be dead code. + """ + + def _mock_features_all_supported(self, mock_client): + def mock_is_supported(feat, type_=bool): + if type_ is str: + return "full" + return True + + mock_client.features.is_supported = mock.Mock(side_effect=mock_is_supported) + mock_client.features.backward_compatibility_mode = False + + def test_unloaded_object_not_returned_by_batch_is_excluded( + self, mock_client: DAVClient, mock_url: str + ) -> None: + """Objects that remain unloaded after _batch_load_objects are excluded from results. + + With the old per-object LOAD_OBJECT loop, exceptions were thrown into the generator + to skip objects. With the new LOAD_OBJECTS_BATCH approach, _batch_load_objects + handles errors internally; objects it cannot populate remain unloaded and are + filtered out by the post-batch is_loaded()/has_component() check. + """ + self._mock_features_all_supported(mock_client) + + good = Event(client=mock_client, url=mock_url + "/good", data=SIMPLE_EVENT) + bad = Event(client=mock_client, url=mock_url + "/bad") # unloaded: server skipped it + + calendar = mock.Mock() + calendar.client = mock_client + calendar._request_report_build_resultlist.return_value = (mock.Mock(), [good, bad]) + # Default mock._batch_load_objects does nothing: bad remains unloaded + + searcher = CalDAVSearcher(event=True) + result = searcher.search(calendar) + + assert good in result + assert bad not in result + + def test_unhandled_action_exception_propagates( + self, mock_client: DAVClient, mock_url: str + ) -> None: + """An exception the generator does NOT catch must still propagate out of + search() (gen.throw re-raises it) rather than being swallowed.""" + self._mock_features_all_supported(mock_client) + + calendar = mock.Mock() + calendar.client = mock_client + calendar._request_report_build_resultlist.side_effect = RuntimeError("boom") + + searcher = CalDAVSearcher(event=True) + with pytest.raises(RuntimeError, match="boom"): + searcher.search(calendar) + + +class TestCombinedIsLogicalAndWorkaround: + """§2.6: combined-is-logical-and workaround must apply property filters client-side. + + When search.combined-is-logical-and is False, the workaround strips all + property filters from the server query (sending only the time range) and + must apply them client-side afterward. The bug passed the ambient + post_filter=None instead of True, so _filter_search_results short-circuited + and returned all objects in the time range unfiltered. + """ + + def _make_calendar_with_features(self) -> "tuple": + """Return (client, calendar) with combined-is-logical-and: unsupported.""" + from caldav import Calendar + from caldav.compatibility_hints import FeatureSet + + features = FeatureSet( + { + "search.combined-is-logical-and": "unsupported", + "search.text": "full", + "search.text.substring": "full", + "search.text.case-sensitive": "full", + "search.time-range.accurate": "full", + "search.unlimited-time-range": "full", + } + ) + from caldav.davclient import DAVClient + + client = DAVClient(url="https://cal.example.com/") + client.features = features + cal = Calendar(client=client, url="https://cal.example.com/cal/") + return client, cal + + def test_summary_filter_applied_client_side(self) -> None: + """Time-range + SUMMARY filter on combined-is-logical-and:unsupported server + must return only events whose SUMMARY matches — not every event in the range.""" + client, cal = self._make_calendar_with_features() + + special = Event( + client=client, + url="https://cal.example.com/cal/special.ics", + data=SPECIAL_EVENT, + parent=cal, + ) + unrelated = Event( + client=client, + url="https://cal.example.com/cal/unrelated.ics", + data=UNRELATED_EVENT, + parent=cal, + ) + + mock_response = mock.MagicMock() + cal._request_report_build_resultlist = mock.Mock( + return_value=(mock_response, [special, unrelated]) + ) + + start = datetime(2024, 6, 15, 0, 0, tzinfo=timezone.utc) + end = datetime(2024, 6, 16, 0, 0, tzinfo=timezone.utc) + searcher = CalDAVSearcher(event=True, start=start, end=end) + searcher.add_property_filter("SUMMARY", "Special", operator="contains") + + results = searcher.search(cal) + + summaries = [str(r.icalendar_component["SUMMARY"]) for r in results] + assert len(results) == 1, f"Expected 1 result, got {len(results)}: {summaries}" + assert "Special" in summaries[0] + + +class TestExactMatchOperator: + """§2.8: operator='==' must be enforced client-side via post-filtering. + + The docstring documents that '==' means exact match enforced client-side, + but no code path inspected the '==' operator — post_filter was never set + for '==' searches, so server substring semantics leaked through. + """ + + _exact_match_event = """BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Test//Test//EN +BEGIN:VEVENT +UID:exact-event@example.com +DTSTAMP:20240101T120000Z +DTSTART:20240615T140000Z +DTEND:20240615T150000Z +SUMMARY:rain +END:VEVENT +END:VCALENDAR""" + + _substring_event = """BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Test//Test//EN +BEGIN:VEVENT +UID:substring-event@example.com +DTSTAMP:20240101T120000Z +DTSTART:20240615T160000Z +DTEND:20240615T170000Z +SUMMARY:Training session +END:VEVENT +END:VCALENDAR""" + + def _make_calendar_with_features(self): + from caldav.lib.url import URL + + client = mock.Mock(spec=DAVClient) + client.url = URL("https://cal.example.com/") + features = mock.Mock() + features.is_supported = mock.Mock(return_value=False) + client.features = features + cal = mock.Mock() + cal.client = client + cal.url = URL("https://cal.example.com/cal/") + return client, cal + + def test_exact_match_excludes_substrings(self): + """operator='==' must exclude events where the value is a substring of the summary.""" + client, cal = self._make_calendar_with_features() + + exact_ev = Event( + client=client, + url="https://cal.example.com/cal/exact.ics", + data=self._exact_match_event, + parent=cal, + ) + substring_ev = Event( + client=client, + url="https://cal.example.com/cal/substring.ics", + data=self._substring_event, + parent=cal, + ) + + cal._request_report_build_resultlist = mock.Mock( + return_value=(mock.MagicMock(), [exact_ev, substring_ev]) + ) + + searcher = CalDAVSearcher(event=True) + searcher.add_property_filter("SUMMARY", "rain", operator="==") + results = searcher.search(cal) + + summaries = [str(r.icalendar_component["SUMMARY"]) for r in results] + assert len(results) == 1, f"Expected 1 result (exact), got {len(results)}: {summaries}" + assert results[0].icalendar_component["SUMMARY"] == "rain" + + +class TestBatchLoadObjects: + """Calendar._batch_load_objects fetches N objects with one _multiget REPORT. + + Replaces the per-object LOAD_OBJECT loop in search post-processing (issue #5.4). + Before this fix, a 200-event search triggered 200 individual GET requests; + after, one batched calendar-multiget REPORT is sent. + """ + + CAL_URL = "https://cal.example.com/cal/" + + def _make_calendar(self) -> Calendar: + client = mock.Mock(spec=DAVClient) + client.url = URL("https://cal.example.com/") + return Calendar(client=client, url=self.CAL_URL) + + def test_batch_load_calls_multiget_once(self) -> None: + """_batch_load_objects calls _multiget exactly once for N unloaded objects.""" + cal = self._make_calendar() + ev1 = Event(client=cal.client, url=self.CAL_URL + "ev1.ics") + ev2 = Event(client=cal.client, url=self.CAL_URL + "ev2.ics") + + cal._multiget = mock.Mock( + return_value=iter([("/cal/ev1.ics", SIMPLE_EVENT), ("/cal/ev2.ics", SIMPLE_EVENT)]) + ) + + cal._batch_load_objects([ev1, ev2]) + + assert cal._multiget.call_count == 1 + + def test_batch_load_populates_object_data(self) -> None: + """_batch_load_objects sets data on matched unloaded objects.""" + cal = self._make_calendar() + ev = Event(client=cal.client, url=self.CAL_URL + "ev1.ics") + assert not ev.is_loaded() + + cal._multiget = mock.Mock(return_value=iter([("/cal/ev1.ics", SIMPLE_EVENT)])) + + cal._batch_load_objects([ev]) + + assert ev.is_loaded() + + def test_batch_load_skips_already_loaded_in_multiget_request(self) -> None: + """Already-loaded objects are not included in the multiget URL list.""" + cal = self._make_calendar() + loaded = Event(client=cal.client, url=self.CAL_URL + "loaded.ics", data=SIMPLE_EVENT) + unloaded = Event(client=cal.client, url=self.CAL_URL + "unloaded.ics") + + cal._multiget = mock.Mock(return_value=iter([("/cal/unloaded.ics", SIMPLE_EVENT)])) + + cal._batch_load_objects([loaded, unloaded]) + + assert cal._multiget.called + urls_passed = [str(u) for u in cal._multiget.call_args[0][0]] + assert not any("loaded.ics" in u and "unloaded" not in u for u in urls_passed) + + def test_batch_load_fallback_on_multiget_error(self) -> None: + """When _multiget raises, _batch_load_objects falls back to individual obj.load().""" + cal = self._make_calendar() + ev = Event(client=cal.client, url=self.CAL_URL + "ev1.ics") + ev.load = mock.Mock() + cal._multiget = mock.Mock(side_effect=RuntimeError("network error")) + + cal._batch_load_objects([ev]) + + ev.load.assert_called() + + +class TestSearchBatchLoadIntegration: + """search() must use one batched _multiget call instead of N individual load() calls. + + The fix replaces the per-object LOAD_OBJECT loops in _search_impl with a single + LOAD_OBJECTS_BATCH action that delegates to Calendar._batch_load_objects. + """ + + def _make_calendar_all_supported(self) -> "tuple[mock.Mock, Calendar]": + """Return (client, calendar) with all features marked supported.""" + client = mock.Mock(spec=DAVClient) + client.url = URL("https://cal.example.com/") + + def mock_is_supported(feat: str, type_: type = bool): + return "full" if type_ is str else True + + client.features.is_supported = mock.Mock(side_effect=mock_is_supported) + client.features.backward_compatibility_mode = False + + cal = Calendar(client=client, url="https://cal.example.com/cal/") + return client, cal + + def test_search_calls_multiget_once_not_n_individual_loads(self) -> None: + """For N unloaded search results, search() must call _multiget once, not load() N times. + + With the old per-object LOAD_OBJECT loop, 2 unloaded objects caused 2 GET requests. + With the new LOAD_OBJECTS_BATCH action, a single calendar-multiget REPORT is issued. + """ + client, cal = self._make_calendar_all_supported() + + ev1 = Event(client=client, url="https://cal.example.com/cal/ev1.ics", parent=cal) + ev2 = Event(client=client, url="https://cal.example.com/cal/ev2.ics", parent=cal) + ev1.load = mock.Mock(return_value=ev1) + ev2.load = mock.Mock(return_value=ev2) + + cal._request_report_build_resultlist = mock.Mock(return_value=(mock.Mock(), [ev1, ev2])) + cal._multiget = mock.Mock( + return_value=iter( + [ + ("/cal/ev1.ics", SIMPLE_EVENT), + ("/cal/ev2.ics", SIMPLE_EVENT), + ] + ) + ) + + searcher = CalDAVSearcher(event=True) + searcher.search(cal) + + assert cal._multiget.call_count == 1, ( + f"Expected one batched _multiget call, got {cal._multiget.call_count}. " + "search() is still loading unloaded objects one-by-one." + ) + + def test_search_results_populated_after_batch_load(self) -> None: + """search() returns populated objects when batch-loading succeeds.""" + client, cal = self._make_calendar_all_supported() + + ev = Event(client=client, url="https://cal.example.com/cal/ev1.ics", parent=cal) + ev.load = mock.Mock(return_value=ev) + + cal._request_report_build_resultlist = mock.Mock(return_value=(mock.Mock(), [ev])) + cal._multiget = mock.Mock(return_value=iter([("/cal/ev1.ics", SIMPLE_EVENT)])) + + searcher = CalDAVSearcher(event=True) + results = searcher.search(cal) + + assert len(results) == 1 diff --git a/tests/test_servers.yaml.example b/tests/test_servers.yaml.example deleted file mode 100644 index c07af076..00000000 --- a/tests/test_servers.yaml.example +++ /dev/null @@ -1,138 +0,0 @@ -# Test server configuration for caldav tests -# -# Copy this file to test_servers.yaml and customize for your setup. -# See tests/README.md for documentation. -# -# Environment variables can be used with ${VAR} or ${VAR:-default} syntax. - -test-servers: - # ========================================================================= - # Embedded servers (run in-process, no external setup required) - # ========================================================================= - - radicale: - type: embedded - enabled: true - host: ${RADICALE_HOST:-localhost} - port: ${RADICALE_PORT:-5232} - username: user1 - password: "" - - xandikos: - type: embedded - enabled: true - host: ${XANDIKOS_HOST:-localhost} - port: ${XANDIKOS_PORT:-8993} - username: sometestuser - password: "" - - # ========================================================================= - # Docker servers (require docker-compose, see docker-test-servers/) - # ========================================================================= - # - # Set enabled to: - # - true: always enable - # - false: always disable - # - "auto": enable if docker is available (default for docker servers) - - baikal: - type: docker - enabled: ${TEST_BAIKAL:-auto} - host: ${BAIKAL_HOST:-localhost} - port: ${BAIKAL_PORT:-8800} - username: ${BAIKAL_USERNAME:-testuser} - password: ${BAIKAL_PASSWORD:-testpass} - # Path within the CalDAV server - # path: /dav.php - - nextcloud: - type: docker - enabled: ${TEST_NEXTCLOUD:-false} - host: ${NEXTCLOUD_HOST:-localhost} - port: ${NEXTCLOUD_PORT:-8801} - username: ${NEXTCLOUD_USERNAME:-testuser} - password: ${NEXTCLOUD_PASSWORD:-testpass} - - cyrus: - type: docker - enabled: ${TEST_CYRUS:-false} - host: ${CYRUS_HOST:-localhost} - port: ${CYRUS_PORT:-8802} - username: ${CYRUS_USERNAME:-testuser@test.local} - password: ${CYRUS_PASSWORD:-testpassword} - - sogo: - type: docker - enabled: ${TEST_SOGO:-false} - host: ${SOGO_HOST:-localhost} - port: ${SOGO_PORT:-8803} - username: ${SOGO_USERNAME:-testuser} - password: ${SOGO_PASSWORD:-testpassword} - - bedework: - type: docker - enabled: ${TEST_BEDEWORK:-false} - host: ${BEDEWORK_HOST:-localhost} - port: ${BEDEWORK_PORT:-8804} - username: ${BEDEWORK_USERNAME:-admin} - password: ${BEDEWORK_PASSWORD:-bedework} - - davical: - type: docker - enabled: ${TEST_DAVICAL:-false} - host: ${DAVICAL_HOST:-localhost} - port: ${DAVICAL_PORT:-8805} - username: ${DAVICAL_USERNAME:-admin} - password: ${DAVICAL_PASSWORD:-davical} - - # ========================================================================= - # External/private servers (your own CalDAV server) - # ========================================================================= - # - # Uncomment and configure to test against your own server: - - # my-server: - # type: external - # enabled: true - # url: ${CALDAV_URL:-https://caldav.example.com/dav/} - # username: ${CALDAV_USERNAME} - # password: ${CALDAV_PASSWORD} - # # Optional: SSL verification (default: true) - # ssl_verify: true - # # Optional: specify server limitations/features - # features: - # - no-expand # Server doesn't support EXPAND - # - no-sync-token # Server doesn't support sync tokens - # - no-freebusy # Server doesn't support freebusy queries - -# ========================================================================= -# RFC6638 scheduling test users (optional) -# ========================================================================= -# -# For testing calendar scheduling (meeting invites, etc.), define at least -# three users on the same CalDAV server that can send invites to each other. -# This section lives at the TOP LEVEL (not under test-servers). -# -# Cyrus (pre-creates user1-user5 with password 'x'): -# rfc6638_users: -# - url: http://localhost:8802/dav/calendars/user/user1 -# username: user1 -# password: x -# - url: http://localhost:8802/dav/calendars/user/user2 -# username: user2 -# password: x -# - url: http://localhost:8802/dav/calendars/user/user3 -# username: user3 -# password: x -# -# Baikal (user1-user3 are in the pre-seeded db.sqlite, passwords testpass1-3): -# rfc6638_users: -# - url: http://localhost:8800/dav.php/ -# username: user1 -# password: testpass1 -# - url: http://localhost:8800/dav.php/ -# username: user2 -# password: testpass2 -# - url: http://localhost:8800/dav.php/ -# username: user3 -# password: testpass3 diff --git a/tests/test_vcal.py b/tests/test_vcal.py index 4593864a..cb31e405 100644 --- a/tests/test_vcal.py +++ b/tests/test_vcal.py @@ -131,6 +131,23 @@ def create_and_validate(**args): ) assert re.search(b"DTSTART(;VALUE=DATE-TIME)?:20321010T101010Z", some_ical) + ## ical_fragment with alarm_* props: fragment must land in VEVENT, not VALARM (§2.2) + raw_ical = create_ical( + summary="alarm-test", + dtstart=datetime(2032, 10, 10, 10, 10, 10, tzinfo=utc), + duration=timedelta(hours=1), + alarm_action="DISPLAY", + alarm_description="reminder", + alarm_trigger=timedelta(minutes=-15), + ical_fragment="RRULE:FREQ=DAILY;COUNT=3", + ) + raw_bytes = to_wire(raw_ical) + assert b"RRULE:FREQ=DAILY" in raw_bytes, "ical_fragment must appear in output" + assert b"BEGIN:VALARM" in raw_bytes, "alarm must be present" + end_valarm_pos = raw_bytes.index(b"END:VALARM") + rrule_pos = raw_bytes.index(b"RRULE:FREQ=DAILY") + assert rrule_pos > end_valarm_pos, "RRULE must not be inside VALARM" + def test_vcal_fixups(self): """ There is an obscure function lib.vcal that attempts to fix up @@ -281,6 +298,76 @@ def test_vcal_fixups(self): for ical in non_broken_ical: assert vcal.fix(ical) == ical + def test_trailing_whitespace_stripped_per_line(self) -> None: + """Bug §2.3: re.sub(' *$', '', fixed) without re.MULTILINE only strips + trailing spaces at the very end of the document, leaving per-line + trailing spaces intact (e.g. iCloud X-APPLE-STRUCTURED-LOCATION fold + lines with trailing spaces that distort base64 content).""" + ical = ( + "BEGIN:VCALENDAR\n" + "VERSION:2.0\n" + "BEGIN:VEVENT\n" + "UID:test\n" + "DTSTAMP:20190103T070319Z\n" + "DTSTART:20190117T180000Z\n" + "SUMMARY:test\n" + "X-APPLE-STRUCTURED-LOCATION;X-TITLE=Somewhere:CAESvAEaEgmX \n" + " 5esy/OVJQBGXkXpP5aQYQCJi=\n" + "END:VEVENT\n" + "END:VCALENDAR\n" + ) + + fixed = vcal.fix(ical) + assert not re.search(r" +\n", fixed), ( + "fix() must strip trailing spaces from each line, not just the document end" + ) + + def test_backslash_unescape_single_and_double_quotes(self) -> None: + """Bug §2.4: re.sub(r"\\+('\")", r"\1", fixed) used a group ('\"') + which matches only the literal two-char sequence '\" — not a character + class. Backslash before a lone single quote or lone double quote was + therefore not unescaped.""" + ical_single = ( + "BEGIN:VCALENDAR\n" + "VERSION:2.0\n" + "BEGIN:VEVENT\n" + "UID:test\n" + "DTSTAMP:20190103T070319Z\n" + "DTSTART:20190117T180000Z\n" + "SUMMARY:it\\'s here\n" + "END:VEVENT\n" + "END:VCALENDAR\n" + ) + ical_double = ical_single.replace("\\'", '\\"') + fixed_single = vcal.fix(ical_single) + fixed_double = vcal.fix(ical_double) + assert "SUMMARY:it's here" in fixed_single, "fix() must strip backslash before single quote" + assert 'SUMMARY:it"s here' in fixed_double, "fix() must strip backslash before double quote" + + def test_completed_date_fixup_preserves_next_property(self) -> None: + """Bug §2.1: COMPLETED date fixup regex consumed the trailing newline, + merging the next property line into COMPLETED and destroying it.""" + ical = """BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Example Corp.//CalDAV Client//EN +BEGIN:VTODO +UID:20070313T123432Z-456553@example.com +DTSTAMP:20070313T123432Z +COMPLETED:20070501 +SUMMARY:Submit Quebec Income Tax Return for 2006 +STATUS:NEEDS-ACTION +END:VTODO +END:VCALENDAR""" + fixed = vcal.fix(ical) + cal = icalendar.Calendar.from_ical(fixed) + todo = list(cal.walk("VTODO"))[0] + assert str(todo["SUMMARY"]) == "Submit Quebec Income Tax Return for 2006", ( + "SUMMARY was destroyed by COMPLETED fixup (newline consumed)" + ) + assert "SUMMARY" not in str(todo["COMPLETED"].dt), ( + "COMPLETED value should not contain SUMMARY text" + ) + def test_missing_dtstamp_fix(self) -> None: """ Test that missing DTSTAMP is added by the fix function. @@ -355,3 +442,14 @@ def test_missing_dtstamp_fix(self) -> None: # Verify the fixed ical is valid self.verifyICal(fixed) + + def test_fix_does_not_crash_on_truncated_input(self) -> None: + """§1.12: vcal.fix() must not raise AssertionError on truncated/garbage iCalendar. + + Truncated data (no END: line) previously triggered a bare assert on line 93 + which gave no useful error message and failed silently under python -O. + """ + truncated = "BEGIN:VCALENDAR\nVERSION:2.0\nBEGIN:VEVENT\nUID:trunc@example.com\n" + # Must not raise — return something (possibly unchanged input) + result = vcal.fix(truncated) + assert result is not None diff --git a/tox.ini b/tox.ini index a8131b94..16eb1fae 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,8 @@ -[tox:tox] -envlist = y39,py310,py311,py312,py313,py314,docs,style,deptry +## [tox:tox] is the section name to use when the configuration lives in +## setup.cfg; in a tox.ini it has to be [tox], or the settings below are +## silently ignored (which they were, until 2026-08). +[tox] +envlist = py310,py311,py312,py313,py314,docs,style,deptry,audit [testenv] deps = --editable .[test] @@ -45,6 +48,27 @@ basepython = python3.13 deps = --editable .[test] commands = deptry caldav --known-first-party caldav +[testenv:audit] +## Audits the resolved runtime dependency tree against the PyPI advisory +## database. We declare open-ended version ranges, so a vulnerable release is +## normally resolved away by itself - the value here is catching the case where +## one of our *upper* bounds (currently only icalendar-searcher<2) would hold +## users back on a release with a known vulnerability. Test-only +## dependencies are deliberately not covered: pip-audit reads pyproject.toml +## metadata and skips the optional extras. +## +## No basepython pin: the audit is essentially interpreter-independent, and +## pinning would make this env unrunnable on machines lacking that version. +## Needs network access (PyPI advisory lookups). The socket timeout is raised +## from the 15s default because the advisory lookups are one request per +## dependency and a single slow response is enough to fail the whole run. +skip_install = true +deps = pip-audit +## A dedicated HTTP cache dir is used because pip-audit otherwise shares pip's +## cache, where it hits entries it cannot deserialize and logs a screenful of +## warnings on every run - noise is the enemy of a job nobody looks at. +commands = pip-audit --strict --desc on --timeout 30 --cache-dir {toxworkdir}/pip-audit-cache {toxinidir} + [build_sphinx] source-dir = docs/source build-dir = docs/build