diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml new file mode 100644 index 00000000..08b6a0b0 --- /dev/null +++ b/.github/workflows/audit.yml @@ -0,0 +1,48 @@ +--- +name: audit + +# Dependency vulnerability audit (pip-audit, see the `audit` env in tox.ini). +# +# This is deliberately *not* wired into the `tests` workflow: a new advisory +# can be published against an unchanged dependency tree, so the audit is +# time-triggered rather than change-triggered. The pull_request trigger is +# narrowed to the files that can change the dependency tree. +on: + pull_request: + paths: + - pyproject.toml + - tox.ini + - .github/workflows/audit.yml + workflow_dispatch: + schedule: + # Mondays 04:17 UTC, well clear of the nightly link check (22:03). + - cron: "17 4 * * 1" + +# Least privilege for GITHUB_TOKEN: this workflow only reads the repository. +# Flagged by CodeQL, see https://github.com/python-caldav/caldav/pull/694 +permissions: + contents: read + +concurrency: + group: audit-${{ github.ref }} + cancel-in-progress: false + +jobs: + pip-audit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + # hatch-vcs derives the version from git tags; without them the + # project metadata pip-audit reads cannot be built. + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + - uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: pip|${{ hashFiles('pyproject.toml') }}|${{ hashFiles('tox.ini') }} + - run: pip install tox + - name: Audit dependencies for known vulnerabilities + run: tox -e audit diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml index 061798b9..9a1a1717 100644 --- a/.github/workflows/linkcheck.yml +++ b/.github/workflows/linkcheck.yml @@ -7,6 +7,10 @@ on: schedule: - cron: "03 22 * * *" +concurrency: + group: linkcheck-${{ github.ref }} + cancel-in-progress: false + jobs: linkcheck: runs-on: ubuntu-latest @@ -14,6 +18,12 @@ jobs: issues: write steps: - uses: actions/checkout@v5 + - name: Restore lychee cache + uses: actions/cache@v4 + with: + path: .lycheecache + key: cache-lychee-${{ github.run_id }} + restore-keys: cache-lychee- - name: Check links with Lychee id: lychee uses: lycheeverse/lychee-action@v2 @@ -21,15 +31,41 @@ jobs: fail: false args: >- --root-dir "$(pwd)" - --timeout 20 - --max-retries 3 + --timeout 30 + --max-retries 6 + --retry-wait-time 2 --cache --max-cache-age 14d . - - name: Create Issue From File - if: steps.lychee.outputs.exit_code != 0 - uses: peter-evans/create-issue-from-file@v5 - with: - title: Link Checker Report - content-filepath: ./lychee/out.md - labels: report, automated issue + # The exit_code comparisons are quoted on purpose. A missing output is the + # empty string, and GitHub coerces '' to 0 when comparing against a number - + # so an unquoted `== 0` would treat "lychee did not run" as "all links are + # healthy" and close every open report. Comparing two strings does no + # coercion. + - name: Create or update Link Checker issue + if: steps.lychee.outputs.exit_code != '' && steps.lychee.outputs.exit_code != '0' && github.ref == 'refs/heads/master' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Keep the oldest open report issue as canonical, fold duplicates in. + # Both labels are required: `gh issue list` ANDs them, so a + # human-written issue that merely carries "report" cannot become the + # canonical one and have its body overwritten by lychee output. + ISSUES=$(gh issue list --label "report" --label "automated issue" --state open --json number --jq '.[].number' | sort -n) + CANON=$(printf '%s\n' "$ISSUES" | head -1) + for n in $(printf '%s\n' "$ISSUES" | tail -n +2); do + gh issue close "$n" --comment "Duplicate of #${CANON} - auto-closed by the link checker." + done + if [ -n "$CANON" ]; then + gh issue edit "$CANON" --body-file ./lychee/out.md + else + gh issue create --title "Link Checker Report" --body-file ./lychee/out.md --label "report" --label "automated issue" + fi + - name: Close Link Checker issue if all links are healthy + if: steps.lychee.outputs.exit_code == '0' && github.ref == 'refs/heads/master' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + for n in $(gh issue list --label "report" --label "automated issue" --state open --json number --jq '.[].number'); do + gh issue close "$n" --comment "All links are now healthy." + done diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml new file mode 100644 index 00000000..630ae472 --- /dev/null +++ b/.github/workflows/package.yml @@ -0,0 +1,62 @@ +--- +name: package + +# Builds the release artifacts and verifies their contents. +# +# Nothing in CI used to build an sdist at all, so what actually went into a +# release was only ever discovered after it was published: caldav-3.2.1.tar.gz +# shipped .claude/settings.json and 1755 files under venv/. The check runs on +# every change to the packaging configuration, and nightly, so a stray file in +# a contributor's tree cannot ride along into a tarball unnoticed. +# +# See tests/tools/check_dist.py for what is verified. +on: + push: + branches: + - master + pull_request: + paths: + - pyproject.toml + - tox.ini + - MANIFEST.in + - .gitignore + - tests/tools/check_dist.py + - .github/workflows/package.yml + workflow_dispatch: + schedule: + # Sundays 05:23 UTC, clear of the Monday audit (04:17) and the nightly + # link check (22:03). + - cron: "23 5 * * 0" + +concurrency: + group: package-${{ github.ref }} + cancel-in-progress: true + +# Least privilege for GITHUB_TOKEN: this workflow only reads the repository. +# Flagged by CodeQL, see https://github.com/python-caldav/caldav/pull/694 +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + # hatch-vcs derives the version from git tags. + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + - uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: pip|${{ hashFiles('pyproject.toml') }}|${{ hashFiles('tox.ini') }} + - run: pip install tox + - name: Build sdist and wheel, and check what is in them + run: tox -e package + - uses: actions/upload-artifact@v4 + with: + name: dist + path: .tox/package/tmp/dist/* + if-no-files-found: error diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 778054ad..24cc7e2d 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -97,7 +97,7 @@ jobs: - uses: actions/cache@v4 with: path: ~/.cache/pip - key: pip|${{ hashFiles('setup.py') }}|${{ hashFiles('tox.ini') }} + key: pip|${{ hashFiles('pyproject.toml') }}|${{ hashFiles('tox.ini') }} - run: pip install tox - name: Configure Baikal with pre-seeded database run: | @@ -183,13 +183,28 @@ jobs: docker exec ${{ job.services.nextcloud.id }} php occ config:system:set ratelimit.whitelist.0 --value='172.17.0.0/16' || true docker exec ${{ job.services.nextcloud.id }} php occ config:system:set ratelimit.whitelist.1 --value='127.0.0.1' || true - # Clear rate limit cache - docker exec ${{ job.services.nextcloud.id }} php -r " - \$db = new PDO('sqlite:/var/www/html/data/nextcloud.db'); - \$db->exec('DELETE FROM oc_ratelimit_entries'); - \$db->exec('DELETE FROM oc_bruteforce_attempts'); - echo 'Cleared rate limit and bruteforce caches\n'; - " || true + # Clear rate limit cache. The SQLite file is named after the `dbname` + # config value, which defaults to `owncloud` — look it up rather than + # guessing, and check it exists first: PDO creates a missing SQLite + # file, so a wrong path silently yields "no such table" for every + # DELETE below. + DB_NAME=$(docker exec ${{ job.services.nextcloud.id }} php occ config:system:get dbname 2>/dev/null | tr -d '\r\n') + DB_PATH="/var/www/html/data/${DB_NAME:-owncloud}.db" + if docker exec ${{ job.services.nextcloud.id }} test -f "$DB_PATH"; then + docker exec ${{ job.services.nextcloud.id }} php -r " + \$db = new PDO('sqlite:$DB_PATH'); + foreach (['oc_ratelimit_entries', 'oc_bruteforce_attempts'] as \$table) { + try { + \$db->exec(\"DELETE FROM \$table\"); + } catch (PDOException \$e) { + fwrite(STDERR, \"skipping \$table: \" . \$e->getMessage() . \"\n\"); + } + } + echo \"Cleared rate limit and bruteforce caches\n\"; + " || true + else + echo "No database found at $DB_PATH — skipping cache cleanup" + fi echo "Nextcloud is configured!" - name: Configure Cyrus @@ -326,7 +341,7 @@ jobs: - uses: actions/cache@v4 with: path: ~/.cache/pip - key: pip|${{ hashFiles('setup.py') }}|${{ hashFiles('tox.ini') }} + key: pip|${{ hashFiles('pyproject.toml') }}|${{ hashFiles('tox.ini') }} - run: pip install tox - run: tox -e docs style: @@ -339,7 +354,7 @@ jobs: - uses: actions/cache@v4 with: path: ~/.cache/pip - key: pip|${{ hashFiles('setup.py') }}|${{ hashFiles('tox.ini') }} + key: pip|${{ hashFiles('pyproject.toml') }}|${{ hashFiles('tox.ini') }} - uses: actions/cache@v4 with: path: ~/.cache/pre-commit @@ -356,7 +371,7 @@ jobs: - uses: actions/cache@v4 with: path: ~/.cache/pip - key: pip|${{ hashFiles('setup.py') }}|${{ hashFiles('tox.ini') }} + key: pip|${{ hashFiles('pyproject.toml') }}|${{ hashFiles('tox.ini') }} - run: pip install tox - run: tox -e deptry # The three async-* jobs below exist to test the async backend *selection* logic, @@ -418,6 +433,41 @@ jobs: " - name: Run async tests with httpxyz run: pytest tests/test_async_davclient.py -v + async-httpx2: + # Uninstalls niquests and httpxyz and installs httpx2 - Pydantic's + # continuation of httpx, a separate package rather than a new httpx release. + # Unlike httpxyz it does not register itself in sys.modules as "httpx", so + # this job is what catches code that reaches for httpx by name. + # Runs unit tests plus the Xandikos async integration tests (embedded, no + # service container needed), so the backend is exercised over real HTTP. + name: async (httpx2 fallback) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install dependencies with httpx2, without niquests or httpxyz + run: | + pip install --editable .[test] + pip uninstall -y niquests httpxyz httpx + pip install httpx2 + - name: Verify httpx2 is used + run: | + python -c " + from caldav.async_davclient import _HTTPX_FLAVOUR, _USE_HTTPX, _USE_HTTPXYZ, _USE_NIQUESTS + assert not _USE_NIQUESTS, 'niquests should not be available' + assert not _USE_HTTPXYZ, 'httpxyz should not be available' + assert _USE_HTTPX, '_USE_HTTPX should be set when httpx2 is used' + assert _HTTPX_FLAVOUR == 'httpx2', f'expected httpx2, got {_HTTPX_FLAVOUR}' + print('✓ Using httpx2 for async HTTP') + " + - name: Run async tests with httpx2 + # Xandikos runs embedded, so no service container is needed; the + # selection is deliberately narrow rather than "everything not baikal", + # since the runner has docker and would otherwise auto-discover the + # docker test servers. + run: pytest tests/test_async_davclient.py tests/test_async_integration.py -v -k "xandikos or Xandikos" async-httpx: # Uninstalls both niquests and httpxyz to force the plain-httpx fallback path. # Runs unit tests + a real integration test against Baikal (the lightest server) diff --git a/.gitignore b/.gitignore index bbbc3491..708a129a 100644 --- a/.gitignore +++ b/.gitignore @@ -21,12 +21,15 @@ accountific.db tests/.noseids *.bak *~ -#*# +# Emacs auto-save files. The backslashes are required: a bare "#" starts a +# comment, so the pattern was silently a no-op. +\#*\# caldav.egg-info/ tests/conf_private.py .tox .eggs .venv +venv caldav/_version.py tests/docker-test-servers/baikal/baikal-backup/ tests/docker-test-servers/*/baikal-backup/ @@ -34,3 +37,7 @@ tests/docker-test-servers/*/baikal-backup/ !tests/docker-test-servers/baikal/Specific/ # Local test server configuration (may contain credentials) tests/caldav_test_servers.yaml +# Lychee link checker cache +.lycheecache +# Scratch files from AI sessions (review notes, draft commit messages) +docs/design/tmp-* diff --git a/.lycheeignore b/.lycheeignore index 2a9c7c03..c04f7500 100644 --- a/.lycheeignore +++ b/.lycheeignore @@ -2,6 +2,7 @@ https?://your\.server\.example\.com/.* https?://.*\.example\.com(:\d+)?(/.*)?$ https?://domain/.* +https?://evil.attacker.com/caldav/ # Localhost URLs for test servers (not accessible in CI) http://localhost:\d+/.* @@ -17,6 +18,7 @@ https://caldav\.gmx\.net/.* https://caldav\.icloud\.com/.* https://p\d+-caldav\.icloud\.com/.* https://posteo\.de:\d+/.* +https://sync\.infomaniak\.com/.* https://purelymail\.com/.* https://webmail\.all-inkl\.com/.* https://www\.google\.com/calendar/dav/.* @@ -36,6 +38,10 @@ https://oauth2\.googleapis\.com/.* # Personal/demo test server (may be down) https?://davical\.bekkenstenveien53c\.oslo\.no/.* +# Sites that serve 403 to non-browser clients. The links are fine in a +# browser; lychee just isn't one. +https://stackoverflow\.com/.* + # Dead or broken links we can't fix http://fsf\.org/.* http://oxpedia\.org/.* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e25a8425..a2489f42 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,21 +1,21 @@ --- repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.9.4 + rev: v0.15.20 hooks: - - id: ruff + - id: ruff-check args: [--fix] - id: ruff-format - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - - id: check-byte-order-marker + - id: fix-byte-order-marker - id: trailing-whitespace - id: end-of-file-fixer - repo: https://github.com/pycalendar/ai-prompt-auto-commit - rev: v0.0.5 + rev: v0.0.8 hooks: - id: unstage-ai-prompts - id: append-ai-prompts @@ -26,13 +26,13 @@ repos: stages: [manual] - repo: https://github.com/compilerla/conventional-pre-commit - rev: v3.4.0 + rev: v4.4.0 hooks: - id: conventional-pre-commit stages: [commit-msg] - repo: https://github.com/lycheeverse/lychee - rev: lychee-v0.24.1 + rev: lychee-v0.24.2 hooks: - id: lychee args: ["--no-progress", "--timeout", "10", "--exclude-path", ".lycheeignore", "--max-cache-age=30d", "--cache"] diff --git a/CHANGELOG.md b/CHANGELOG.md index 486aa3ed..2c0b1136 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,82 @@ Changelogs prior to v3.0 is pruned, but was available in the v3.1 release This project should adhere to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), though for pre-releases PEP 440 takes precedence. +## [Unreleased] + +### Added + +* The async client can now use **httpx2** - Pydantic's continuation of httpx - as its HTTP library. The async fallback chain is niquests, httpx2, httpxyz, then httpx; the first one installed wins, and niquests remains the default and the recommended choice. Note that httpx2 is a separate module rather than a new release of httpx, so `pip install httpx2` is what enables it. See https://github.com/python-caldav/caldav/issues/611 - support for the httpx family in *sync* mode is still not there and is not planned for 3.3. +* New `write-delay` server-peculiarity in `compatibility_hints.py`: for servers that process writes asynchronously (a PUT/DELETE/MKCALENDAR/PROPPATCH/... returns before the change is queryable, so an immediate read-back 404s or returns stale data), a client must wait a bit after every write. This is the general, write-side counterpart of `search-cache` (which only delays searches). Configured as `{'behaviour': 'delay', 'delay': }`; the integration test suites (sync and async) honour it by sleeping after every write request. The new Infomaniak profile uses `write-delay` (16s) rather than `search-cache`, since the asynchronicity there is server-wide rather than search-specific. +* New compatibility flag `save-load.event.recurrences.exception.reschedule`: whether the server accepts re-anchoring a whole recurring event (moving the master `DTSTART`) while detached exceptions (`RECURRENCE-ID`) are attached. OX App Suite rejects this with `409 Conflict` even with a matching `If-Match` etag, although rescheduling an exception-free recurring event works there. `testEditSingleRecurrence` now gates its final `save(all_recurrences=True)` dtstart/dtend step on this flag. +* The JMAP clients now keep a persistent HTTP session, so connections are reused across requests instead of a fresh TCP+TLS handshake per JMAP call. `JMAPClient` and `AsyncJMAPClient` can be used as (async) context managers, and the session can also be released explicitly with `client.close()` / `await client.aclose()`. +* `caldav.config.extract_conn_params_from_section` is now public API (renamed from `_extract_conn_params_from_section`), so that downstream tools like plann can map plann-style config sections (`caldav_url`, `caldav_user`, `features`, etc.) to `DAVClient` parameters without duplicating the logic. +* New compatibility feature `create-calendar.stable-url` (default `full`): whether a calendar, once created, remains addressable at the URL derived from the requested `cal_id`. Some servers assign a different *canonical* URL: Zimbra relocates the collection to a display-name-derived path when a display name is set (a collection alias lingers at the `cal_id` and answers `PROPFIND`/`REPORT`, but a `GET` on a child object under it 404s, so the `cal_id` is not a usable address); OX always exposes an opaque `cal://0/NNN` (base64-segment) canonical URL. Both are marked `create-calendar.stable-url: unsupported`. For such servers `Calendar._create()` now discovers and adopts the canonical URL after creation (re-pointing `self.url`) instead of dropping the display name, so the calendar keeps its name *and* every later URL-based operation resolves — identical handling for Zimbra and OX. +* `caldav[niquests]` is now a valid install target. It changes nothing today - `niquests` is still an ordinary dependency - but v4.0 is planned to ship without a default HTTP library dependency, and `caldav[niquests]` is how the current behaviour will be kept. Downstream projects can depend on it now and not have to change anything at that point. See https://github.com/python-caldav/caldav/issues/611 +* New `compatibility_workarounds` parameter on `Calendar.search()` / `CalDAVSearcher.search()` / `async_search()`. When `False`, all server-compatibility workarounds are disabled and the query is sent verbatim (a single REPORT, no comp-type splitting, no filter rewriting, no fallback retries). Mainly for the server-compatibility checker, to observe raw server behaviour. + +### Fixed + +* A response body that contains no iCalendar at all - an empty object, an HTML error page delivered with a 200, a notification carrying only headers - now raises `caldav.lib.error.ResponseError` naming the URL and quoting what arrived, instead of `ValueError: Found no components where exactly one is required` from inside the icalendar library. Data that does contain an iCalendar component is unaffected and still parsed by icalendar as before. +* The source distribution no longer ships stray local files. hatchling's VCS-ignore support only honours the *root* `.gitignore`, so files hidden from `git status` by a nested `.gitignore` or by the packager's global git ignore file were invisible locally and packaged anyway — `caldav-3.2.1.tar.gz` contains `.claude/settings.json` and 1755 files under `venv/`, and the current tree would have added 443 files under `.prompts/`. A new `package` tox environment (run in CI, and now part of the release procedure) builds both artifacts and fails if anything git does not track turns up in them. +* Looking up a calendar or object that does not exist now raises `NotFoundError` also when the server reports the 404 inside a `207 Multi-Status` (a bare `` on the `` element, which RFC 4918 §14.24 allows) rather than as a plain HTTP 404. Previously the property lookup found no `propstat` elements, ignored the 404, and returned `None` for every requested property — so e.g. `calendar.get_display_name()` on a non-existent calendar silently returned `None` while `calendar.get_events()` on the same calendar raised. Observed against Xandikos. +* `compatibility_hints.py`: `testCheckCompatibility` had a blind spot — a sub-feature the server-tester explicitly probed whose *observed* status happened to equal the type default (e.g. a server-feature observed as `full`) was dropped from the compacted observed dict, and if its *declared* status was only inherited from a parent (not an explicit key) it was absent from the compacted expected dict too. Being in neither dict, it was never compared, so a real conflict went unreported. Concretely, Infomaniak declares `search.comp-type` `unsupported` while genuinely supporting the optional-comp-type behaviour (`search.comp-type.optional`), and the mismatch slipped through. The comparison is now a unit-tested `FeatureSet.compare()` method that also iterates the probed feature set, and the Infomaniak profile declares `search.comp-type.optional: full` explicitly. +* `jmap/client.py` and `jmap/async_client.py` `update_event()`: to honour RFC 8620 PatchObject merge semantics, the update null-injects every optional property absent from the new iCalendar so removed properties are actually cleared server-side. Some servers (observed with Stalwart) reject a property they do not support — e.g. `recurrenceRules`/`excludedRecurrenceRules` — as `invalidProperties` even when it is being set to `null`, which made every `update_event()` against such a server fail. Nulling an absent property is harmless cleanup, so the update now drops the server-rejected null-cleanup keys and retries (looping, since some servers report only one offending property per response) until the update succeeds. A rejection of a property the client actually assigned a value still surfaces as `JMAPMethodError`. +* `async_davclient.py` `_async_request()`: the issue-#158 connection-abort workaround sent a probe GET to detect the auth challenge; if the probe returned anything other than 401+WWW-Authenticate (e.g. a 200 HTML login page), the code fell through to `response = DAVResponse(probe_r, self)` — returning the probe response as if it were the original request's response, and silently swallowing the real connection error. Now the original exception is re-raised when the probe does not yield a challenge. +* `collection.py` `freebusy_request()`: for async clients, `add_attendee()` was called on each attendee before the `is_async_client` check dispatched to `_async_freebusy_request()`. For a `Principal` attendee on an async client, `get_vcal_address()` returns a coroutine; `add_attendee` then tried to set `.params` on the coroutine → AttributeError. `_async_save_with_invites` already awaited `get_vcal_address()` correctly. Fixed by passing attendees to `_async_freebusy_request()` and performing the await there. +* `search.py`: the documented `operator='=='` exact-match guarantee was never enforced — `post_filter` was not set to `True` for `==` searches, so the server's substring semantics leaked through. `icalendar_searcher.check_component()` already handles `==` as exact-match; the fix adds `==` to the `post_filter=True` trigger conditions. +* `async_davclient.py` `aio.get_calendars(calendar_name=...)`: name-based lookup iterated `self.get_calendars()` synchronously through the non-async `Principal.calendar(name=...)` path, which returns a coroutine for async clients; the loop body was iterating over the coroutine object, never the calendars, so name-based lookup returned nothing. Fixed by calling `await principal.get_calendars()` and filtering by display-name in the async path. +* `async_davclient.py` `get_calendars()`: lacked the GMX principal-URL fallback present in the sync client — when `calendar-home-set` was missing the async path returned `[]` immediately instead of falling back to the principal URL as calendar home. Parity restored. +* `collection.py` `Principal.calendar(cal_id=...)`: for async clients a bare (non-URL) `cal_id`/`name` raised `TypeError: argument of type 'coroutine' is not a container or iterable`, because the synchronous `calendar_home_set` property evaluated `"@" in ` without awaiting the async `get_property`. It now returns a coroutine that resolves the calendar home set (PROPFIND) before constructing the `Calendar`; a full-URL `cal_id`/`cal_url` needs no home set and still returns synchronously. The async integration tests' pre-test calendar cleanup relied on this call and swallowed the error in a bare `except`, so leftover calendars were never removed and a later MKCALENDAR `405 "resource already exists"` followed (seen against Infomaniak). Cleanup is now centralised in the `adelete_calendar_if_present()` test helper with a narrow `except`. +* `search.py`: the `undef` operator branch used `property.upper()` without the `category→CATEGORIES` alias mapping that the regular filter branch applies, so `add_property_filter('category', '', operator='undef')` queried the nonexistent `CATEGORY` property. `is-not-defined` on a nonexistent property matches every object, so the filter silently returned all events regardless of whether they had categories. +* `davclient.py` and `async_davclient.py`: rate-limit retry raised `TypeError: unsupported operand type(s) for +=: 'NoneType' and 'float'` on the second 429 response when the server provides no usable `Retry-After` value (`compute_sleep_seconds` returns `None`). The `sleep_seconds += rate_limit_time_slept / 2` line executed before the `sleep_seconds is None` guard. Now re-raise `RateLimitError` first, then update the sleep estimate. +* `davclient.py` `DAVClient.__init__()`: (a) `DAVClient(url='https://user@host/', password='secret')` crashed with `TypeError` — `unquote(self.url.password)` was called unconditionally when the URL had a username, but `self.url.password` is `None` when the URL contains no password. (b) URL-embedded credentials silently overrode explicit `username`/`password` kwargs; the async client already gave explicit kwargs higher precedence. Now: explicit kwargs win; URL credentials are only used as fallback when kwargs are absent. An explicit `username` discards the URL credentials wholesale rather than merging them field by field — otherwise `DAVClient(url='https://bob:hunter2@cal.example.com/', username='alice')` would ship alice's login with bob's password. Overriding only the password keeps the URL's username, which stays a coherent pair. +* `config.py` `resolve_features()`: returning a named profile (`features='xandikos'`) returned the module-level dict object directly without copying it. Any code that then mutated the returned dict (e.g. `testing.py` patching `auto-connect.url.domain`) permanently corrupted the module-level dict for the whole process lifetime, so a second `DAVClient(features='xandikos')` would see the mutated domain. Similarly `testing.py` `XandikosServer`/`RadicaleServer` used a shallow `.copy()` — nested dict mutation still reached the module level. All three now use `copy.deepcopy()`. +* `config.py` `get_connection_params()`: explicit keyword arguments (e.g. `get_davclient(password='secret')`) were only respected when `url` or `features` was also present. When an env-var or config-file source was found instead, explicit params were silently dropped. Now the explicit params are merged (overlaid) on top of whatever lower-priority source wins — including the test-server source, which previously returned its configuration verbatim. A keyword argument whose value is `None` counts as "not supplied" rather than "unset it", so the common `get_davclient(url=args.url, username=args.user, password=args.password)` CLI wrapper no longer wipes `CALDAV_URL` when the user only passed `--password`. An empty string is still explicit, since that is meaningful for servers with no authentication. +* `calendarobjectresource.py` `_complete_recurring_safe()`: completing a recurring task passed the caller-supplied `completion_timestamp` to `_next()` correctly but then called `completed.complete()` without it, so the completed copy always recorded the current wall-clock time as `COMPLETED` regardless of the timestamp the caller specified. The async twin already passed `completion_timestamp` through; sync is now consistent. +* `calendarobjectresource.py` `_get_duration()`: `isinstance(i["DTSTART"], datetime)` tested the `vDDDTypes` wrapper object (which is never a `datetime`), so the date-vs-datetime branch always took the "is a date" path. A VTODO with a timed DTSTART and no DUE/DURATION got `duration = timedelta(days=1)` instead of `timedelta(0)`, shifting the next due date by one day when completing a recurring task. Fixed: test `isinstance(i["DTSTART"].dt, datetime)`. +* `jmap/client.py` and `jmap/async_client.py` `create_task()`: a JMAP server response that returned an empty `created` dict (with neither a `created` entry nor a `notCreated` entry for `"new-0"`) raised a bare `KeyError` instead of the documented `JMAPMethodError`. The `create_event()` method already had the required guard; `create_task()` was missing it in both sync and async clients. +* `lib/vcal.py` `fix()`: truncated iCalendar data (no `END:` line) triggered a bare `assert` which gave no useful message and was silently skipped under `python -O`. Now logs a warning and returns the data unchanged instead. +* `lib/vcal.py` `create_ical()`: when both `alarm_*` props and `ical_fragment` were supplied, the fragment was injected before the first `END:V` line — which is `END:VALARM`, placing e.g. an `RRULE` *inside* the alarm component. The regex now targets `END:V(EVENT|TODO|JOURNAL)` specifically. +* `lib/vcal.py` `fix()`: the backslash-unescape step used `('\"')` as a regex group, which matches only the literal two-character sequence `'"`. A backslash before a lone `'` or lone `"` was silently left in place. Fixed by using the character class `['\"]`. +* `lib/vcal.py` `fix()`: the trailing-whitespace fixup (`re.sub(" *$", "", fixed)`) lacked `re.MULTILINE`, so it only stripped trailing spaces at the very end of the document and never per-line. It now strips per-line — but deliberately *not* from a line that is continued by a folded line: RFC 5545 3.1 folds blind at 75 octets, so the fold may land right after a space belonging to the value, and stripping it would join two words together. Junk whitespace in front of a fold (e.g. in iCloud `X-APPLE-STRUCTURED-LOCATION` base64 values) is therefore still left alone; it cannot be told apart from real content. +* `lib/error.py` `PYTHON_CALDAV_COMMDUMP`: when this debug env-var is set, a `logging.warning()` is now emitted at import time to remind the operator that request/response bodies and headers (including credentials and calendar PII) are being written to uniquely-named files under `/tmp` that accumulate indefinitely. +* `jmap/objects/calendar.py` `JMAPCalendar.search()`: `datetime` arguments for `start`/`end` were formatted with `datetime.isoformat()`, which produces `+HH:MM` offsets for aware non-UTC datetimes and no timezone indicator for naive datetimes. JMAP requires UTCDate format (`YYYY-MM-DDTHH:MM:SSZ`). Fixed by converting to UTC and using `strftime`. +* `jmap/client.py` and `jmap/async_client.py` `get_objects_by_sync_token()`: the `newState` from `CalendarEvent/changes` was discarded into `_`, so callers could not chain sync calls without a separate `get_sync_token()` round-trip — a race window where intervening changes would be silently missed. The method now returns a 4-tuple `(added, modified, deleted, new_sync_token)` instead of a 3-tuple. +* `compatibility_hints.py` `FeatureSet.copyFeatureSet()`: merging a plain-string feature value over an existing string-valued entry in the feature set raised a bare `AssertionError` — the `'support' not in server_node` guard blocked the update branch and fell through to `else: raise AssertionError`. Plain strings are the dominant style in the hint dicts, so any two-layer server config expressing the same feature crashed. Fixed by removing the `not in server_node` condition. +* `compatibility_hints.py` `FeatureSet.copyFeatureSet()`: an unknown feature name in a config file produced only a `UserWarning` but still stored the bad key in `_server_features`; a later `collapse()`/`is_supported()` call then raised a message-less `AssertionError` far from the original config. Fixed by `continue`-ing after the warning so unknown keys are never stored. +* `async_davclient.py`: HTML-on-401 diagnostic hint checked `self.headers` (the client's own request headers) for `Content-Type: text/html` instead of `r.headers`, so the intended "server returned an HTML login page, consider setting auth_type" message could never fire. +* `base_client.py` `get_calendars(calendar_urls=...)`: a calendar explicitly requested by URL was silently omitted from the result when its `displayname` property is the empty string `""`, because the check `if _try(calendar.get_display_name, ...)` was a truthiness test. The async counterpart already used `is not None`; sync is now consistent. +* `config.py` `expand_config_section()`: requesting a section name that is absent from the config raised `KeyError` instead of returning `[]`, causing plain `caldav.get_calendars()` to crash with `KeyError: 'default'` on configs with no `default` section. +* `config.py` `expand_config_section()`: `disable: true` was silently ignored for sections fetched by explicit name or via a `contains` list — the check used the string literal `"section"` as the config key instead of the `section` variable. Only the glob `"*"` path honoured `disable`. +* `lib/auth.py` `extract_auth_types()`: a `WWW-Authenticate` header ending with a trailing comma (seen in the wild) raised `IndexError` in the set comprehension because `h.split()` on an empty segment fails. Added an `if h.strip()` guard. +* `calendarobjectresource.py` `change_attendee_status()`: calling the method on an event with no `ATTENDEE` properties at all raised a bare `KeyError('ATTENDEE')` instead of the expected `NotFoundError`; the `try/except NotFoundError` wrapper in the `Principal` dispatch path could not catch it, so the "Principal is not invited" message was unreachable. Also, the genuine not-found error message contained a literal `%s` that was never substituted with the attendee address. +* `calendarobjectresource.py` `add_attendee()`: passing an attendee address with an uppercase or mixed-case URI scheme (`"MAILTO:user@example.com"`) raised `UnboundLocalError` — the scheme check used `str.startswith("mailto:")` which is case-sensitive, so the address fell through all branches without assigning `attendee_obj`. RFC 3986 §3.1 specifies URI schemes are case-insensitive. +* `response.py`: the XML parser is now constructed with `resolve_entities=False, no_network=True`, so a malicious or MITM server cannot inject arbitrary text into parsed property values through inline DOCTYPE entity definitions. On the lxml versions most people have this was already the effective behaviour (`no_network` defaults to True, and `resolve_entities` defaults to `'internal'` from lxml 5.0), but lxml is an unpinned dependency and the parser should not be relying on another project's defaults for this. +* `discovery.py` `discover_service()`: `require_tls=True` was not enforced on the well-known URI redirect target — a same-domain `Location: http://...` passed the domain-validation check and was returned as `ServiceInfo(tls=False)`, allowing a misconfigured or MITM server to silently downgrade the connection to plaintext. Fixed by checking `well_known_info.tls` against `require_tls` before returning the result. +* `datastate.py` `RawDataState.get_component_type()`: tested for the string `"BEGIN:FREEBUSY"` but real iCalendar data uses `"BEGIN:VFREEBUSY"`, so any `FreeBusy` object holding raw data returned `component_type=None` — making `is_loaded()` and `has_component()` return `False`, `save()` silently no-op at its early return, and `load(only_if_unloaded=True)` reload spuriously on every call. The same typo appeared in the base-class `get_uid()` and `get_component_type()` fallback parsers which looked for `comp.name == "FREEBUSY"` instead of `"VFREEBUSY"`. +* `calendarobjectresource.py` `_set_data()`: the raw-string branch cleared the legacy `_data`/`_vobject_instance`/`_icalendar_instance` attributes but never reset `self._state`. Once `_state` was populated by an earlier call to `_ensure_state()` (triggered by e.g. `.id` or `is_loaded()`), all subsequent reads via `get_data()`, `get_icalendar_instance()`, and `.id` served the pre-reload content even after `load()` fetched new data from the server. +* `search.py`: the `search.combined-is-logical-and: unsupported` workaround (triggered on e.g. Nextcloud) stripped all property filters from the server query to send only the time range, but passed `post_filter=None` (the ambient value) to `filter()` instead of `True`. `_filter_search_results` short-circuits when `post_filter` is falsy, so a search with both a time range and a property filter (e.g. `SUMMARY contains "foo"`) returned every object in the time range — the property filter was silently dropped. The sibling workarounds in the same function already used `post_filter=True`; this one now does too. +* `URL.canonical()`: two related bugs — (a) the canonical form was built from `self.url_parsed` (which still contains `user:pass@` in the netloc) rather than the auth-stripped URL, so `canonical()` leaked credentials into the returned URL and `__eq__`/`__hash__` comparisons between an authenticated client URL and a server-returned href (no credentials) were False; (b) when a URL had no auth part, `unauth()` returned `self` and `canonical()` then overwrote `url_raw`/`url_parsed` in place — a bare `==` or `hash()` call silently mutated the URL object, potentially re-encoding special characters (e.g. `+` → `%2B`) and causing subsequent requests to target the wrong resource. Fixed by using the auth-stripped URL's parsed form for `arr` and always returning a fresh `URL` object. +* `_post_put`: a 302 response to `PUT` always raised `IndexError` instead of following the redirect — iterating the headers dict yields key strings, not tuples, so `x[0]` was the first character of each header name, never `"location"`. Fixed by using `r.headers.get("location")`. +* `vcal.fix()`: the `COMPLETED` date-to-datetime regex consumed the trailing newline, merging the following iCal property into the `COMPLETED` value on every inbound object from a server that stores `COMPLETED` as a plain date (e.g. SOGo). Fixed by using a lookahead `(?=\s)` instead of consuming `\s`. + +* Time-range searches without a component type (`search(start=..., end=...)` with no `event`/`todo`/`journal`/`comp_class`) crashed against SabreDAV-based servers (Baikal, Nextcloud, ...) with `ReportError`: *"You cannot add time-range filters on the VCALENDAR component"*. A `CALDAV:time-range` is only valid inside a `VEVENT`/`VTODO`/`VJOURNAL`/`VFREEBUSY`/`VALARM` comp-filter (RFC4791 section 9.7), never directly under `VCALENDAR`. The library now splits such a search into one query per component type, and additionally recovers from the server rejection at runtime if it occurs anyway. See https://github.com/python-caldav/caldav/issues/681 +* Property-filter searches without a component type (e.g. `search(category=...)` or other attribute filters with no `event`/`todo`/`journal`/`comp_class`) silently returned nothing on most servers (Xandikos, SabreDAV, ...): the prop-filter landed under the `VCALENDAR` comp-filter, which has no component properties like `CATEGORIES` to match. The library now splits such a search into one query per component type as well (`search.text.comp-type-optional`). See https://github.com/python-caldav/caldav/issues/681 Results from such a split are deduplicated by URL, since a resource that legally holds both a `VEVENT` and a `VTODO` matches two of the three queries; they come back grouped by component type rather than in server order, so pass a sort key if the order matters. +* `search()`'s generator driver now feeds exceptions raised while executing a request back into the search logic, so the server-compatibility fallbacks and per-object load error handling actually take effect (previously dead code). Applies to both the sync and async code paths. +* `compatibility_hints`: OX was pinned to `create-calendar.set-displayname: unsupported` (a value masked by a checker bug that verified the feature by display-name lookup, which a leftover/colliding calendar would shadow); OX stores the display name as a property separate from the calendar URL and honours it at creation time, so the expectation is corrected to `full`. +* `compatibility_hints`: Stalwart's `search.recurrences.expanded.exception` was inheriting the default `full`, but Stalwart's server-side `CALDAV:expand` only suppresses the exception-overridden occurrence when `SEQUENCE` is absent. With `SEQUENCE` present (as real-world clients always emit) it returns both the original occurrence and the override, so the expectation is corrected to `fragile`. +* Config file sections with `features` but no `caldav_url` were rejected, even though the URL can be derived from the `auto-connect.url` compatibility hints. Explicitly passed parameters already worked this way; now `get_davclient(config_section=...)` and friends behave consistently. +* `jmap/convert/jscal_to_ical.py`: a `recurrenceOverrides` entry that does not include a `"start"` key (the common case — title-only change, description update, etc.) produced a child `VEVENT` with `DTSTART` copied from the master event's start time rather than from the override key. This effectively relocated every non-rescheduled override to the master's first occurrence, breaking all override display. Default is now the override key itself. +* `jmap/convert/jscal_to_ical.py`: `EXDATE` and `RECURRENCE-ID` values were always emitted as floating (timezone-less) `DATE-TIME` regardless of the event's `timeZone` or `showWithoutTime` flag. Per RFC 5545 §3.8.5.1 the value type must match `DTSTART`; a floating `EXDATE` on a `TZID`-anchored event does not match any instance, so excluded occurrences reappear. Override keys are now parsed with the event timezone applied (`TZID`-anchored events) or as `date` objects (all-day events). +* `jmap/convert/_utils.py` `_format_local_dt()`: UTC datetimes produced a `Z`-suffixed string. RFC 8984 §1.4 defines `LocalDateTime` (the type required for `recurrenceOverrides` keys and `recurrenceRules.until`) as a bare `YYYY-MM-DDThh:mm:ss` without any suffix; `Z`-suffixed override keys cannot match `LocalDateTime` occurrence keys, causing mismatches on strict servers. The function now returns a bare local representation — converted into the event's own timezone first, since a `LocalDateTime` is local *to the event*: dropping the offset from a UTC value instead of converting it would shift `EXDATE`/`RECURRENCE-ID`/`UNTIL` by the UTC offset on every `TZID`-anchored event, and emit a floating `UNTIL` against a `TZID` `DTSTART`, which RFC 5545 §3.3.10 forbids. +* `jmap/convert/ical_to_jscal.py` and `jmap/convert/jscal_to_ical.py`: the `STATUS` property was silently dropped in both conversion directions. `STATUS:CANCELLED` round-tripped as `status: confirmed` (JSCalendar default), so cancelled meetings appeared active. Mappings `CONFIRMED ↔ confirmed`, `TENTATIVE ↔ tentative`, `CANCELLED ↔ cancelled` are now implemented. +* `jmap/client.py` and `jmap/async_client.py` `update_event()`: RFC 8620 §3.3 specifies that absent keys in a PatchObject preserve the server value. `update_event` sent the full converted JSCalendar dict as the patch; properties the caller removed (e.g. LOCATION, VALARM) were absent from the patch and therefore silently persisted on the server. `update_event` now explicitly sets all optional top-level JSCalendar properties to `null` when they are absent from the conversion result, ensuring the server removes them. + +### Changed + +* Search results that need loading are now fetched with a single `calendar-multiget` REPORT instead of one `GET` per object — a 200-event search made 200 requests before. If the multiget fails the library falls back to the per-object loads, so the failure semantics of `search()` are unchanged, but a server that answers the batched REPORT badly will now show up as a search problem rather than as one bad object. +* `compatibility_hints`: fourteen directly-probed feature *nodes* that also have refinement sub-features now carry their own explicit `default` (`calendar-color`, `delete-calendar`, `get-current-user-principal`, `propfind`, `propfind.allprop`, `save-load.event`, `save-load.journal`, `save-load.mutable`, `save-load.todo`, `save-load.todo.recurrences`, `scheduling`, `search.recurrences.includes-implicit.todo`, `search.text.category`, `sync-token`). This marks them as *independent* features: `is_supported()` and `collapse()` no longer derive/fold them away from their children, so e.g. `sync-token` stays `full` even when `sync-token.delete` is `unsupported`. Each such node has a corresponding check in the server-tester (`search.comp-type` gained one); `principal-search` deliberately keeps no default since it is a genuine OR-grouping of its sub-searches. + ## [3.2.1] - 2026-05-28 The changeset in 3.2.1 is predominently added async integration tests. Those tests should now be replicating all the logic in the good old sync integration tests under `test_caldav.py`. Some few more bugs were found while adding those tests. @@ -79,7 +155,7 @@ The two most significant news in v3.2 are **relatively well-tested support for s ### Added * `add_organizer()` now accepts an optional explicit *organizer* argument (a `Principal`, `vCalAddress`, or email string) -* Complete support for **Schedule-Tag** (RFC 6638 §3.2–3.3) and **Etag**. Headers from upstream will be catched and stored in the properties. If those properties exists, `If-Schedule-Tag-Match` or `If-Match` headers will be sent. A `ScheduleTagMismatchError` or `EtagMismatchError` will be raised on 412. +* Complete support for **Schedule-Tag** (RFC 6638 §3.2–3.3) and **Etag**. Headers from upstream will be caught and stored in the properties. If those properties exists, `If-Schedule-Tag-Match` or `If-Match` headers will be sent. A `ScheduleTagMismatchError` or `ETagMismatchError` will be raised on 412. ### Changed diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6110246c..3e33f20a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,7 +20,7 @@ The types used should (as for now) be one of: The `compatibility_hints.py` has been moved from the test directory to the codebase not so very long ago. Some special rules here: * Adjusting the feature set for some calendar server? Check if there exists some workarounds etc in the code for said feature, if so, then it should be considered a fix or a feature. Perhaps even a breaking change. Otherwise, use `test: ...`. (because it is relevant for the compatibility test, if nothing else). -* Adding a new feature hint? Ensure it's covered by the caldav-server-tester. Since we have a compatibility test, it will be relevant for the test - so use `test: (...)`. It should be covered by the caldav-serveer-tester, so refer to some issue or pull request for the caldav-server-tester in the commit message. +* Adding a new feature hint? Ensure it's covered by the caldav-server-tester. Since we have a compatibility test, it will be relevant for the test - so use `test: (...)`. It should be covered by the caldav-server-tester. * Changing some descriptions? That goes as `docs: ...` even if it's actually changing a variable in the code. This is not set in stone. If you feel strongly for using something else, use something else in the commit message and update this file in the same commit. diff --git a/SECURITY.md b/SECURITY.md index 0a1a63aa..1da5501e 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,48 +1,84 @@ # Security policy -Issues should be fixed ASAP, and information on any security issue should be published as soon as it's fixed. Use the GitHub issue tracker or check up [CONTACT](CONTACT.md) or even the [CODE OF CONDUCT](CODE_OF_CONDUCT) file to get in touch with the maintainer. +Issues should be fixed ASAP, and information on any security issue should be published as soon as it's fixed. Serious issues should be reported privately and kept under wraps until a fix is released — use [GitHub's private vulnerability reporting](https://github.com/python-caldav/caldav/security/advisories/new) (the "Report a vulnerability" button on the Security tab), or get in touch with the maintainer via [CONTACT](CONTACT.md) or the [CODE OF CONDUCT](CODE_OF_CONDUCT) file. Use the public GitHub issue tracker only for non-sensitive issues. There are no "LTS"-releases of the CalDAV package, but the maintainer will always consider backporting security fixes if it's deemed relevant. The maintainer is doing most of the maintenance on hobby-basis and may have other things in life preventing him from dealing with issues on the go, so no guarantees are given. -All contributions are carefully reviewed by the maintainer, and all releases are carefully tested and tagged with a PGP-signed commit. +All contributions are carefully reviewed by Tobias Brox, and AI-tools are used for code reviews prior to each release. All releases are carefully tested and tagged with a PGP-signed commit. # Known security issues and risks ## RFC6764 -I do see a major security flaw with the RFC6764 discovery. If the DNS is not to be trusted, someone can highjack the connection by spoofing the service records, and also spoofing the TLS setting, encouraging the client to connect over plain-text HTTP without certificate validation. Utilizing this it may be possible to steal the credentials. This flaw can be mitigated by using DNSSEC, but DNSSEC is not widely used, and fixing support for DNSSEC validation in the CalDAV library was found to be non-trivial (perhaps I'll look into it again some time after 3.0 has been released). This has been mitigated by adding a require_tls` connection parameter that is True by default, plus by ensuring one isn't routed to a different domain. +**Summary**: auto-discovery of the CalDAV-URL seems to be insecure by design, anyone controlling your local resolver (or upstream resolvers) may try to fish out username and password. -## DDoS/OOM risk +**Mitigation**: Leave `require_tls`, `ssl_verify_cert` to the default - or better still: use a URL rather than a domain when configuring the library. + +RFC6764 discovery depends on correct DNS-lookups, but DNS is not to be trusted. The proper solution is DNSSEC, unfortunately DNSSEC is not widely used, and fixing support for DNSSEC validation in the CalDAV library was found to be non-trivial (some work has been done, but it was found to be too difficult - the pull request is stalled as for now). + +Connections can be hijacked if someone spoofs the service records. This has been partly mitigated by adding a `require_tls` connection parameter that is `True` by default, cert validation, and ensuring one isn't routed to a different domain. + +Auto-discovery and HTTP redirects also mean the host you end up talking to may not be the one you configured. If you run the library in a context where it can reach internal/private network resources (server-side request forgery, SSRF), be aware that a malicious DNS resolver or a malicious/compromised server can steer requests towards other hosts. + +## DDoS/OOM risk - recurring events/tasks search + +**Summary:** If you allow untrusted parties to specify search-terms towards a calendar containing recurring events/tasks, bad things may happen. The package offers both client-side and server-side expansion of recurring events and tasks. It currently does not offer expansion for open-ended date searches - but with a large enough timespan and a frequent enough RRULE, there may be millions of recurrences returned. Those recurrences are returned as a generator, so things will not break down immediately. However, there is no guaranteed sort order of the recurrences ... and once you add sorting parameters to the search, bad things may happen. +## XML parsing + +**Summary:** XML responses are parsed defensively by default; only relax this against servers you trust. + +The library parses XML responses from the server using lxml. By default the parser is configured to resist common XML attacks: external entity resolution is disabled (`resolve_entities=False`) and network access during parsing is blocked (`no_network=True`), guarding against XXE (XML External Entity) attacks, and lxml's built-in limits protect against oversized "billion laughs"-style entity-expansion payloads. + +The `huge_tree` connection option (default off) disables lxml's built-in parser limits so that very large calendar objects can be handled. With `huge_tree` enabled, a malicious or compromised server can exhaust available memory with a crafted XML payload — only enable it against servers you trust. See the [lxml XMLParser documentation](https://lxml.de/api/lxml.etree.XMLParser-class.html). + ## Bugs causing weird things happening -Weird things may happen due to bugs both on in the CalDAV package, on your side and on the server side. Here are some weird experiences with Zimbra: +**Summary:** Always expect the unexpected -* I have experiences that cancelling participation in an event caused the event to be cancelled for all participants (even if the person deciding to not go to the event was not an organizer and should have no permissions to edit the event). Clearly a server-side issue. -* I once tried to restore from backup and push ten years of ical code to the calendar server. The calendar server responded by re-inviting people to the meetings we had ten years ago. I'm inclined to call that also a server side bug. -* Many other things may happen. +Weird things may happen due to bugs both on in the CalDAV package, on your side and on the server side. Some anecdotes from using Zimbra: -## Malicious usage +* I once tried to restore from backup and push ten years of ical code to the calendar server. The calendar server responded by re-inviting people to the meetings we had ten years ago. I'm inclined to call that a server side bug - but it also highlights the risk of using the CalDAV library for doing operations that ordinary calendaring clients aren't doing. +* It's been observed that cancelling participation in an event caused the event to be cancelled for all participants (even if the person deciding to not go to the event was not an organizer and should have no permissions to edit the event). Clearly a server-side issue. -Beware of risks and exposure when creating applications: +## Other things to consider -* Your code may handle username and password, be careful not to expose such credentials. Even the URL to the calendar server and/or calendar may be something people want to keep private. +**Summary:** Beware of risks and exposure when creating applications: + +* Your code may handle username and password, be careful not to expose such credentials. Even the URL to the calendar server and/or calendar may be something people want to keep private. The library includes code for reading this data from a standard config file - please use it rather than reinventing the wheel or hard-coding credentials directly into your code. * Consider that calendar events and such is personal data, which deserves protection. In the EU with the GDPR, such protection is even mandated by law. * If you allow arbitrary people to create calendar content to be saved to a server, there may be some risks involved: * Depending on the server implementation, it may be possible to use the caldav library for sending spam emails. * Be aware of DoS-attacks: By storing too much / too big / specially crafted icalendar data, the server and/or client may crash or consume all available resources. - * If allowing anonymous parties to save and retrieve data from your server, you may end up with responsibility for spreading illicit information. This may include things like child porn. Political or religious propaganda may be legitimate and legal in some countries, but may involve death penalty in other countries. Your calendar server may also be used for coordinating criminal activity. -* If you allow arbitrary people to fetch calendar content from the server, there may also be some risks involved - in particular, a DoS-attack by requesting a large time span of expanded events. + * If allowing anonymous parties to save and retrieve data from your server, you may end up with responsibility for spreading illicit information. This may include things like child sexual abuse material. Political or religious propaganda may be legitimate and legal in some countries, but may involve death penalty in other countries. Your calendar server may also be used for coordinating criminal activity. +* If you allow arbitrary people to fetch calendar content from the server, there may also be some risks involved - see the separate section on DoS-attack by requesting a large time span of expanded events. + +## Supply attack risk -## Malicious code +**Summary:** Stick to released versions and check the PGP signature in the release-tag -All code contributions are carefully reviewed by Tobias Brox. Version tags are signed with PGP. Of course there is always a risk that someone takes over my PGP key and github access (It's hard to be immune against a [5$ wrench attack](https://xkcd.com/538/)). The original owner of the repository is still alive and may take over the project again should something happen to me. I would anyway encourage using AI to do risk assessments. +All code contributions are carefully reviewed by Tobias Brox. Version tags are signed with PGP. Of course there is always a risk that someone takes over my PGP key and GitHub access (It's hard to be immune against a [$5 wrench attack](https://xkcd.com/538/)). The original owner of the repository is still alive and may take over the project again should something happen to me. I would encourage using AI to do risk assessments. The library comes with a number of dependencies, one may need to evaluate the security of those too. The pyproject contains the current list. Some notes: * niquests is an optional dependency - you may replace it with requests if you don't trust niquests -* recurring-ical-events and icalendar both has the same maintainer (Nicco Kunzmann). He is considered trustworthy. -* Tobias now has a policy of moving code not related to CalDAV into separate packages. Packages under the `python-caldav` ownership on GitHub should be considered to be of the same quality and security level as the CalDAV library. -* No security review have been done of the other dependencies. +* recurring-ical-events and icalendar both have the same maintainer (Nicco Kunzmann). He is considered trustworthy. +* Tobias now has a policy of moving code not related to CalDAV into separate packages. Those packages are most of the time either published under the `python-caldav` or `pycalendar` ownership on GitHub, and should be considered to be of the same quality and security level as the CalDAV library. +* No independent security review has been done of the other dependencies - those are all considered to be mature and robust projects. + +## Communication dumper debug hook + +**Summary:** If someone has the ability to both alter the environment and full read access to /tmp (basically, someone has root access to the computer where the code is run), it will be possible to get access to all communication. Also, anyone using this debug hook must take responsibility of deleting the dumped files. + +**Mitigation:** If this worries you, set `caldav.lib.error.debug_dump_communication=False` after importing caldav. + +The following was written when `PYTHON_CALDAV_COMMDUMP` was introduced in v1.4.0: + +* An attacker that has access to alter the environment the application is running under may cause a DoS-attack, filling up available disk space with debug logging. +* An attacker that has access to alter the environment the application is running under, and access to read files under /tmp (files being 0600 and owned by the uid the application is running under), will be able to read the communication between the server and the client, communication that may be private and confidential. + +Thinking it through three times, I'm not too concerned — if someone has access to alter the environment the process is running under and access to read files run by the uid of the application, then this someone should already be trusted and will probably have the possibility to DoS the system or gather this communication through other means. + +As of v3.3 (to be released towards the end of 2026-06), a warning is logged at import time when this variable is set, reminding the operator that request/response bodies and headers (including credentials and calendar PII) are written to uniquely-named files under `/tmp` that accumulate indefinitely. diff --git a/caldav/async_davclient.py b/caldav/async_davclient.py index 2c4bd006..fa2d1476 100644 --- a/caldav/async_davclient.py +++ b/caldav/async_davclient.py @@ -7,6 +7,8 @@ """ import asyncio +import importlib +import inspect import logging import sys from collections.abc import Mapping @@ -18,11 +20,47 @@ from caldav.calendarobjectresource import CalendarObjectResource from caldav.collection import Calendar, Principal -# Try niquests first (preferred), then httpxyz, then httpx +## Async HTTP libraries, in order of preference. niquests is the default and +## the one this project depends on; it is also the only one of these with HTTP/3. +## The rest are the httpx family and share a single API, so one code path covers +## all of them. httpx2 is Pydantic's continuation of httpx under a new name and +## comes first of the three; httpxyz is a maintained community fork of httpx; +## plain httpx is last. Note that httpxyz registers itself in sys.modules as +## "httpx" while httpx2 does not, so code must go through the module object +## below rather than importing httpx by name. +## ref https://github.com/python-caldav/caldav/issues/611 +_ASYNC_HTTPX_CANDIDATES = ("httpx2", "httpxyz", "httpx") + +_NO_ASYNC_LIBRARY_ERROR = ( + "An async HTTP library is required for async_davclient. Install one of: " + + ", ".join(f"pip install {name}" for name in ("niquests", *_ASYNC_HTTPX_CANDIDATES)) + + " (niquests is recommended)" +) + _USE_HTTPX = False _USE_HTTPXYZ = False _USE_NIQUESTS = False _H2_AVAILABLE = False +_HTTPX_FLAVOUR: str | None = None + + +def _import_first_available( + candidates: tuple[str, ...], + importer: Any = importlib.import_module, +) -> tuple[str | None, Any]: + """Import the first importable module named in candidates. + + Returns (name, module), or (None, None) when none of them is installed. + The importer is injectable so the ordering can be tested without having to + install or uninstall anything. + """ + for name in candidates: + try: + return name, importer(name) + except ImportError: + continue + return None, None + try: import niquests @@ -34,11 +72,14 @@ pass if not _USE_NIQUESTS: - try: - import httpxyz as httpx - - _USE_HTTPXYZ = True + _HTTPX_FLAVOUR, httpx = _import_first_available(_ASYNC_HTTPX_CANDIDATES) + if _HTTPX_FLAVOUR is not None: _USE_HTTPX = True + ## Nothing in this module reads _USE_HTTPXYZ; it exists for the + ## `async (httpxyz fallback)` CI job, which imports it to assert that the + ## fallback it set up is the one actually in use. _HTTPX_FLAVOUR carries + ## the same information for anything new. + _USE_HTTPXYZ = _HTTPX_FLAVOUR == "httpxyz" try: import h2 # noqa: F401 @@ -47,32 +88,7 @@ pass class _HttpxBearerAuth(httpx.Auth): - """httpx/httpxyz-compatible bearer token auth.""" - - def __init__(self, password: str) -> None: - self.password = password - - def auth_flow(self, request): - request.headers["Authorization"] = f"Bearer {self.password}" - yield request - - except ImportError: - pass - -if not _USE_NIQUESTS and not _USE_HTTPXYZ: - try: - import httpx - - _USE_HTTPX = True - try: - import h2 # noqa: F401 - - _H2_AVAILABLE = True - except ImportError: - pass - - class _HttpxBearerAuth(httpx.Auth): # type: ignore[no-redef] - """httpx-compatible bearer token auth.""" + """Bearer token auth for the httpx family (httpx, httpxyz, httpx2).""" def __init__(self, password: str) -> None: self.password = password @@ -81,15 +97,9 @@ def auth_flow(self, request): request.headers["Authorization"] = f"Bearer {self.password}" yield request - except ImportError: - pass if not _USE_HTTPX and not _USE_NIQUESTS: - raise ImportError( - "An async HTTP library is required for async_davclient. " - "Install with: pip install niquests (or: pip install httpxyz or: pip install httpx)" - ) - + raise ImportError(_NO_ASYNC_LIBRARY_ERROR) from caldav import __version__ from caldav.base_client import BaseDAVClient @@ -163,6 +173,9 @@ def __init__( features: FeatureSet for server compatibility workarounds. enable_rfc6764: Enable RFC6764 DNS-based service discovery. require_tls: Require TLS for discovered services (security consideration). + Only gates the RFC6764 discovery path; it does NOT reject an + explicitly-passed http:// URL. Global enforcement is deferred to + 4.0 — see https://github.com/python-caldav/caldav/issues/687 rate_limit_handle: When True, automatically sleep and retry on 429/503 responses. When None (default), auto-detected from server features. When False, raise RateLimitError immediately. @@ -223,18 +236,21 @@ def __init__( # Parse and store URL self.url = URL.objectify(url_str) - # Extract auth from URL if present - url_username = None - url_password = None - if self.url.username: - url_username = unquote(self.url.username) - if self.url.password: - url_password = unquote(self.url.password) - - # Combine credentials (explicit params take precedence) - # Use explicit None check to preserve empty strings (needed for servers with no auth) - self.username = username if username is not None else url_username - self.password = password if password is not None else url_password + # Combine credentials (explicit params take precedence). + # An explicit username discards the URL credentials wholesale: they + # belong to a different account, and merging them field by field would + # let AsyncDAVClient(url="https://bob:hunter2@cal.example.com/", + # username="alice") ship alice's login with bob's password. Overriding + # only the password is a different thing - the username still comes + # from the URL, so the pair stays coherent. + # Use explicit None checks to preserve empty strings (needed for + # servers with no auth). + if self.url.username and username is None: + username = unquote(self.url.username) + if password is None and self.url.password: + password = unquote(self.url.password) + self.username = username + self.password = password # Strip credentials from stored URL to avoid leaking them in log messages self.url = self.url.unauth() @@ -258,19 +274,9 @@ def __init__( } self.headers.update(headers) - rate_limit = self.features.is_supported("rate-limit", dict) - if rate_limit_handle is None: - if rate_limit and rate_limit.get("enable"): - rate_limit_handle = True - if "default_sleep" in rate_limit: - rate_limit_default_sleep = rate_limit["default_sleep"] - if "max_sleep" in rate_limit: - rate_limit_max_sleep = rate_limit["max_sleep"] - else: - rate_limit_handle = False - self.rate_limit_handle = rate_limit_handle - self.rate_limit_default_sleep = rate_limit_default_sleep - self.rate_limit_max_sleep = rate_limit_max_sleep + self._init_rate_limit_config( + rate_limit_handle, rate_limit_default_sleep, rate_limit_max_sleep + ) def _create_session(self) -> None: """Create or recreate the async HTTP client with current settings.""" @@ -365,20 +371,7 @@ async def request( try: return await self._async_request(url, method, body, headers) except error.RateLimitError as e: - if not self.rate_limit_handle: - raise - sleep_seconds = error.compute_sleep_seconds( - e.retry_after_seconds, - self.rate_limit_default_sleep, - self.rate_limit_max_sleep, - ) - if rate_limit_time_slept: - sleep_seconds += rate_limit_time_slept / 2 - if sleep_seconds is None or ( - self.rate_limit_max_sleep is not None - and rate_limit_time_slept > self.rate_limit_max_sleep - ): - raise + sleep_seconds = self._rate_limit_sleep_seconds(e, rate_limit_time_slept) await asyncio.sleep(sleep_seconds) return await self.request( url, method, body, headers, rate_limit_time_slept + sleep_seconds @@ -432,7 +425,7 @@ async def _async_request( log.debug(f"server responded with {r.status_code} {reason}") if ( r.status_code == 401 - and "text/html" in self.headers.get("Content-Type", "") + and "text/html" in r.headers.get("Content-Type", "") and not self.auth ): msg = ( @@ -484,7 +477,11 @@ async def _async_request( # Retry original request with auth request_kwargs["auth"] = self.auth r = await self.session.request(**request_kwargs) - response = DAVResponse(r, self) + response = DAVResponse(r, self) + else: + # Probe GET did not give us a 401+WWW-Authenticate challenge — + # auth negotiation failed; re-raise the original connection error + raise # Handle 429/503 rate-limit responses error.raise_if_rate_limited(r.status_code, str(url_obj), r.headers.get("Retry-After")) @@ -936,14 +933,6 @@ async def get_calendars(self, principal: Optional["Principal"] = None) -> list[" for cal in calendars: print(f"Calendar: {cal.get_display_name()}") """ - from caldav.collection import Calendar - from caldav.collection import ( - _extract_calendar_home_set_from_results as extract_home_set, - ) - from caldav.collection import ( - _extract_calendars_from_propfind_results as extract_calendars, - ) - if principal is None: principal = await self.get_principal() @@ -953,12 +942,7 @@ async def get_calendars(self, principal: Optional["Principal"] = None) -> list[" props=self.CALENDAR_HOME_SET_PROPS, depth=0, ) - calendar_home_url = extract_home_set(response.results) - if not calendar_home_url: - return [] - - # Make URL absolute if relative - calendar_home_url = self._make_absolute_url(calendar_home_url) + calendar_home_url = self._calendar_home_url(response, principal) # Fetch calendars via PROPFIND response = await self.propfind( @@ -967,14 +951,7 @@ async def get_calendars(self, principal: Optional["Principal"] = None) -> list[" depth=1, ) - # Process results using shared helper - calendar_infos = extract_calendars(response.results) - - # Convert CalendarInfo objects to Calendar objects - return [ - Calendar(client=self, url=info.url, name=info.name, id=info.cal_id) - for info in calendar_infos - ] + return self._build_calendars_from_propfind(response) async def search_calendar( self, @@ -1222,7 +1199,11 @@ async def get_calendars( for cal in calendars: print(await cal.get_display_name()) """ - from caldav.base_client import CalendarCollection, _normalize_to_list + from caldav.base_client import ( + CalendarCollection, + _normalize_to_list, + _warn_unreadable_display_name, + ) def _try(coro_result, errmsg): """Handle errors based on raise_errors flag.""" @@ -1256,6 +1237,12 @@ def _try(coro_result, errmsg): calendar = principal.calendar(cal_url=cal_url) else: calendar = principal.calendar(cal_id=cal_url) + ## A bare cal_id has to resolve the calendar home set first, so + ## principal.calendar() hands back a coroutine here. Without the + ## await the AttributeError below was caught by the broad except + ## and the calendar was silently dropped from the collection. + if inspect.isawaitable(calendar): + calendar = await calendar try: display_name = await calendar.get_display_name() @@ -1267,13 +1254,31 @@ def _try(coro_result, errmsg): raise # Fetch specific calendars by name - for cal_name in calendar_names: + if calendar_names: try: - calendar = await principal.calendar(name=cal_name) - if calendar: - calendars.append(calendar) + all_cals_for_name = await principal.get_calendars() + for cal_name in calendar_names: + for cal in all_cals_for_name: + try: + display_name = await cal.get_display_name() + if display_name == cal_name: + calendars.append(cal) + break + except Exception as e: + # Skip calendars whose display name can't be read; warn + # only when the failure is unexpected (see helper). + # Continuing ensures one unreadable calendar doesn't abort + # the whole name lookup. + _warn_unreadable_display_name(client, cal, cal_name, e) + continue + else: + log.error(f"No calendar with name '{cal_name}' found") + if raise_errors: + raise error.NotFoundError(f"No calendar with name '{cal_name}' found") + except error.NotFoundError: + raise except Exception as e: - log.error(f"Problems fetching calendar by name '{cal_name}': {e}") + log.error(f"Problems fetching calendars by name: {e}") if raise_errors: raise diff --git a/caldav/base_client.py b/caldav/base_client.py index d1974bb7..5186a3c7 100644 --- a/caldav/base_client.py +++ b/caldav/base_client.py @@ -255,6 +255,99 @@ def _raise_authorization_error(self, url_str: str, reason_source: Any) -> NoRetu reason = "None given" raise error.AuthorizationError(url=url_str, reason=reason) + # ── Rate-limit handling ───────────────────────────────────────────────── + # Shared by the sync (DAVClient) and async (AsyncDAVClient) __init__ and + # request() retry loops, which are otherwise byte-identical apart from + # time.sleep vs asyncio.sleep. + + def _init_rate_limit_config( + self, + rate_limit_handle: bool | None, + rate_limit_default_sleep: int | None, + rate_limit_max_sleep: int | None, + ) -> None: + """Resolve and store rate-limit settings on self. + + When ``rate_limit_handle`` is None it is auto-detected from the + ``rate-limit`` feature; an enabled feature may also supply default and + max sleep durations (explicit constructor arguments are not overridden + because the feature values only fill in the auto-detected branch). + """ + rate_limit = self.features.is_supported("rate-limit", dict) + if rate_limit_handle is None: + if rate_limit and rate_limit.get("enable"): + rate_limit_handle = True + if "default_sleep" in rate_limit: + rate_limit_default_sleep = rate_limit["default_sleep"] + if "max_sleep" in rate_limit: + rate_limit_max_sleep = rate_limit["max_sleep"] + else: + rate_limit_handle = False + self.rate_limit_handle = rate_limit_handle + self.rate_limit_default_sleep = rate_limit_default_sleep + self.rate_limit_max_sleep = rate_limit_max_sleep + + def _rate_limit_sleep_seconds( + self, + exc: error.RateLimitError, + rate_limit_time_slept: float, + ) -> float: + """Decide how long to sleep before retrying a rate-limited request. + + Returns the sleep duration in seconds. Re-raises ``exc`` when no retry + should happen (rate-limit handling disabled, no usable duration, or the + accumulated sleep already exceeds ``rate_limit_max_sleep``). The caller + is responsible for actually sleeping (time.sleep vs asyncio.sleep) and + retrying with ``rate_limit_time_slept + ``. + """ + if not self.rate_limit_handle: + raise exc + sleep_seconds = error.compute_sleep_seconds( + exc.retry_after_seconds, + self.rate_limit_default_sleep, + self.rate_limit_max_sleep, + ) + if sleep_seconds is None or ( + self.rate_limit_max_sleep is not None + and rate_limit_time_slept > self.rate_limit_max_sleep + ): + raise exc + if rate_limit_time_slept: + sleep_seconds += rate_limit_time_slept / 2 + return sleep_seconds + + # ── Calendar discovery post-processing ────────────────────────────────── + # Pure result-handling shared by sync/async get_calendars; only the two + # awaited PROPFIND calls and the principal lookup differ between the twins. + + def _calendar_home_url(self, home_set_response: Any, principal: Any) -> str: + """Extract the calendar-home-set URL from a PROPFIND response. + + Falls back to the principal URL when the server does not advertise a + calendar-home-set (e.g. GMX), then makes the result absolute. + """ + from caldav.collection import ( + _extract_calendar_home_set_from_results as extract_home_set, + ) + + calendar_home_url = extract_home_set(home_set_response.results) + if not calendar_home_url: + calendar_home_url = str(principal.url) + return self._make_absolute_url(calendar_home_url) + + def _build_calendars_from_propfind(self, list_response: Any) -> list: + """Build Calendar objects from a calendar-home PROPFIND response.""" + from caldav.collection import Calendar + from caldav.collection import ( + _extract_calendars_from_propfind_results as extract_calendars, + ) + + calendar_infos = extract_calendars(list_response.results) + return [ + Calendar(client=self, url=info.url, name=info.name, id=info.cal_id) + for info in calendar_infos + ] + # ── XML builders ────────────────────────────────────────────────────────── # All methods are static: no I/O, no server interaction, pure data # transformation. Both DAVClient and AsyncDAVClient inherit these so @@ -645,6 +738,32 @@ def _normalize_to_list(obj: Any) -> list: return list(obj) +def _warn_unreadable_display_name(client: Any, calendar: Any, name: Any, exc: Exception) -> None: + """Log a warning when a calendar's display name couldn't be read during a + lookup by name -- unless the failure is expected per the compatibility matrix. + + Shared by the sync (:meth:`caldav.collection.CalendarSet.calendar`) and async + (:func:`caldav.async_davclient.get_calendars`) name-matching loops so the + warn-or-suppress decision lives in one place. + + The failure is treated as expected (and silently skipped) only when we + positively know the server doesn't support reading the DAV:displayname + property via PROPFIND (``propfind.displayname`` non-supported -- which falls + back to the ``propfind`` parent when not probed explicitly). When the + feature is supported, or when we have no feature matrix to consult, the + failure is unexpected and is warned about. + + The caller is responsible for continuing the loop afterwards, so that one + unreadable calendar never aborts the whole name lookup. + """ + features = getattr(client, "features", None) + if features is None or features.is_supported("propfind.displayname"): + log.warning( + f"Could not read display name for calendar " + f"{getattr(calendar, 'url', calendar)} while matching name '{name}': {exc}" + ) + + def _fetch_calendars_for_client( client: Any, calendar_url: Any | None, @@ -686,7 +805,7 @@ def _try(meth, kwargs, errmsg): calendar = principal.calendar(cal_url=cal_url) else: calendar = principal.calendar(cal_id=cal_url) - if _try(calendar.get_display_name, {}, f"calendar {cal_url}"): + if _try(calendar.get_display_name, {}, f"calendar {cal_url}") is not None: calendars.append(calendar) for cal_name in calendar_names: diff --git a/caldav/calendarobjectresource.py b/caldav/calendarobjectresource.py index a0f33976..153af940 100644 --- a/caldav/calendarobjectresource.py +++ b/caldav/calendarobjectresource.py @@ -722,7 +722,7 @@ def add_attendee(self, attendee, no_default_parameters: bool = False, **paramete raise NotImplementedError( "do we need to support this anyway? Should be trivial, but can't figure out how to do it with the icalendar.Event/vCalAddress objects right now" ) - elif attendee.startswith("mailto:"): + elif attendee.lower().startswith("mailto:"): attendee_obj = vCalAddress(attendee) elif "@" in attendee and ":" not in attendee and ";" not in attendee: attendee_obj = vCalAddress("mailto:" + attendee) @@ -1000,11 +1000,7 @@ def load(self, only_if_unloaded: bool = False) -> "Self | Coroutine[Any, Any, Se except Exception: return self.load_by_multiget() - ## consider refactoring - this is repeated many places now - if "Etag" in r.headers: - self.props[dav.GetEtag.tag] = r.headers["Etag"] - if "Schedule-Tag" in r.headers: - self.props[cdav.ScheduleTag.tag] = r.headers["Schedule-Tag"] + self._update_tag_props(r) return self async def _async_load(self, only_if_unloaded: bool = False) -> Self: @@ -1046,10 +1042,7 @@ async def _async_load(self, only_if_unloaded: bool = False) -> Self: except Exception: return await self.load_by_multiget() - if "Etag" in r.headers: - self.props[dav.GetEtag.tag] = r.headers["Etag"] - if "Schedule-Tag" in r.headers: - self.props[cdav.ScheduleTag.tag] = r.headers["Schedule-Tag"] + self._update_tag_props(r) return self def load_by_multiget(self) -> "Self | Coroutine[Any, Any, Self]": @@ -1155,6 +1148,20 @@ async def _async_put(self, headers, retry_on_failure=True): # _post_put returned a retry coroutine (self._put(False) for async client) await result + def _update_tag_props(self, r) -> None: + """Capture the ETag / Schedule-Tag response headers into self.props. + + Called after both PUT (`_post_put`) and GET (`load`/`_async_load`); + keys are matched case-insensitively by the response header dict. + See RFC 6638 for Schedule-Tag. + """ + if not r.headers: + return + if "Etag" in r.headers: + self.props[dav.GetEtag.tag] = r.headers["Etag"] + if r.headers.get("Schedule-Tag"): + self.props[cdav.ScheduleTag.tag] = r.headers["Schedule-Tag"] + def _post_put(self, r, retry_on_failure): if r.status == 412: if self.schedule_tag: @@ -1164,7 +1171,7 @@ def _post_put(self, r, retry_on_failure): else: raise error.PutError(errmsg(r)) elif r.status == 302: - self.url = URL.objectify([x[1] for x in r.headers if x[0] == "location"][0]) + self.url = URL.objectify(r.headers.get("location")) elif r.status not in (204, 201): if retry_on_failure: try: @@ -1178,31 +1185,7 @@ def _post_put(self, r, retry_on_failure): return self._put(False) else: raise error.PutError(errmsg(r)) - if "Etag" in r.headers: - self.props[dav.GetEtag.tag] = r.headers["Etag"] - if r.headers and r.headers.get("schedule-tag"): - self.props[cdav.ScheduleTag.tag] = r.headers["schedule-tag"] - - if r.status == 302: - path = [x[1] for x in r.headers if x[0] == "location"][0] - self.url = URL.objectify(path) - elif r.status not in (204, 201): - if retry_on_failure: - try: - import vobject # noqa: F401 - except ImportError: - retry_on_failure = False - if retry_on_failure: - ## This seems like a noop, but it may "wash" the object - dummy = self.vobject_instance - return self._put(False) - else: - raise error.PutError(errmsg(r)) - ## TODO: refactor - those code lines are repeated all over the place - if "Etag" in r.headers: - self.props[dav.GetEtag.tag] = r.headers["Etag"] - if r.headers and r.headers.get("schedule-tag"): - self.props[cdav.ScheduleTag.tag] = r.headers["schedule-tag"] + self._update_tag_props(r) def _create( self, id=None, path=None, retry_on_failure=True @@ -1269,7 +1252,12 @@ def change_attendee_status(self, attendee: Any | None = None, **kwargs) -> None: return ical_obj = self.icalendar_component - attendee_lines = ical_obj["attendee"] + try: + attendee_lines = ical_obj["attendee"] + except KeyError: + raise error.NotFoundError( + f"Participant {attendee!r} not found in attendee list (no ATTENDEE properties)" + ) from None if isinstance(attendee_lines, str): attendee_lines = [attendee_lines] @@ -1281,7 +1269,7 @@ def strip_mailto(x): attendee_line.params.update(kwargs) cnt += 1 if not cnt: - raise error.NotFoundError("Participant %s not found in attendee list") + raise error.NotFoundError(f"Participant {attendee!r} not found in attendee list") error.assert_(cnt == 1) def save( @@ -1302,7 +1290,11 @@ def save( obj_type is only used in conjunction with no_overwrite and no_create. - is_schedule_tag_match is currently ignored. (TODO - fix or remove) + Conditional requests are handled automatically: if a + Schedule-Tag or an ETag has been cached for the object, an + ``If-Schedule-Tag-Match`` or ``If-Match`` header is sent, and a + 412 response is raised as ``ScheduleTagMismatchError`` or + ``ETagMismatchError``. The SEQUENCE should be increased when saving a new version of the object. If this behaviour is unwanted, then @@ -1570,6 +1562,7 @@ def _set_data(self, data): self._data = vcal.fix(data) self._vobject_instance = None self._icalendar_instance = None + self._state = RawDataState(self._data) return self def _get_data(self): @@ -1684,7 +1677,7 @@ def _get_icalendar_instance(self): if not self._icalendar_instance: if not self.data: return None - self.icalendar_instance = icalendar.Calendar.from_ical(to_unicode(self.data)) + self.icalendar_instance = vcal.parse_ical(to_unicode(self.data), context=self.url) return self._icalendar_instance icalendar_instance: Any = property( @@ -1940,7 +1933,7 @@ def _get_duration(self, i): start = datetime(start.year, start.month, start.day) end = datetime(end.year, end.month, end.day) return end - start - elif "DTSTART" in i and not isinstance(i["DTSTART"], datetime): + elif "DTSTART" in i and not isinstance(i["DTSTART"].dt, datetime): return timedelta(days=1) else: return timedelta(0) @@ -2119,55 +2112,54 @@ def _reduce_count(self, i=None) -> bool: i["RRULE"]["COUNT"][0] -= 1 return True - def _complete_recurring_safe(self, completion_timestamp): - """This mode will create a new independent task which is - marked as completed, and modify the existing recurring task. - It is probably the most safe way to handle the completion of a - recurrence of a recurring task, though the link between the - completed task and the original task is lost. + def _build_recurring_safe_completed(self, completion_timestamp) -> "Todo | None": + """Pure (no-I/O) part of the "safe" recurring-completion strategy. + + Advances ``self`` to its next occurrence in memory and returns a + freshly-built standalone copy marked as completed. Returns + ``None`` when the task is not (or no longer) recurring, in which + case the caller should fall back to a plain completion. The + caller is responsible for saving both ``self`` and the returned + copy (one PUT each). """ ## If count is one, then it is not really recurring if not self._reduce_count(): - return self.complete(handle_rrule=False) + return None next_dtstart = self._next(completion_timestamp) if not next_dtstart: - return self.complete(handle_rrule=False) + return None completed = self.copy() completed.url = self.parent.url.join(completed.id + ".ics") completed.icalendar_component.pop("RRULE") - completed.save() - completed.complete() + completed._complete_ical(completion_timestamp=completion_timestamp) duration = self.get_duration() i = self.icalendar_component i.pop("DTSTART", None) i.add("DTSTART", next_dtstart) self.set_duration(duration, movable_attr="DUE") + return completed + def _complete_recurring_safe(self, completion_timestamp): + """This mode will create a new independent task which is + marked as completed, and modify the existing recurring task. + It is probably the most safe way to handle the completion of a + recurrence of a recurring task, though the link between the + completed task and the original task is lost. + """ + completed = self._build_recurring_safe_completed(completion_timestamp) + if completed is None: + self.complete(handle_rrule=False) + return + completed.save() self.save() - def _complete_recurring_thisandfuture(self, completion_timestamp) -> None: - """The RFC is not much helpful, a lot of guesswork is needed - to consider what the "right thing" to do wrg of a completion of - recurring tasks is ... but this is my shot at it. - - 1) The original, with rrule, will be kept as it is. The rrule - string is fetched from the first subcomponent of the - icalendar. - - 2) If there are multiple recurrence instances in subcomponents - and the last one is marked with RANGE=THISANDFUTURE, then - select this one. If it has the rrule property set, use this - rrule rather than the original one. Drop the RANGE parameter. - Calculate the next RECURRENCE-ID from the DTSTART of this - object. Mark task as completed. Increase SEQUENCE. - - 3) Create a new recurrence instance with RANGE=THISANDFUTURE, - without RRULE set (Ref - https://github.com/Kozea/Radicale/issues/1264). Set the - RECURRENCE-ID to the one calculated in #2. Calculate the - DTSTART based on rrule and completion timestamp/date. + def _prepare_recurring_thisandfuture(self, completion_timestamp) -> None: + """Pure (no-I/O) in-memory mutation behind + ``_complete_recurring_thisandfuture``; see that method for the + algorithm description. The caller does the single + ``save(increase_seqno=False)`` that follows. """ recurrences = self.icalendar_instance.subcomponents orig = recurrences[0] @@ -2219,7 +2211,6 @@ def _complete_recurring_thisandfuture(self, completion_timestamp) -> None: [x for x in recurrences if not self.is_pending(x)] ): self._complete_ical(recurrences[0], completion_timestamp=completion_timestamp) - self.save(increase_seqno=False) return rrule = rrule2 or rrule @@ -2231,6 +2222,30 @@ def _complete_recurring_thisandfuture(self, completion_timestamp) -> None: thisandfuture.add("DTSTART", next_dtstart) self._set_duration(i=thisandfuture, duration=duration, movable_attr="DUE") self.icalendar_instance.subcomponents.append(thisandfuture) + + def _complete_recurring_thisandfuture(self, completion_timestamp) -> None: + """The RFC is not much helpful, a lot of guesswork is needed + to consider what the "right thing" to do wrg of a completion of + recurring tasks is ... but this is my shot at it. + + 1) The original, with rrule, will be kept as it is. The rrule + string is fetched from the first subcomponent of the + icalendar. + + 2) If there are multiple recurrence instances in subcomponents + and the last one is marked with RANGE=THISANDFUTURE, then + select this one. If it has the rrule property set, use this + rrule rather than the original one. Drop the RANGE parameter. + Calculate the next RECURRENCE-ID from the DTSTART of this + object. Mark task as completed. Increase SEQUENCE. + + 3) Create a new recurrence instance with RANGE=THISANDFUTURE, + without RRULE set (Ref + https://github.com/Kozea/Radicale/issues/1264). Set the + RECURRENCE-ID to the one calculated in #2. Calculate the + DTSTART based on rrule and completion timestamp/date. + """ + self._prepare_recurring_thisandfuture(completion_timestamp) self.save(increase_seqno=False) def complete( @@ -2284,85 +2299,15 @@ async def _async_complete( async def _async_complete_recurring_safe(self, completion_timestamp: datetime) -> None: """Async version of _complete_recurring_safe.""" - if not self._reduce_count(): + completed = self._build_recurring_safe_completed(completion_timestamp) + if completed is None: return await self._async_complete(completion_timestamp, handle_rrule=False) - next_dtstart = self._next(completion_timestamp) - if not next_dtstart: - return await self._async_complete(completion_timestamp, handle_rrule=False) - - completed = self.copy() - completed.url = self.parent.url.join(completed.id + ".ics") - completed.icalendar_component.pop("RRULE") await completed.save() - completed._complete_ical(completion_timestamp=completion_timestamp) - await completed.save() - - duration = self.get_duration() - i = self.icalendar_component - i.pop("DTSTART", None) - i.add("DTSTART", next_dtstart) - self.set_duration(duration, movable_attr="DUE") await self.save() async def _async_complete_recurring_thisandfuture(self, completion_timestamp: datetime) -> None: """Async version of _complete_recurring_thisandfuture.""" - recurrences = self.icalendar_instance.subcomponents - orig = recurrences[0] - if "STATUS" not in orig: - orig["STATUS"] = "NEEDS-ACTION" - - if len(recurrences) == 1: - just_completed = orig.copy() - just_completed.pop("RRULE") - just_completed.add("RECURRENCE-ID", orig.get("DTSTART", completion_timestamp)) - seqno = just_completed.pop("SEQUENCE", 0) - just_completed.add("SEQUENCE", seqno + 1) - recurrences.append(just_completed) - - prev = recurrences[-1] - rrule = prev.get("RRULE", orig["RRULE"]) - thisandfuture = prev.copy() - seqno = thisandfuture.pop("SEQUENCE", 0) - thisandfuture.add("SEQUENCE", seqno + 1) - - if len(recurrences) > 2: - if prev["RECURRENCE-ID"].params.get("RANGE", None) == "THISANDFUTURE": - prev["RECURRENCE-ID"].params.pop("RANGE") - else: - raise NotImplementedError( - "multiple instances found, but last one is not of type THISANDFUTURE, possibly this has been created by some incompatible client, but we should deal with it" - ) - self._complete_ical(prev, completion_timestamp) - - thisandfuture.pop("RECURRENCE-ID", None) - thisandfuture.add("RECURRENCE-ID", self._next(i=prev, rrule=rrule)) - thisandfuture["RECURRENCE-ID"].params["RANGE"] = "THISANDFUTURE" - rrule2 = thisandfuture.pop("RRULE", None) - - if rrule2 is not None: - count = rrule2.get("COUNT", None) - if count is not None and count[0] in (0, 1): - for i in recurrences: - self._complete_ical(i, completion_timestamp=completion_timestamp) - thisandfuture.add("RRULE", rrule2) - else: - count = rrule.get("COUNT", None) - if count is not None and count[0] <= len( - [x for x in recurrences if not self.is_pending(x)] - ): - self._complete_ical(recurrences[0], completion_timestamp=completion_timestamp) - await self.save(increase_seqno=False) - return - - rrule = rrule2 or rrule - - duration = self._get_duration(i=prev) - thisandfuture.pop("DTSTART", None) - thisandfuture.pop("DUE", None) - next_dtstart = self._next(i=prev, rrule=rrule, ts=completion_timestamp) - thisandfuture.add("DTSTART", next_dtstart) - self._set_duration(i=thisandfuture, duration=duration, movable_attr="DUE") - self.icalendar_instance.subcomponents.append(thisandfuture) + self._prepare_recurring_thisandfuture(completion_timestamp) await self.save(increase_seqno=False) def _complete_ical(self, i=None, completion_timestamp=None) -> None: diff --git a/caldav/collection.py b/caldav/collection.py index 37b87b96..edef522b 100644 --- a/caldav/collection.py +++ b/caldav/collection.py @@ -10,6 +10,7 @@ A SynchronizableCalendarObjectCollection contains a local copy of objects from a calendar on the server. """ +import inspect import logging import uuid import warnings @@ -31,7 +32,7 @@ from collections.abc import Coroutine, Iterable, Iterator, Sequence from typing import Literal -from .base_client import ICALH +from .base_client import ICALH, _warn_unreadable_display_name from .calendarobjectresource import ( CalendarObjectResource, Event, @@ -78,6 +79,19 @@ def _extract_calendar_id_from_url(url: str) -> str | None: return None +def _safe_display_name(cal) -> str | None: + """Return ``cal``'s DAV:displayname, or None if it can't be read. + + Used when discovering a relocated calendar's canonical URL after creation + (see Calendar._adopt_canonical_url); a calendar that refuses to report its + display name simply isn't a match. + """ + try: + return cal.get_display_name() + except Exception: + return None + + def _quote_url_path(url: str) -> str: """Quote the path component of a URL to handle unencoded spaces (e.g. Zimbra).""" parsed = urlparse(url) @@ -184,6 +198,43 @@ def calendars(self) -> "list[Calendar] | Coroutine[Any, Any, list[Calendar]]": """ return self.get_calendars() + def _find_calendar_by_name( + self, calendars: "list[tuple[Calendar, str | None]]", name: str + ) -> "Calendar": + """Pick the calendar whose display name is ``name``. + + The display names are supplied already resolved, so that the sync and + async twins of :meth:`calendar` can share the matching and the error. + """ + for calendar, display_name in calendars: + if display_name == name: + return calendar + raise error.NotFoundError(f"No calendar with name {name} found under {self.url}") + + def _first_calendar(self, calendars: "list[Calendar]") -> "Calendar": + """Return the first calendar, or raise if there are none.""" + if not calendars: + raise error.NotFoundError("no calendars found") + return calendars[0] + + async def _async_calendar(self, name: str | None = None) -> "Calendar": + """Async twin of :meth:`calendar` for the lookups that need the server.""" + calendars = await self.get_calendars() + if not name: + return self._first_calendar(calendars) + named = [] + for calendar in calendars: + try: + display_name = await calendar.get_display_name() + except Exception as e: + ## Skip calendars whose display name can't be read; warn only + ## when the failure is unexpected (see helper). Continuing + ## ensures one unreadable calendar doesn't abort the lookup. + _warn_unreadable_display_name(self.client, calendar, name, e) + continue + named.append((calendar, display_name)) + return self._find_calendar_by_name(named, name) + def make_calendar( self, name: str | None = None, @@ -236,7 +287,9 @@ async def _async_make_calendar( ) return await calendar.save(method=method) - def calendar(self, name: str | None = None, cal_id: str | None = None) -> "Calendar": + def calendar( + self, name: str | None = None, cal_id: str | None = None + ) -> "Calendar | Coroutine[Any, Any, Calendar]": """ The calendar method will return a calendar object. If it gets a cal_id but no name, it will not initiate any communication with the server @@ -246,21 +299,31 @@ def calendar(self, name: str | None = None, cal_id: str | None = None) -> "Calen cal_id: return the calendar with this calendar id or URL Returns: - Calendar(...)-object - """ - # For name-based lookup, use calendars() which already uses async delegation + Calendar(...)-object. A lookup by ``name``, or with neither + argument, has to list the calendars on the server, so on an async + client it returns a coroutine that must be awaited. + """ + ## A lookup by name (or with no arguments at all) lists the calendars + ## and reads their display names - round-trips, so the async client + ## needs its own path. A cal_id is pure URL arithmetic and stays + ## synchronous for both. + if not cal_id and self.is_async_client: + return self._async_calendar(name) if name and not cal_id: + named = [] for calendar in self.get_calendars(): - display_name = calendar.get_display_name() - if display_name == name: - return calendar - if name and not cal_id: - raise error.NotFoundError(f"No calendar with name {name} found under {self.url}") + try: + display_name = calendar.get_display_name() + except Exception as e: + ## Skip calendars whose display name can't be read; warn only + ## when the failure is unexpected (see helper). Continuing + ## ensures one unreadable calendar doesn't abort the lookup. + _warn_unreadable_display_name(self.client, calendar, name, e) + continue + named.append((calendar, display_name)) + return self._find_calendar_by_name(named, name) if not cal_id and not name: - cals = self.get_calendars() - if not cals: - raise error.NotFoundError("no calendars found") - return cals[0] + return self._first_calendar(self.get_calendars()) if self.client is None: raise ValueError("Unexpected value None for self.client") @@ -450,10 +513,15 @@ def calendar( name: str | None = None, cal_id: str | None = None, cal_url: str | None = None, - ) -> "Calendar": + ) -> "Calendar | Coroutine[Any, Any, Calendar]": """ The calendar method will return a calendar object. - It will not initiate any communication with the server. + + For a full-URL ``cal_id`` or a ``cal_url`` it does not initiate any + communication with the server and returns the Calendar directly (also + for async clients). For a bare ``cal_id``/``name`` it needs the + calendar home set, which on an async client is resolved with a PROPFIND; + in that case it returns a coroutine that must be awaited. """ if not cal_url: ## For full-URL cal_id, skip calendar_home_set (which may be async-lazy) @@ -467,6 +535,11 @@ def calendar( if self.client is None: raise ValueError("Unexpected value None for self.client") return Calendar(self.client, url=URL.objectify(cal_id)) + ## A bare cal_id/name needs the calendar home set. On async clients + ## that resolution awaits a PROPFIND, so we must hand back a coroutine + ## rather than evaluating the (lazy, coroutine-valued) home set here. + if self.is_async_client: + return self._async_calendar(name, cal_id) return self.calendar_home_set.calendar(name, cal_id) else: if self.client is None: @@ -474,6 +547,20 @@ def calendar( return Calendar(self.client, url=self.client.url.join(cal_url)) + async def _async_calendar( + self, + name: str | None = None, + cal_id: str | None = None, + ) -> "Calendar": + """Async implementation of calendar() for a bare cal_id/name.""" + calendar_home_set = await self._async_get_calendar_home_set() + calendar = calendar_home_set.calendar(name, cal_id) + ## A bare cal_id resolves synchronously even on an async client; a + ## name lookup hands back a coroutine. + if inspect.isawaitable(calendar): + calendar = await calendar + return calendar + def get_vcal_address(self) -> "vCalAddress | Coroutine[Any, Any, vCalAddress]": """ Returns the principal, as an icalendar.vCalAddress object. @@ -598,22 +685,27 @@ def freebusy_request( freebusy_ical.add_component(freebusy_comp) outbox = self.schedule_outbox() caldavobj = FreeBusy(data=freebusy_ical, parent=self) - for attendee in attendees: - caldavobj.add_attendee(attendee, no_default_parameters=True) if self.is_async_client: - return self._async_freebusy_request(outbox, caldavobj) + return self._async_freebusy_request(outbox, caldavobj, attendees) + + for attendee in attendees: + caldavobj.add_attendee(attendee, no_default_parameters=True) caldavobj.add_organizer() response = self.client.post(outbox.url, caldavobj.data, headers=ICALH) return response._parse_scheduling_response_objects(parent=self) - async def _async_freebusy_request(self, outbox, fb_obj) -> dict: + async def _async_freebusy_request(self, outbox, fb_obj, attendees) -> dict: """Async implementation of freebusy_request() for async clients.""" ## TODO: could we have common headers as global variable? headers = ICALH outbox = await outbox + for attendee in attendees: + if isinstance(attendee, Principal): + attendee = await attendee.get_vcal_address() + fb_obj.add_attendee(attendee, no_default_parameters=True) ## TODO: it's really bad that arbitrary methods returns ## a coroutine in async mode. It's needed to make it much ## more clear what methods involves I/O and what methods @@ -747,16 +839,38 @@ def _create( prop = dav.Prop() display_name = None - # Some servers (e.g. Zimbra) use the DisplayName from the MKCALENDAR body - # as the calendar URL, ignoring the actual request path. When the server - # does not support setting a separate display name, omit it from the body so - # the request URL path is used as the calendar identifier. supports_displayname = not self.client or self.client.features.is_supported( "create-calendar.set-displayname" ) + stable_url = not self.client or self.client.features.is_supported( + "create-calendar.stable-url" + ) + # A few servers assign a calendar a canonical URL that differs from the + # requested cal_id when a display name is set: Zimbra relocates the + # collection to a display-name-derived path (a collection-level alias + # lingers at the cal_id and answers PROPFIND/REPORT, but a GET on a child + # object under it 404s, so the cal_id is not a usable address), while OX + # always exposes an opaque cal://0/NNN canonical URL. We still send the + # display name (it sticks); afterwards, for such servers + # (create-calendar.stable-url unsupported), we DISCOVER and ADOPT the + # canonical URL (see _adopt_canonical_url) so that self.url - and every + # later URL-based operation - points at the address that actually + # resolves. This replaces the older "drop the display name" workaround + # and behaves identically for Zimbra and OX. We only omit the display + # name when the server cannot set one at creation at all + # (create-calendar.set-displayname unsupported). if name and supports_displayname: display_name = dav.DisplayName(name) prop += [display_name] + elif name: # not supports_displayname + log.warning( + "Creating calendar %r without the requested display name %r: the " + "server does not support setting a display name when a calendar is " + "created (create-calendar.set-displayname). The calendar keeps its " + "requested URL but will have no display name.", + id, + name, + ) if supported_calendar_component_set: sccs = cdav.SupportedCalendarComponentSet() for scc in supported_calendar_component_set: @@ -769,7 +883,7 @@ def _create( mkcol = (dav.Mkcol() if method == "mkcol" else cdav.Mkcalendar()) + set if self.is_async_client: - return self._async_create(path, mkcol, method, name, display_name) + return self._async_create(path, mkcol, method, name, display_name, stable_url) self._query(root=mkcol, query_method=method, url=path, expected_return_value=201) @@ -792,7 +906,84 @@ def _create( exc_info=True, ) - async def _async_create(self, path, mkcol, method, name, display_name) -> None: + # On servers that don't keep the calendar at the requested cal_id when a + # display name is set (create-calendar.stable-url unsupported), re-point + # self.url to the canonical URL the server actually assigned. + if display_name and not stable_url: + self._adopt_canonical_url(name) + + def _adopt_canonical_url(self, name) -> None: + """Re-point ``self.url`` to the server's canonical URL for this calendar. + + Called only for servers where ``create-calendar.stable-url`` is + unsupported: the calendar just created is reachable under a canonical URL + that differs from the requested cal_id (Zimbra: a display-name-derived + path; OX: an opaque ``cal://0/NNN`` segment). The requested cal_id is not + a reliable address there (on Zimbra a collection alias answers + PROPFIND/REPORT but a GET on a child object 404s). We locate the calendar + by the display name we just set and adopt its URL so later URL-based + operations resolve. + + Best effort: if the calendar can't be located (or its name is ambiguous + because another calendar already shares it), ``self.url`` is left at the + requested URL. + """ + requested = self.url.canonical() + try: + relocated = [ + cal.url + for cal in self.parent.calendars() + if _safe_display_name(cal) == name and cal.url.canonical() != requested + ] + except Exception: + log.warning("Could not list calendars to discover canonical URL", exc_info=True) + return + self._adopt_relocated_url(name, relocated) + + async def _async_adopt_canonical_url(self, name) -> None: + """Async twin of :meth:`_adopt_canonical_url`.""" + try: + cals = await self.parent.calendars() + except Exception: + log.warning("Could not list calendars to discover canonical URL (async)", exc_info=True) + return + requested = self.url.canonical() + relocated = [] + for cal in cals: + try: + display_name = await cal.get_display_name() + except Exception: + ## a calendar that refuses to report its display name is not a match + continue + if display_name == name and cal.url.canonical() != requested: + relocated.append(cal.url) + self._adopt_relocated_url(name, relocated) + + def _adopt_relocated_url(self, name, relocated: list) -> None: + """Adopt the one relocated URL found, or keep the requested one. + + Shared by :meth:`_adopt_canonical_url` and its async twin. An + ambiguous display name is deliberately *not* resolved by picking the + first candidate: the other candidate is typically a pre-existing, + unrelated calendar, and adopting it would send every later + ``add_event()``, ``search()`` and ``delete()`` to the wrong calendar + while orphaning the one just created. + """ + if not relocated: + return + if len(relocated) > 1: + log.warning( + "%d calendars are named %r, so the canonical URL of the calendar just " + "created is ambiguous; keeping the requested URL (%s) rather than risk " + "adopting a pre-existing unrelated calendar", + len(relocated), + name, + self.url, + ) + return + self.url = relocated[0] + + async def _async_create(self, path, mkcol, method, name, display_name, stable_url) -> None: """Async implementation of _create (call via _create, not directly).""" await self._query(root=mkcol, query_method=method, url=path, expected_return_value=201) @@ -810,6 +1001,10 @@ async def _async_create(self, path, mkcol, method, name, display_name) -> None: exc_info=True, ) + # See _adopt_canonical_url (sync) - re-point self.url on unstable servers. + if display_name and not stable_url: + await self._async_adopt_canonical_url(name) + def delete(self, wipe=None): """Delete the calendar. @@ -1139,28 +1334,41 @@ async def _async_save(self, display_name, method=None): # def data2object_class - def _multiget(self, event_urls: Iterable[URL], raise_notfound: bool = False) -> Iterable[str]: - """ - get multiple events' data. - TODO: Does it overlap the _request_report_build_resultlist method - ## WARNING: async logic is duplicated in _async_multiget — mirror any changes there + def _build_multiget_root(self, event_urls: Iterable[URL]) -> cdav.CalendarMultiGet: + """Build the calendar-multiget REPORT body for the given hrefs. + + Pure (no I/O) — shared by the sync and async multiget twins. """ if self.url is None: raise ValueError("Unexpected value None for self.url") - prop = dav.Prop() + cdav.CalendarData() - root = cdav.CalendarMultiGet() + prop + [dav.Href(value=u.path) for u in event_urls] - # RFC 4791 section 7.9: "the 'Depth' header MUST be ignored by the - # server and SHOULD NOT be sent by the client" for calendar-multiget - response = self._query(root, None, "report") + return cdav.CalendarMultiGet() + prop + [dav.Href(value=u.path) for u in event_urls] + + def _extract_multiget_results( + self, response: Any, raise_notfound: bool + ) -> list[tuple[str, str]]: + """Turn a multiget REPORT response into ``(href, calendar_data)`` tuples. + + Pure (no I/O) — shared by the sync and async multiget twins. + """ results = response.expand_simple_props([cdav.CalendarData()]) if raise_notfound: - for href in response.statuses: - status = response.statuses[href] + for href, status in response.statuses.items(): if status and "404" in status: raise error.NotFoundError(f"Status {status} in {href}") - for r in results: - yield (r, results[r][cdav.CalendarData.tag]) + return [(r, results[r][cdav.CalendarData.tag]) for r in results] + + def _multiget( + self, event_urls: Iterable[URL], raise_notfound: bool = False + ) -> list[tuple[str, str]]: + """get multiple events' data. + + TODO: Does it overlap the _request_report_build_resultlist method? + """ + # RFC 4791 section 7.9: "the 'Depth' header MUST be ignored by the + # server and SHOULD NOT be sent by the client" for calendar-multiget + response = self._query(self._build_multiget_root(event_urls), None, "report") + return self._extract_multiget_results(response, raise_notfound) def _post_multiget(self, results: Iterable[tuple[str, str]]) -> list[_CC]: """Post-processing shared by multiget and _async_multiget_objects.""" @@ -1188,20 +1396,8 @@ def multiget(self, event_urls: Iterable[URL], raise_notfound: bool = False) -> I async def _async_multiget( self, event_urls: Iterable[URL], raise_notfound: bool = False ) -> list[tuple[str, str]]: - ## WARNING: sync logic is duplicated in _multiget — mirror any changes there - if self.url is None: - raise ValueError("Unexpected value None for self.url") - - prop = dav.Prop() + cdav.CalendarData() - root = cdav.CalendarMultiGet() + prop + [dav.Href(value=u.path) for u in event_urls] - response = await self._query(root, None, "report") - results = response.expand_simple_props([cdav.CalendarData()]) - if raise_notfound: - for href in response.statuses: - status = response.statuses[href] - if status and "404" in status: - raise error.NotFoundError(f"Status {status} in {href}") - return [(r, results[r][cdav.CalendarData.tag]) for r in results] + response = await self._query(self._build_multiget_root(event_urls), None, "report") + return self._extract_multiget_results(response, raise_notfound) async def _async_multiget_objects( self, event_urls: Iterable[URL], raise_notfound: bool = False @@ -1211,6 +1407,84 @@ async def _async_multiget_objects( await self._async_multiget(event_urls, raise_notfound=raise_notfound) ) + def _assign_multiget_data(self, unloaded: list, results: Iterable[tuple[str, str]]) -> None: + """Assign multiget (href, data) results onto the matching unloaded objects. + + Shared post-processing for _batch_load_objects and its async twin: index + the results by normalised URL (quoting to match servers that return + unencoded spaces, e.g. Zimbra) and set obj.data on each match. + + Both sides of the comparison are unquoted before matching. Servers + disagree on what to percent-encode, and the object URLs themselves + went through `quote()` with the default `safe="/"` -- so a UID of the + conventional `@` form ends up as `%40` on one side + and `@` on the other. Comparing the unquoted forms makes the two + spellings equal instead of silently dropping the object. + """ + url_to_data = { + unquote(str(self.url.join(quote(unquote(str(href)), safe="/:@")))): data + for href, data in results + } + for obj in unloaded: + key = unquote(str(obj.url)) + if key in url_to_data: + obj.data = url_to_data[key] + + def _batch_load_objects(self, objects: list) -> None: + """Load unloaded objects from the list in a single calendar-multiget REPORT. + + Already-loaded objects are skipped. If the REPORT fails, falls back to + individual obj.load(only_if_unloaded=True) calls per object; a per-object + failure is logged at debug level and otherwise swallowed, so callers can + filter on is_loaded() afterward. + """ + unloaded = [o for o in objects if not o.is_loaded()] + if not unloaded: + return + try: + self._assign_multiget_data(unloaded, self._multiget([o.url for o in unloaded])) + except Exception: + logging.error("Batch multiget failed, falling back to individual loads", exc_info=True) + for obj in unloaded: + try: + obj.load(only_if_unloaded=True) + except Exception: + ## Deliberate: this method's contract is that the caller + ## filters on is_loaded() afterwards, so one object that + ## cannot be fetched must not abort the rest. Logged at + ## debug rather than warning because the batch failure above + ## is already an error-level line, and a large batch would + ## otherwise produce a wall of warnings. + log.debug("Individual load failed for %s", obj.url, exc_info=True) + + async def _async_batch_load_objects(self, objects: list) -> None: + """Async version of _batch_load_objects. + + The post-processing is shared via _assign_multiget_data(); the only + sync/async difference is the await on the multiget REPORT and on the + per-object fallback load(). + """ + unloaded = [o for o in objects if not o.is_loaded()] + if not unloaded: + return + try: + self._assign_multiget_data( + unloaded, await self._async_multiget([o.url for o in unloaded]) + ) + except Exception: + logging.error( + "Async batch multiget failed, falling back to individual loads", exc_info=True + ) + for obj in unloaded: + try: + load_result = obj.load(only_if_unloaded=True) + if inspect.isawaitable(load_result): + await load_result + except Exception: + ## Same contract as the sync twin above: skip and let the + ## caller filter on is_loaded(). + log.debug("Individual load failed for %s", obj.url, exc_info=True) + def calendar_multiget(self, *largs, **kwargs): """ get multiple events' data @@ -1417,6 +1691,7 @@ def search( filters=None, post_filter=None, _hacks=None, + compatibility_workarounds: bool | None = None, **searchargs, ) -> "list[_CC] | Coroutine[Any, Any, list[_CC]]": """Sends a search request towards the server, processes the @@ -1529,11 +1804,25 @@ def search( # For async clients, use async_search if self.is_async_client: return my_searcher.async_search( - self, server_expand, split_expanded, props, xml, post_filter, _hacks + self, + server_expand, + split_expanded, + props, + xml, + post_filter, + _hacks, + compatibility_workarounds, ) return my_searcher.search( - self, server_expand, split_expanded, props, xml, post_filter, _hacks + self, + server_expand, + split_expanded, + props, + xml, + post_filter, + _hacks, + compatibility_workarounds, ) def freebusy_request( @@ -1834,6 +2123,64 @@ def _generate_fake_sync_token(self, objects: list["CalendarObjectResource"]) -> hash_value = hashlib.md5(combined.encode(), usedforsecurity=False).hexdigest() return f"fake-{hash_value}" + ## The three helpers below carry the pure (no-I/O) logic shared between the + ## get_objects_by_sync_token sync/async twins, so only the awaited + ## server round-trips differ between them. + + def _should_use_sync_token(self, sync_token: Any, disable_fallback: bool) -> bool: + """Decide whether to attempt a real sync-collection REPORT. + + Raises ReportError when the server can't do sync-tokens and the caller + forbade the full-retrieval fallback. + """ + sync_support = self.client.features.is_supported("sync-token", return_type=dict) + if sync_support.get("support") == "unsupported": + if disable_fallback: + raise error.ReportError("Sync tokens are not supported by the server") + return False + ## A fake token means we emulated sync support last time; don't try a real one. + if sync_token and isinstance(sync_token, str) and sync_token.startswith("fake-"): + return False + return True + + def _apply_fallback_etags(self, response: Any, all_objects: list) -> None: + """Map ETags from a depth-1 PROPFIND response onto the given objects. + + ETags are crucial for detecting content changes in the fallback + mechanism (which can otherwise only see additions/deletions). + """ + etag_props = response.expand_simple_props([dav.GetEtag()]) + url_to_obj = {str(obj.url.canonical()): obj for obj in all_objects} + log.debug(f"Fallback: Fetching ETags for {len(url_to_obj)} objects") + for url_str, props in etag_props.items(): + canonical_url_str = str(self.url.join(url_str).canonical()) + if canonical_url_str in url_to_obj: + if not hasattr(url_to_obj[canonical_url_str], "props"): + url_to_obj[canonical_url_str].props = {} + url_to_obj[canonical_url_str].props.update(props) + log.debug(f"Fallback: Added ETag to {canonical_url_str}") + + def _build_fallback_sync_result( + self, all_objects: list, sync_token: Any + ) -> "SynchronizableCalendarObjectCollection": + """Build the fallback collection from a full object list, emulating + sync-token semantics: if the caller passed back our previous fake + token and nothing changed, return an empty collection. + """ + fake_sync_token = self._generate_fake_sync_token(all_objects) + if ( + sync_token + and isinstance(sync_token, str) + and sync_token.startswith("fake-") + and sync_token == fake_sync_token + ): + return SynchronizableCalendarObjectCollection( + calendar=self, objects=[], sync_token=fake_sync_token + ) + return SynchronizableCalendarObjectCollection( + calendar=self, objects=all_objects, sync_token=fake_sync_token + ) + def get_objects_by_sync_token( self, sync_token: Any | None = None, @@ -1869,23 +2216,9 @@ def get_objects_by_sync_token( the server truly supports sync tokens. """ if self.is_async_client: - ## TODO: lots of code duplication here. It's difficult, since there is a lot of - ## forth and back between the client and the server in this method. return self._async_get_objects_by_sync_token(sync_token, load_objects, disable_fallback) - ## Check if we should attempt to use sync tokens - ## (either server supports them, or we haven't checked yet, or this is a fake token) - use_sync_token = True - sync_support = self.client.features.is_supported("sync-token", return_type=dict) - if sync_support.get("support") == "unsupported": - if disable_fallback: - raise error.ReportError("Sync tokens are not supported by the server") - use_sync_token = False - ## If sync_token looks like a fake token, don't try real sync-collection - if sync_token and isinstance(sync_token, str) and sync_token.startswith("fake-"): - use_sync_token = False - - if use_sync_token: + if self._should_use_sync_token(sync_token, disable_fallback): try: root = self.client._build_sync_collection_body( sync_token=sync_token, props=["getetag"] @@ -1931,50 +2264,17 @@ def get_objects_by_sync_token( pass ## Fetch ETags for all objects if not already present - ## ETags are crucial for detecting changes in the fallback mechanism if all_objects and ( not hasattr(all_objects[0], "props") or dav.GetEtag.tag not in all_objects[0].props ): - ## Use PROPFIND to fetch ETags for all objects try: ## Do a depth-1 PROPFIND on the calendar to get all ETags response = self._query_properties([dav.GetEtag()], depth=1) - etag_props = response.expand_simple_props([dav.GetEtag()]) - - ## Map ETags to objects by URL (using string keys for reliable comparison) - url_to_obj = {str(obj.url.canonical()): obj for obj in all_objects} - log.debug(f"Fallback: Fetching ETags for {len(url_to_obj)} objects") - for url_str, props in etag_props.items(): - canonical_url_str = str(self.url.join(url_str).canonical()) - if canonical_url_str in url_to_obj: - if not hasattr(url_to_obj[canonical_url_str], "props"): - url_to_obj[canonical_url_str].props = {} - url_to_obj[canonical_url_str].props.update(props) - log.debug(f"Fallback: Added ETag to {canonical_url_str}") + self._apply_fallback_etags(response, all_objects) except Exception as e: - ## If fetching ETags fails, we'll fall back to URL-based tokens - ## which can't detect content changes, only additions/deletions log.debug(f"Failed to fetch ETags for fallback sync: {e}") - pass - ## Generate a fake sync token based on current state - fake_sync_token = self._generate_fake_sync_token(all_objects) - - ## If a sync_token was provided, check if anything has changed - if sync_token and isinstance(sync_token, str) and sync_token.startswith("fake-"): - ## Compare the provided token with the new token - if sync_token == fake_sync_token: - ## Nothing has changed, return empty collection - return SynchronizableCalendarObjectCollection( - calendar=self, objects=[], sync_token=fake_sync_token - ) - ## If tokens differ, return all objects (emulating a full sync) - ## In a real implementation, we'd return only changed objects, - ## but that requires storing previous state which we don't have - - return SynchronizableCalendarObjectCollection( - calendar=self, objects=all_objects, sync_token=fake_sync_token - ) + return self._build_fallback_sync_result(all_objects, sync_token) def objects_by_sync_token( self, *largs, **kwargs @@ -1996,20 +2296,7 @@ async def _async_get_objects_by_sync_token( disable_fallback: bool = False, ) -> "SynchronizableCalendarObjectCollection": """Async implementation of get_objects_by_sync_token.""" - - ## TODO: lots of code duplication here. It's difficult, since there is a lot of - ## forth and back between the client and the server in this method. - - use_sync_token = True - sync_support = self.client.features.is_supported("sync-token", return_type=dict) - if sync_support.get("support") == "unsupported": - if disable_fallback: - raise error.ReportError("Sync tokens are not supported by the server") - use_sync_token = False - if sync_token and isinstance(sync_token, str) and sync_token.startswith("fake-"): - use_sync_token = False - - if use_sync_token: + if self._should_use_sync_token(sync_token, disable_fallback): try: root = self.client._build_sync_collection_body( sync_token=sync_token, props=["getetag"] @@ -2048,30 +2335,11 @@ async def _async_get_objects_by_sync_token( ): try: response = await self._query_properties([dav.GetEtag()], depth=1) - etag_props = response.expand_simple_props([dav.GetEtag()]) - url_to_obj = {str(obj.url.canonical()): obj for obj in all_objects} - log.debug(f"Fallback: Fetching ETags for {len(url_to_obj)} objects") - for url_str, props in etag_props.items(): - canonical_url_str = str(self.url.join(url_str).canonical()) - if canonical_url_str in url_to_obj: - if not hasattr(url_to_obj[canonical_url_str], "props"): - url_to_obj[canonical_url_str].props = {} - url_to_obj[canonical_url_str].props.update(props) - log.debug(f"Fallback: Added ETag to {canonical_url_str}") + self._apply_fallback_etags(response, all_objects) except Exception as e: log.debug(f"Failed to fetch ETags for fallback sync: {e}") - fake_sync_token = self._generate_fake_sync_token(all_objects) - - if sync_token and isinstance(sync_token, str) and sync_token.startswith("fake-"): - if sync_token == fake_sync_token: - return SynchronizableCalendarObjectCollection( - calendar=self, objects=[], sync_token=fake_sync_token - ) - - return SynchronizableCalendarObjectCollection( - calendar=self, objects=all_objects, sync_token=fake_sync_token - ) + return self._build_fallback_sync_result(all_objects, sync_token) def get_journals(self) -> "list[Journal] | Coroutine[Any, Any, list[Journal]]": """ diff --git a/caldav/compatibility_hints.py b/caldav/compatibility_hints.py index 0ff4e690..ea2bd4bf 100644 --- a/caldav/compatibility_hints.py +++ b/caldav/compatibility_hints.py @@ -80,8 +80,17 @@ class FeatureSet: "url": { "type": "client-hints", }, + "well-known": { + "description": "Server handles /.well-known/caldav discovery as specified in RFC 6764 section 5. A conformant server should respond with a redirect (301/302/307/308) from /.well-known/caldav to the actual CalDAV endpoint. 'full' means a redirect was observed; 'unsupported' means the server returned 404 or similar; 'unknown' means the check was skipped (e.g. localhost or request failed). Note: well-known is often provided by infrastructure (reverse proxy/hosting) rather than the CalDAV server itself, so 'unknown' is the expected default for self-hosted or test setups.", + "default": {"support": "unknown"}, + "links": ["https://datatracker.ietf.org/doc/html/rfc6764#section-5"], + }, "get-current-user-principal": { "description": "Support for RFC5397, current principal extension. Most CalDAV servers have this, but it is an extension to the DAV standard. Possibly observed missing on mail.ru, DavMail gateway and it is possible to configure the support in some sabre-based servers", + ## Independent feature (directly probed): the default marks it so the + ## node uses its own probed value rather than being derived from + ## subfeatures such as .has-calendar. + "default": {"support": "full"}, "links": ["https://datatracker.ietf.org/doc/html/rfc5397"], }, "get-current-user-principal.has-calendar": { @@ -91,9 +100,45 @@ class FeatureSet: "description": "Server returns the supported-calendar-component-set property (RFC 4791 section 5.2.3). The property is optional: when absent the RFC mandates that all component types are accepted, so 'unsupported' here is not a protocol violation, but the client cannot determine the actual supported set without trying.", "links": ["https://datatracker.ietf.org/doc/html/rfc4791#section-5.2.3"], }, + "propfind": { + "description": "Server supports the PROPFIND method (RFC4918 section 9.1): a PROPFIND for a named property returns a multistatus response. Independent feature (not just a grouping node) so that a server lacking a sub-feature like propfind.allprop.resourcetype is not mistaken for one that does not support PROPFIND at all.", + "default": {"support": "full"}, + "links": ["https://datatracker.ietf.org/doc/html/rfc4918#section-9.1"], + }, + "propfind.allprop": { + "description": "An PROPFIND returns a multistatus response. This is independent of whether resourcetype in particular is included (see propfind.allprop.resourcetype).", + "default": {"support": "full"}, + "links": ["https://datatracker.ietf.org/doc/html/rfc4918#section-9.1"], + }, + "propfind.allprop.resourcetype": { + "description": "An PROPFIND returns the DAV:resourcetype live property. RFC4918 section 9.1 lists resourcetype among the live properties an allprop request should return, so 'full' (the default) is the conformant behaviour; a few servers (Bedework) omit it.", + "default": {"support": "full"}, + "links": ["https://datatracker.ietf.org/doc/html/rfc4918#section-9.1"], + }, "create-calendar.with-supported-component-types": { "description": "Server honours the supported-calendar-component-set restriction set at MKCALENDAR time. When 'full', the server both advertises (or enforces) the restriction; when 'unsupported', the restriction is silently ignored (wrong-type objects can be saved to the calendar). When 'ungraceful', the MKCALENDAR request itself fails when a component set is specified.", }, + "calendar-color": { + "description": "Server stores the nonstandard Apple/Mozilla {http://apple.com/ns/ical/}calendar-color property (set with a colour name like 'blue') on a calendar collection. 'full' covers servers that normalise the name to a hex value (the set value still tracks the input); 'broken' is a read-only property (the same value comes back regardless of what is set). Not described by RFC4791/RFC5545, so a server that rejects or ignores it ('unsupported') is not breaching any RFC. The default is 'fragile' because the behaviour varies a lot between servers and is rarely worth asserting on.", + "default": {"support": "fragile"}, + "note": +"""The real default ought to be False because this is not a part +of any published standard AFAIK. We should never expect servers +to support this. However, the compatibility test would trip on +servers that supports it if we leave it as False. +"Fragile" sort of makes sense, because until it has been tested +one should assume the support to maybe exist and maybe not - +hence, "fragile". +""" + }, + "calendar-color.hex": { + "description": "Like calendar-color, but the property is set with a hex value (e.g. '#FF0000FF') rather than a colour name. Some servers accept one form but not the other.", + "default": {"support": "fragile"}, + }, + "calendar-order": { + "description": "Server stores the nonstandard Apple/Mozilla {http://apple.com/ns/ical/}calendar-order property on a calendar collection (a get/set round-trip). 'broken' is a read-only property (e.g. the server returns the calendar's own position regardless of what is set). Not described by RFC4791/RFC5545, so a server that rejects or ignores it ('unsupported') is not breaching any RFC. The default is 'fragile' because the behaviour varies a lot between servers.", + "default": {"support": "fragile"}, + }, "rate-limit": { "type": "client-feature", "description": "client (or test code) must sleep a bit between requests. Pro-active rate limiting is done through interval and count, server-flagged rate-limiting is controlled through default_sleep/max_sleep", @@ -110,6 +155,15 @@ class FeatureSet: "delay": "after this number of seconds, we may be reasonably sure that the search results are updated", } }, + "write-delay": { + "type": "server-peculiarity", + "default": {"support": "full"}, + "description": "The server processes write operations (PUT/DELETE/MKCALENDAR/PROPPATCH/...) asynchronously: the request returns success before the change has fully taken effect, so an immediate read-back (of any kind, not just a search) may 404 or return stale data. A client must wait a bit after every write. This is the general, write-side counterpart of 'search-cache' (which only delays searches). 'full' (the default) means writes take effect synchronously.", + "extra_keys": { + "behaviour": "'delay' to enable the post-write sleep", + "delay": "sleep this number of seconds after every write request before relying on the change being visible", + } + }, "tests-cleanup-calendar": { "type": "tests-behaviour", "description": "Deleting a calendar does not delete the objects, or perhaps create/delete of calendars does not work at all. For each test run, every calendar resource object should be deleted for every test run", @@ -127,10 +181,38 @@ class FeatureSet: "description": "Accessing a calendar which does not exist automatically creates it", }, "create-calendar.set-displayname": { - "description": "It's possible to set the displayname on a calendar upon creation" + "description": "It's possible to set the displayname on a calendar upon creation", + ## Independent feature (directly probed). + "default": {"support": "full"}, + }, + "create-calendar.stable-url": { + "description": ( + "After a calendar is created it remains addressable at the URL derived from the " + "requested cal_id. 'full' (the normal case): the calendar's canonical URL is the " + "requested URL. 'unsupported': the server assigns a DIFFERENT canonical URL and the " + "requested cal_id is not a reliable address for the calendar's object resources, so " + "clients must discover and adopt the canonical URL after creation (the caldav library " + "does this automatically). Two known patterns are handled identically: Zimbra " + "relocates the collection to a display-name-derived path - a collection-level alias " + "may linger at the cal_id and answer PROPFIND/REPORT, but a GET on a child object " + "(...//.ics) 404s, so it is not a usable address (cf. save-load.get-by-url); " + "OX always exposes an opaque cal://0/NNN (base64-segment) canonical URL. Note: on " + "Zimbra the URL only becomes unstable when a display name is supplied at creation; a " + "nameless MKCALENDAR stays at the requested cal_id." + ), + "default": {"support": "full"}, + }, + "propfind.displayname": { + "description": "Server returns the DAV:displayname property for a calendar collection via PROPFIND (RFC4918 section 15.2). This is a standard live property; virtually all CalDAV servers support it. 'broken' means the property is absent from the PROPFIND response even though a displayname was supplied at creation time.", + "default": {"support": "full"}, + "links": ["https://datatracker.ietf.org/doc/html/rfc4918#section-15.2"], }, "delete-calendar": { "description": "RFC4791 says nothing about deletion of calendars, so the server implementation is free to choose weather this should be supported or not. Section 3.2.3.2 in RFC 6638 says that if a calendar is deleted, all the calendarobjectresources on the calendar should also be deleted - but it's a bit unclear if this only applies to scheduling objects or not. Some calendar servers moves the object to a trashcan rather than deleting it", + ## Independent feature (directly probed): the default marks it so the + ## node uses its own probed value rather than being derived from + ## .free-namespace. + "default": {"support": "full"}, "links": ["https://datatracker.ietf.org/doc/html/rfc6638#section-3.2.3.2"], }, "delete-calendar.free-namespace": { @@ -142,9 +224,12 @@ class FeatureSet: "default": { "support": "fragile" }, }, "save-load": { - "description": "it's possible to save and load objects to the calendar" + "description": "it's possible to save and load objects to the calendar", + }, + "save-load.event": { ## TODO: make this DRY + "description": "it's possible to save and load events to the calendar", + "default": { "support": "full" } }, - "save-load.event": {"description": "it's possible to save and load events to the calendar"}, "save-load.event.recurrences": {"description": "it's possible to save and load recurring events to the calendar - events with an RRULE property set, including recurrence sets", "default": {"support": "full"}}, "save-load.event.recurrences.count": {"description": "The server will receive and store a recurring event with a count set in the RRULE", "default": {"support": "full"}}, ## This was Claude's suggestion and it works as of today, the @@ -157,17 +242,32 @@ class FeatureSet: ## information was simply discarded, and the current search behaviour would in ## such a case be incorrect if the exception is simply discarded. "save-load.event.recurrences.exception": {"description": "When a VCALENDAR containing a master VEVENT (with RRULE) and exception VEVENT(s) (with RECURRENCE-ID) is stored, the server keeps them together as a single calendar object resource. When unsupported, the server splits exception VEVENTs into separate calendar objects, making client-side expansion unreliable (the master expands without knowing about its exceptions)."}, - "save-load.todo": {"description": "it's possible to save and load tasks to the calendar"}, - "save-load.todo.recurrences": {"description": "it's possible to save and load recurring tasks to the calendar"}, + "save-load.event.recurrences.exception.reschedule": {"description": "The server accepts a PUT that reschedules an entire recurring event - changing the master VEVENT's DTSTART (re-anchoring the whole series) while detached exception VEVENT(s) (with RECURRENCE-ID) are present and their RECURRENCE-IDs are shifted to line up with the new series. This is unsupported for Ox, the server rejects such a PUT with 409 Conflict even when a matching If-Match etag is supplied. Rescheduling a recurring event that has no exceptions still works. Exercised by save(all_recurrences=True) after changing dtstart/dtend.", "default": {"support": "full"}}, + "save-load.todo": { + "description": "it's possible to save and load tasks to the calendar", + "default": { "support": "full" } + }, + "save-load.todo.recurrences": {"description": "it's possible to save and load recurring tasks to the calendar", "default": {"support": "full"}}, "save-load.todo.recurrences.count": {"description": "The server will receive and store a recurring task with a count set in the RRULE", "default": {"support": "full"}}, "save-load.todo.recurrences.thisandfuture": {"description": "Completing a recurring task with rrule_mode='thisandfuture' works (modifies RRULE and saves back to server)", "default": {"support": "full"}}, "save-load.todo.mixed-calendar": {"description": "The same calendar may contain both events and tasks (Zimbra only allows tasks to be placed on special task lists)", "default": {"support": "full"}}, - "save-load.journal": {"description": "The server will even accept journals"}, + "save-load.journal": { + "description": "The server will even accept journals", + "default": { "support": "full" } + }, ## TODO: zimbra cannot mix events and tasks, but then davis surprised me by not allowing journals on the same calendar. But this may be a miss in the checking script - it may be that mixing is allowed, but that the calendar has to be set up from scratch with explicit support for both VJOURNAL and other things "save-load.journal.mixed-calendar": {"description": "The same calendar may contain events, tasks and journals (some servers require journals on a dedicated VJOURNAL calendar)", "default": {"support": "full"}}, "save-load.get-by-url": { "description": "GET requests to calendar object resource URLs work correctly. When unsupported, the server returns 404 on GET even for valid object URLs. The client works around this by falling back to UID-based lookup.", }, + "non-existing-raises-not-found": { + "description": "Looking up a non-existing calendar object resource raises NotFoundError (the server answers 404). 'full' (the default) is the expected behaviour; some servers answer 403 instead (raising AuthorizationError) - e.g. Robur, probably to avoid leaking whether a resource exists - which is a legitimate choice rather than an RFC breach, so it is recorded as 'unsupported' rather than 'broken'.", + "default": {"support": "full"}, + }, + "save-load.stable-url": { + "description": "The server reports a calendar object resource under the same URL the client used to store it. When 'unsupported', the server canonicalizes the URL: e.g. OX App Suite exposes a calendar both under its display name and under an internal 'cal://0/NNN' identifier, so an object looked up via a calendar-query REPORT (object_by_uid / search) is reported under a different calendar path than the PUT URL. A direct GET on the original URL still works (the server keeps an alias). Clients should therefore not assume that a searched object's URL equals the URL it was created at.", + "default": {"support": "full"}, + }, "save-load.reuse-deleted-uid": { "description": "After deleting an event, the server allows creating a new event with the same UID. When 'broken', the server keeps deleted events in a trashbin with a soft-delete flag, causing unique constraint violations on UID reuse. See https://github.com/nextcloud/server/issues/30096" }, @@ -184,6 +284,15 @@ class FeatureSet: "description": "A saved calendar object resource can be modified and PUT back to the server; the server accepts the update and returns the modified data on the next GET/REPORT. When 'unsupported', the server treats calendar objects as immutable after initial creation (e.g. Google Calendar's legacy CalDAV API). Replaces the old 'no_overwrite' compatibility flag.", "default": {"support": "full"}, }, + "save-load.mutable.attendee-partstat": { + "description": "A client can modify an attendee's PARTSTAT on an existing event and PUT it back directly to the calendar. When 'unsupported', the server forbids direct modification of attendee participation status via PUT (e.g. OX App Suite returns 403 Forbidden even with a matching If-Match etag) and expects the change to be made through iTIP scheduling instead. See https://github.com/python-caldav/caldav/issues/399", + "default": {"support": "full"}, + "links": ["https://github.com/python-caldav/caldav/issues/399"], + }, + "save-load.mutable.if-match-optional": { + "description": "The If-Match precondition is optional when overwriting an existing calendar object resource: the server accepts a PUT that carries no If-Match etag (i.e. add_event()/save() on an object that was not first fetched). When 'unsupported', the server requires an If-Match etag for updates and rejects a no-If-Match overwrite with 409 Conflict (e.g. OX App Suite enforces optimistic concurrency). Such servers still support save-load.mutable via a fetch-then-save (etag-conditional) update; only the blind-overwrite path is affected.", + "default": {"support": "full"}, + }, "search": { "description": "calendar MUST support searching for objects using the REPORT method, as specified in RFC4791, section 7", "links": ["https://datatracker.ietf.org/doc/html/rfc4791#section-7"], @@ -208,7 +317,17 @@ class FeatureSet: "description": "Time-range searches should only return events/todos that actually fall within the requested time range. Some servers incorrectly return recurring events whose recurrences fall outside (after) the search interval, or events with no recurrences in the requested time range at all. RFC4791 section 9.9 specifies that a VEVENT component overlaps a time range if the condition (start < search_end AND end > search_start) is true.", "links": ["https://datatracker.ietf.org/doc/html/rfc4791#section-9.9"], }, + "search.time-range.comp-type-optional": { + "description": "Whether the server accepts a calendar-query carrying a time-range filter but NOT specifying a component type. Per RFC4791 section 9.7 a CALDAV:time-range element is only valid inside a comp-filter for VEVENT/VTODO/VJOURNAL/VFREEBUSY/VALARM - never directly under the VCALENDAR comp-filter. A query without a component type therefore has nowhere RFC-legal to put the time-range. Consequently 'unsupported' (the default) is FULLY RFC-COMPLIANT and is NOT a server defect: SabreDAV-based servers (Baikal, Nextcloud, ...) correctly reject such queries with HTTP 400 'You cannot add time-range filters on the VCALENDAR component'. When unsupported, the library splits the search into one query per component type. See https://github.com/python-caldav/caldav/issues/681", + "default": {"support": "unsupported"}, + "links": ["https://datatracker.ietf.org/doc/html/rfc4791#section-9.7"], + }, "search.time-range.todo": {"description": "basic time range searches for tasks works", "default": {"support": "full"}}, + "search.time-range.todo.no-dtstart": { + "description": "A VTODO without DTSTART (but with DUE) is returned by a date-range search. RFC5545 and RFC4791 section 9.9 say such a task has a defined time span and should be found, so 'full' (the default) is the compliant behaviour; some servers (Davical, Stalwart, Synology) skip any task lacking DTSTART. Probed with a closed window; servers that skip such tasks only in closed ranges (returning them in open-ended ones) are instead tracked by the 'vtodo_datesearch_nodtstart_task_is_skipped_in_closed_date_range' flag.", + "default": {"support": "full"}, + "links": ["https://datatracker.ietf.org/doc/html/rfc4791#section-9.9"], + }, "search.time-range.todo.old-dates": {"description": "time range searches for tasks with old dates (e.g. year 2000) work - some servers enforce a min-date-time restriction"}, "search.time-range.todo.strict": { "description": "Bounded VTODO time-range searches do not return tasks whose time span falls entirely outside the searched range (no false positives).", @@ -264,6 +383,11 @@ class FeatureSet: "search.text": { "description": "Search for text attributes should work" }, + "search.text.comp-type-optional": { + "description": "Whether the server returns matching objects for a calendar-query that carries a prop-filter (CATEGORIES, SUMMARY, ...) but does NOT specify a component type. Such a prop-filter ends up directly under the VCALENDAR comp-filter, where it filters on VCALENDAR's own properties - which do not include component properties like CATEGORIES - so most servers (e.g. Xandikos, SabreDAV) match nothing. 'unsupported' (the default) is therefore the common, RFC-reasonable case; when unsupported the library splits the search into one query per component type. Analogous to search.time-range.comp-type-optional. See https://github.com/python-caldav/caldav/issues/681", + "default": {"support": "unsupported"}, + "links": ["https://datatracker.ietf.org/doc/html/rfc4791#section-9.7"], + }, "search.text.case-sensitive": { "description": "In RFC4791, section-9.7.5, a text-match may pass a collation, and i;ascii-casemap MUST be the default, this is not checked (yet - TODO) by the caldav-server-checker project. Section 7.5 describes that the servers also are REQUIRED to support i;octet. The definitions of those collations are given in RFC4790, i;octet is a case-sensitive byte-by-byte comparition (fastest). search.text.case-sensitive is supported if passing the i;octet collation to search causes the search to be case-sensitive.", "links": [ @@ -285,6 +409,10 @@ class FeatureSet: }, "search.text.category": { "description": "Search for category should work. This is not explicitly specified in RFC4791, but covered in section 9.7.5. No examples targets categories explicitly, but there are some text match examples in section 7.8.6 and following sections", + ## Independent feature (directly probed): the default marks it so the + ## node uses its own probed value rather than being derived from + ## .substring. + "default": {"support": "full"}, "links": [ "https://datatracker.ietf.org/doc/html/rfc4791#section-9.7.5", "https://datatracker.ietf.org/doc/html/rfc4791#section-7.8.6", @@ -301,7 +429,11 @@ class FeatureSet: "links": ["https://datatracker.ietf.org/doc/html/rfc4791#section-7.4"], }, "search.recurrences.includes-implicit.todo": { - "description": "tasks can also be recurring" + "description": "tasks can also be recurring", + ## Independent feature (directly probed): the default marks it so the + ## node uses its own probed value rather than being derived from + ## .pending. + "default": {"support": "full"}, }, "search.recurrences.includes-implicit.todo.pending": { "description": "a future recurrence of a pending task should always be pending and appear in searches for pending tasks", @@ -332,6 +464,10 @@ class FeatureSet: }, "sync-token": { "description": "RFC6578 sync-collection reports are supported. Server provides sync tokens that can be used to efficiently retrieve only changed objects since last sync. Support can be 'full', 'fragile' (occasionally returns more content than expected), or 'unsupported'. Behaviour 'time-based' indicates second-precision tokens requiring sleep(1) between operations", + ## Independent feature (directly probed): the default marks it so the + ## node uses its own probed value rather than being derived from + ## .delete. + "default": {"support": "full"}, "links": ["https://datatracker.ietf.org/doc/html/rfc6578"], }, "sync-token.delete": { @@ -339,6 +475,10 @@ class FeatureSet: }, "scheduling": { "description": "Server supports CalDAV Scheduling (RFC6638). Detected via the presence of 'calendar-auto-schedule' in the DAV response header.", + ## Independent feature (directly probed via the DAV header): the default + ## marks it so the node uses its own probed value rather than being + ## derived from subfeatures such as .calendar-user-address-set. + "default": {"support": "full"}, "links": ["https://datatracker.ietf.org/doc/html/rfc6638"], }, "scheduling.mailbox": { @@ -390,6 +530,11 @@ class FeatureSet: }, "principal-search": { "description": "Server supports searching for principals (CalDAV users). Principal search may be restricted for privacy/security reasons on many servers. (not to be confused with get-current-user-principal)" + ## NB: genuine grouping node - 'supported' iff at least one search + ## method (.by-name / .list-all) works. The checker sets it directly + ## because that OR-semantics cannot be expressed by the library's + ## all-children-agree derivation; it deliberately has NO default so + ## that when all sub-searches fail the node is unsupported. }, "principal-search.by-name": { "description": "Server supports searching for principals by display name. Testing this properly requires setting up another user with a known name, so this check is not yet implemented" @@ -408,6 +553,10 @@ class FeatureSet: "save.duplicate-uid.cross-calendar": { "description": "Server allows events with the same UID to exist in different calendars and treats them as separate entities. Support can be 'full' (allowed), 'ungraceful' (rejected with error), or 'unsupported' (silently ignored or moved). Behaviour 'silently-ignored' means the duplicate is not saved but no error is thrown. Behaviour 'moved-instead-of-copied' means the event is moved from the original calendar to the new calendar (Zimbra behavior)" }, + "save.duplicate-event": { + "description": "Server allows two events with identical content but different UIDs to coexist in the same calendar. Some servers reject or de-duplicate such an event ('duplicates not allowed even with a different UID'), in which case this is 'unsupported' (silently dropped) or 'ungraceful' (rejected with an error). The default 'full' is the usual behaviour.", + "default": {"support": "full"}, + }, ## TODO: as for now, the tests will run towards the first calendar it will find, and most of the tests will assume the calendar is empty. This is bad. "test-calendar": { "type": "tests-behaviour", @@ -485,7 +634,9 @@ def copyFeatureSet(self, feature_set, collapse=True): self._old_flags = feature_set[feature] continue try: - feature_info = self.find_feature(feature) + ## called for the exception, not the return value: an unknown + ## feature name is a typo in the configuration and gets a warning + self.find_feature(feature) except (AssertionError, KeyError): warnings.warn( f"Unknown feature '{feature}' in configuration. " @@ -493,13 +644,14 @@ def copyFeatureSet(self, feature_set, collapse=True): UserWarning, stacklevel=3, ) + continue value = feature_set[feature] if feature not in self._server_features: self._server_features[feature] = {} server_node = self._server_features[feature] if isinstance(value, bool): server_node['support'] = "full" if value else "unsupported" - elif isinstance(value, str) and 'support' not in server_node: + elif isinstance(value, str): self._validate_support_level(value, feature) server_node['support'] = value elif isinstance(value, dict): @@ -541,44 +693,76 @@ def _collapse_key(self, feature_dict): def collapse(self): """ - If all subfeatures are the same, it should be collapsed into the parent - - Messy and complex logic :-( + Compact the stored feature set: a *grouping* parent (one without its own + explicit default) whose grouping children are all explicitly set to the + same status is replaced by a single entry on the parent, and the children + are dropped. + + The parent status comes from the single derivation path, + is_supported() -> _derive_from_subfeatures(). That path already: + * treats a node with an explicit default as an independent feature - + never derived/collapsed from its children (so e.g. save-load.mutable + stays "full" even when every child is "unsupported"), and + * ignores independent children (those with their own default) when + deriving a grouping parent. + collapse() adds only a losslessness check on top: it folds the children + in solely when every grouping child is explicitly set and matches the + derived value, so no per-child information is lost. """ - features = list(self._server_features.keys()) parents = set() - for feature in features: + for feature in self._server_features: if '.' in feature: parents.add(feature[:feature.rfind('.')]) - parents = list(parents) - ## Parents needs to be ordered by the number of dots. We proceed those with most dots first. - parents.sort(key = lambda x: (-x.count('.'), x)) - for parent in parents: + ## Deepest parents first, so a freshly collapsed child can feed its parent. + for parent in sorted(parents, key=lambda x: (-x.count('.'), x)): parent_info = self.find_feature(parent) - if len(parent_info['subfeatures']): - foo = self.is_supported(parent, return_type=dict, return_defaults=False) - if len(parent_info['subfeatures']) > 1 or foo is not None: - dont_collapse = False - foo_key = self._collapse_key(foo) if foo is not None else None - for sub in parent_info['subfeatures']: - bar = self._server_features.get(f"{parent}.{sub}") - if bar is None: - dont_collapse = True - break - bar_key = self._collapse_key(bar) - if foo is None: - foo = bar - foo_key = bar_key - elif bar_key != foo_key: - dont_collapse = True - break - if not dont_collapse: - if parent not in self._server_features: - self._server_features[parent] = {} - for sub in parent_info['subfeatures']: - self._server_features.pop(f"{parent}.{sub}") - self.copyFeatureSet({parent: foo}) + ## Independent node (its own explicit default) is never collapsed. + if 'default' in parent_info: + continue + + ## Independent children (their own default) are separate features: + ## neither folded in nor required to match. + grouping_children = [ + sub + for sub in parent_info['subfeatures'] + if 'default' not in self.find_feature(f"{parent}.{sub}") + ] + if not grouping_children: + continue + + derived = self.is_supported(parent, return_type=dict, return_defaults=False) + if derived is None: + continue + derived_key = self._collapse_key(derived) + + ## Lossless only if every grouping child is explicitly set and matches. + child_nodes = [self._server_features.get(f"{parent}.{sub}") for sub in grouping_children] + if any(node is None or self._collapse_key(node) != derived_key for node in child_nodes): + continue + + ## Folding sets the (previously unset) parent explicitly, which an + ## *independent* child (its own default) that is not itself explicitly + ## set would then inherit - changing its resolved status whenever its + ## default differs from the derived value. Skip the fold in that case + ## so is_supported() stays invariant under collapse(). (e.g. folding + ## save.duplicate-uid into save must not flip the independent sibling + ## save.duplicate-event from its default "full" to "ungraceful".) + independent_children = [ + sub + for sub in parent_info['subfeatures'] + if 'default' in self.find_feature(f"{parent}.{sub}") + ] + if any( + f"{parent}.{sub}" not in self._server_features + and self._collapse_key(self._default(f"{parent}.{sub}")) != derived_key + for sub in independent_children + ): + continue + + for sub in grouping_children: + self._server_features.pop(f"{parent}.{sub}", None) + self.copyFeatureSet({parent: derived}) def _default(self, feature_info): if isinstance(feature_info, str): @@ -619,7 +803,12 @@ def is_supported(self, feature, return_type=bool, return_defaults=True, accept_f if 'default' not in current_info: derived = self._derive_from_subfeatures(feature_, current_info, return_type, accept_fragile) if derived is not None: - return derived + # When visiting an ancestor node (feature_ != feature), only propagate + # the derived status downward if the *original* queried feature is also + # a grouping node (no explicit default). Independent features have their + # own explicit default and must not be overridden by a derived ancestor. + if feature_ == feature or 'default' not in feature_info: + return derived if '.' not in feature_: if not return_defaults: return None @@ -689,8 +878,11 @@ def _derive_from_subfeatures(self, feature, feature_info, return_type, accept_fr if has_positive: if all_same: derived_status = subfeature_statuses[0] + elif not is_complete: + # Incomplete mixed set: unset siblings might be unsupported; inconclusive + return None else: - # Mixed positive/negative → unknown + # All relevant children seen, but mixed positive/negative → unknown derived_status = 'unknown' elif is_complete and all_same: # All relevant subfeatures set, all the same negative status @@ -804,6 +996,67 @@ def dotted_feature_set_list(self, compact=False): ret[x] = feature.copy() return ret + ## Feature types that the server-tester cannot reliably probe and that + ## therefore must not be cross-checked against the declared config. + _UNCHECKABLE_FEATURE_TYPES = ( + "client-feature", + "server-observation", + "tests-behaviour", + "client-hints", + "server-peculiarity", + ) + + def compare(self, observed): + """Compare this *declared* (expected) feature set against an *observed* + feature set and return the list of mismatches. + + Each mismatch is a dict with keys ``feature``, ``expected`` and + ``observed`` holding the resolved (string) support levels that disagree. + + Only server-features are compared; anything resolving to ``fragile`` or + ``unknown`` on either side, and feature types the tester cannot probe + reliably (see ``_UNCHECKABLE_FEATURE_TYPES``), are ignored. + """ + ## Snapshot what the tester explicitly probed *before* compact=True + ## calls collapse(), which mutates _server_features by folding + ## subfeatures into their parent - making probed features look + ## untested. is_supported() still resolves the collapsed values + ## correctly afterwards via the parent. + checked_features = set(observed._server_features.keys()) + observed_dotted = observed.dotted_feature_set_list(compact=True) + expected_dotted = self.dotted_feature_set_list(compact=True) + + mismatches = [] + ## Iterate everything either side made an explicit statement about: + ## the compacted dotted dicts plus every feature the tester probed. + ## Probed features whose observed value equals the default are absent + ## from observed_dotted, yet may still conflict with a non-default + ## status the declared config inherits from a parent (e.g. Infomaniak + ## search.comp-type.optional vs an unsupported search.comp-type). + for feature in set(observed_dotted).union(expected_dotted).union(checked_features): + observation = observed.is_supported(feature, str) + expectation = self.is_supported(feature, str) + if "fragile" in (observation, expectation): + continue + if "unknown" in (observation, expectation): + continue + ## Skip features the tester never explicitly probed - the + ## observation would just be a default, not a real result. + if feature not in observed_dotted and feature not in checked_features: + continue + type_ = observed.find_feature(feature).get("type", "server-feature") + if type_ in self._UNCHECKABLE_FEATURE_TYPES: + continue + if expectation != observation: + mismatches.append( + { + "feature": feature, + "expected": expectation, + "observed": observation, + } + ) + return mismatches + #### OLD STYLE ## THE LIST BELOW IS TO BE REMOVED COMPLETELY. DO NOT USE IT. @@ -825,28 +1078,9 @@ def dotted_feature_set_list(self, compact=False): ## * Perhaps some more readable format should be considered (yaml?). ## * Consider how to get this into the documentation incompatibility_description = { - 'calendar_order': - """Server supports (nonstandard) calendar ordering property""", - - 'calendar_color': - """Server supports (nonstandard) calendar color property""", - - 'duplicates_not_allowed': - """Duplication of an event in the same calendar not allowed """ - """(even with different uid)""", - - 'event_by_url_is_broken': """A GET towards a valid calendar object resource URL will yield 404 (wtf?)""", - 'propfind_allprop_failure': - """The propfind test fails ... """ - """it asserts DAV:allprop response contains the text 'resourcetype', """ - """possibly this assert is wrong""", - - 'vtodo_datesearch_nodtstart_task_is_skipped': - """date searches for todo-items will not find tasks without a dtstart""", - 'vtodo_datesearch_nodtstart_task_is_skipped_in_closed_date_range': """only open-ended date searches for todo-items will find tasks without a dtstart""", @@ -866,24 +1100,21 @@ def dotted_feature_set_list(self, compact=False): """Events should be deleted before the calendar is deleted, """ """and/or deleting a calendar may not have immediate effect""", - 'no_overwrite': - """events cannot be edited""", - 'dav_not_supported': """when asked, the server may claim it doesn't support the DAV protocol. Observed by one baikal server, should be investigated more (TODO) and robur""", 'fastmail_buggy_noexpand_date_search': """The 'blissful anniversary' recurrent example event is returned when asked for a no-expand date search for some timestamps covering a completely different date""", - 'non_existing_raises_other': - """Robur raises AuthorizationError when trying to access a non-existing resource (while 404 is expected). Probably so one shouldn't probe a public name space?""", - 'robur_rrule_freq_yearly_expands_monthly': """Robur expands a yearly event into a monthly event. I believe I've reported this one upstream at some point, but can't find back to it""", } xandikos = { + ## Genuinely returns matching objects for a comp-type-less query that carries + ## a time-range (verified: the event is returned, not just "no error"). + "search.time-range.comp-type-optional": {"support": "full"}, ## Principal property search returns 403 (not implemented) "principal-search": "ungraceful", @@ -900,6 +1131,9 @@ def dotted_feature_set_list(self, compact=False): ## There is much development going on at Radicale as of summar 2025, ## so I'm expecting this list to shrink a lot soon. radicale = { + ## Genuinely returns matching objects for a comp-type-less query that carries + ## a time-range (verified: the event is returned, not just "no error"). + "search.time-range.comp-type-optional": {"support": "full"}, "search.is-not-defined": {"support": "full"}, "search.text.case-sensitive": {"support": "unsupported"}, "search.recurrences.includes-implicit.todo.pending": {"support": "fragile", "behaviour": "inconsistent results between runs"}, @@ -909,11 +1143,9 @@ def dotted_feature_set_list(self, compact=False): ## this only applies for very simple installations "auto-connect.url": {"domain": "localhost", "scheme": "http", "basepath": "/"}, "scheduling": {"support": "unsupported"}, - 'old_flags': [ - ## extra features not specified in RFC4791 - "calendar_order", - "calendar_color" - ] + ## extra properties not specified in RFC4791/RFC5545 + "calendar-color": {"support": "full"}, + "calendar-order": {"support": "full"}, } ## Be aware that nextcloud by default have different rate limits, including how often a user is allowed to create a new calendar. This may break test runs badly. @@ -921,8 +1153,15 @@ def dotted_feature_set_list(self, compact=False): 'auto-connect.url': { 'basepath': '/remote.php/dav', }, - ## I'm surprised, I'm quite sure this was reported ungraceful earlier. Passed with caldav commit a98d50490b872e9b9d8e93e2e401c936ad193003, caldav server checker commit 3cae24cf99da1702b851b5a74a9b88c8e5317dad 2026-02-15. The commit 3cae24cf99da1702b851b5a74a9b88c8e5317dad was however development done on the wrong branch and has been force-pushed awway. It was again observed ungraceful at commits be26d42b1ca3ff3b4fd183761b4a9b024ce12b84 / 537a23b145487006bb987dee5ab9e00cdebb0492 - 'search.comp-type.optional': {'support': 'ungraceful'}, + ## Historically this flip-flopped between "ungraceful" and "full" - that + ## instability was a checker bug (https://github.com/python-caldav/caldav/issues/681): + ## the comp-type.optional probe used to send a comp-type-less query carrying a + ## time-range, which SabreDAV rejects (the time-range belongs in a VEVENT/... + ## comp-filter, not under VCALENDAR). Now that the probe omits the time-range, + ## Nextcloud correctly accepts the bare comp-type-less query. The time-range + ## variant is tracked separately as search.time-range.comp-type-optional + ## (unsupported on SabreDAV, the default). + 'search.comp-type.optional': {'support': 'full'}, 'search.recurrences.expanded.todo': {'support': 'unsupported'}, "search.recurrences.includes-implicit.infinite-scope": False, 'delete-calendar': { @@ -970,13 +1209,42 @@ def dotted_feature_set_list(self, compact=False): ## Zimbra is not very good at it's caldav support zimbra = { 'auto-connect.url': {'basepath': '/dav/'}, + ## Genuinely returns matching objects for a comp-type-less query that carries + ## a time-range (verified: the event is returned, not just "no error"). + 'search.time-range.comp-type-optional': {'support': 'full'}, 'delete-calendar': {'support': 'fragile', 'behaviour': 'may move to trashbin instead of deleting immediately'}, ## This is a zimbra bug when creating calendars with a display ## name. Now mitigated in the calendar creation code. #'save-load.get-by-url': {'support': 'fragile', 'behaviour': '404 most of the time - but sometimes 200. Weird, should be investigated more'}, ## Zimbra treats same-UID events across calendars as aliases of the same event 'save.duplicate-uid.cross-calendar': {'support': 'unsupported'}, - 'create-calendar.set-displayname': {'support': 'unsupported'}, + ## Zimbra DOES apply a display name set at creation (the name sticks, so + ## set-displayname is 'full') - but it couples the display name to the + ## calendar URL. MKCALENDAR lands the calendar at the requested cal_id path; + ## the display name is then applied by a follow-up PROPPATCH, which Zimbra + ## implements as a rename that MOVES the collection: the canonical URL + ## relocates to a display-name-derived path (verified deterministic with a + ## unique name against zcs-foss:latest). + ## + ## So create-calendar.stable-url is 'unsupported': is_supported() returns + ## False, and Calendar._create() therefore discovers and adopts the canonical + ## URL after creation (re-pointing self.url), instead of dropping the display + ## name. This keeps the calendar fully usable (name retained AND object URLs + ## resolve) on both Zimbra and OX with no per-server branching. + ## + ## Two Zimbra quirks worth recording (and someday probing for explicitly), + ## mirrored in caldav-server-tester's CheckMakeDeleteCalendar: + ## * The URL is only unstable when a display name is supplied at creation; + ## a nameless MKCALENDAR stays put at the requested cal_id. + ## * Zimbra keeps a collection-level ALIAS at the original cal_id (PROPFIND/ + ## REPORT on it succeed), yet a GET on a child object under that alias + ## (...//.ics) 404s - the object is only retrievable under + ## the canonical relocated URL. So "the calendar collection is reachable + ## at cal_id" does NOT imply "objects are reachable at cal_id"; the canonical + ## URL must be used. (This also explains the old save-load.get-by-url + ## "404 most of the time but sometimes 200" observation.) + 'create-calendar.set-displayname': {'support': 'full'}, + 'create-calendar.stable-url': {'support': 'unsupported', 'behaviour': 'a display name set at creation relocates the collection to a display-name-derived canonical URL; a collection alias lingers at the requested cal_id but child object GETs under it 404'}, 'save-load.todo.mixed-calendar': {'support': 'unsupported'}, 'save-load.todo.recurrences.count': {'support': 'unsupported'}, ## This is a new problem? 'save-load.journal': {'support': 'ungraceful'}, @@ -987,7 +1255,10 @@ def dotted_feature_set_list(self, compact=False): # sometimes throws a 500 'search.text.category': {'support': 'ungraceful'}, 'search.recurrences.expanded.todo': { "support": "unsupported" }, - 'search.comp-type.optional': {'support': 'fragile'}, ## TODO: more research on this, looks like a bug in the checker, + ## was 'fragile' - that was the checker bug (it compared a comp-type-less + ## search against cnt, which counts objects stored in a separate + ## task/journal calendar). Confirmed full 2026-06-06. + 'search.comp-type.optional': {'support': 'full'}, 'search.time-range.alarm': {'support': 'unsupported'}, 'principal-search': "unsupported", ## Zimbra implements server-side automatic scheduling: invitations are @@ -1011,11 +1282,15 @@ def dotted_feature_set_list(self, compact=False): ## TODO: I just discovered that when searching for a date some ## years after a recurring daily event was made, the event does ## not appear. - - ## extra features not specified in RFC5545 - "calendar_order", - "calendar_color" - ] + ], + ## extra properties not specified in RFC4791/RFC5545. Zimbra stores + ## calendar-order, and stores calendar-color only when set as a hex value - + ## it rejects/ignores a colour name like "blue". (The old 'calendar_color' + ## flag was never actually exercised, because testSetCalendarProperties skips + ## on Zimbra: setting a display name relocates the calendar.) + "calendar-color": {"support": "unsupported"}, + "calendar-color.hex": {"support": "full"}, + "calendar-order": {"support": "full"}, } bedework = { @@ -1040,7 +1315,14 @@ def dotted_feature_set_list(self, compact=False): "search.recurrences": False, "sync-token": { "support": "fragile" }, 'search.comp-type': {'support': 'broken', 'behaviour': 'Server returns everything when searching for events and nothing when searching for todos'}, - 'search.comp-type.optional': {'support': 'ungraceful'}, + ## was 'ungraceful' - that was the checker bug (cnt counted the separately + ## stored journal); confirmed full 2026-06-06. + 'search.comp-type.optional': {'support': 'full'}, + ## Flaps between full and unsupported across runs - the comp-type-less + ## time-range query intermittently returns the in-range object vs nothing, + ## most likely the search-cache delay above. Marked fragile so the checker + ## skips it. Observed 2026-06-06. + 'search.time-range.comp-type-optional': {'support': 'fragile'}, 'search.is-not-defined.dtend': False, "principal-search": { "support": "ungraceful" }, ## Bedework hides past non-recurring events from REPORT without a time-range filter, @@ -1056,11 +1338,11 @@ def dotted_feature_set_list(self, compact=False): ## TODO: play with this and see if it's needed 'save-load.icalendar.related-to': {'support': 'broken', 'behaviour': 'first RELATED-TO line is preserved but subsequent RELATED-TO lines are stripped'}, - 'old_flags': [ - 'propfind_allprop_failure', - 'duplicates_not_allowed', - ], - + ## Bedework omits DAV:resourcetype from an allprop PROPFIND response. + "propfind.allprop.resourcetype": {"support": "unsupported"}, + ## (The old 'duplicates_not_allowed' flag was stale: Bedework does store a + ## second event with the same content under a different UID, so + ## save.duplicate-event is left at the default "full".) } synology = { @@ -1071,7 +1353,8 @@ def dotted_feature_set_list(self, compact=False): 'search.is-not-defined': {'support': 'fragile', 'behaviour': 'works for CLASS but not for CATEGORIES'}, 'search.text.case-sensitive': {'support': 'unsupported'}, 'search.time-range.alarm': {'support': 'unsupported'}, - 'old_flags': ['vtodo_datesearch_nodtstart_task_is_skipped'], + ## Synology skips VTODOs without DTSTART in date-range searches. + 'search.time-range.todo.no-dtstart': {'support': 'unsupported'}, 'test-calendar': {'cleanup-regime': 'wipe-calendar'}, 'scheduling.schedule-tag': False, 'scheduling.mailbox.inbox-delivery': False, @@ -1082,7 +1365,9 @@ def dotted_feature_set_list(self, compact=False): # into their calendar. "scheduling.schedule-tag": False, "http.multiplexing": "fragile", ## ref https://github.com/python-caldav/caldav/issues/564 - 'search.comp-type.optional': {'support': 'ungraceful'}, + ## was 'ungraceful' - that was the checker bug (cnt counted the journal that + ## SabreDAV stores in a separate calendar); confirmed full 2026-06-06. + 'search.comp-type.optional': {'support': 'full'}, 'search.recurrences.expanded.todo': {'support': 'unsupported'}, 'search.recurrences.includes-implicit.todo': {'support': 'unsupported'}, "search.recurrences.includes-implicit.infinite-scope": False, @@ -1091,11 +1376,9 @@ def dotted_feature_set_list(self, compact=False): 'principal-search.by-name.self': {'support': 'unsupported'}, 'principal-search.list-all': {'support': 'ungraceful'}, #'sync-token.delete': {'support': 'unsupported'}, ## Perhaps on some older servers? - 'old_flags': [ - ## extra features not specified in RFC5545 - "calendar_order", - "calendar_color", - ], + ## extra properties not specified in RFC4791/RFC5545 + "calendar-color": {"support": "full"}, + "calendar-order": {"support": "full"}, ## I'm surprised, I'm quite sure this was passing earlier. Caldav commit a98d50490b872e9b9d8e93e2e401c936ad193003, caldav server checker commit 3cae24cf99da1702b851b5a74a9b88c8e5317dad 'search.combined-is-logical-and': False } ## TODO: testPrincipals, testWrongAuthType, testTodoDatesearch fails @@ -1107,7 +1390,10 @@ def dotted_feature_set_list(self, compact=False): } cyrus = { - "search.comp-type.optional": {"support": "ungraceful"}, + ## A bare comp-type-less query is accepted; the previous "ungraceful" was a + ## checker bug where the probe carried a time-range + ## (https://github.com/python-caldav/caldav/issues/681). + "search.comp-type.optional": {"support": "full"}, "search.recurrences.includes-implicit.infinite-scope": False, "search.time-range.alarm": {"support": "ungraceful"}, 'principal-search': {'support': 'ungraceful'}, @@ -1146,29 +1432,41 @@ def dotted_feature_set_list(self, compact=False): # DAViCal delivers iTIP notifications to the attendee inbox AND auto-schedules # into their calendar. "scheduling.schedule-tag": False, - "search.comp-type.optional": { "support": "fragile" }, + ## was 'fragile' - that was the checker bug (cnt mismatch); confirmed full 2026-06-06. + "search.comp-type.optional": { "support": "full" }, + ## Genuinely returns matching objects for a comp-type-less query that carries + ## a time-range (verified: the event is returned, not just "no error"). + "search.time-range.comp-type-optional": { "support": "full" }, "search.time-range.alarm": { "support": "unsupported" }, 'sync-token': {'support': 'fragile'}, 'principal-search': {'support': 'unsupported'}, 'principal-search.list-all': {'support': 'unsupported'}, + ## DAViCal skips VTODOs without DTSTART in date-range searches. + 'search.time-range.todo.no-dtstart': {'support': 'unsupported'}, "old_flags": [ #'no_journal', ## it threw a 500 internal server error! ## for old versions #'nofreebusy', ## for old versions ## 'fragile_sync_tokens' removed - covered by 'sync-token': {'support': 'fragile'} - 'vtodo_datesearch_nodtstart_task_is_skipped', ## no issue raised yet - 'calendar_color', - 'calendar_order', 'vtodo_datesearch_notime_task_is_skipped', ], + ## extra properties not specified in RFC4791/RFC5545 + "calendar-color": {"support": "full"}, + "calendar-order": {"support": "full"}, } sogo = { "scheduling.schedule-tag": False, "scheduling.mailbox.inbox-delivery": False, + ## SOGo rejects the calendar-color property with an error (left at the + ## default "fragile" - rejecting a nonstandard extension is fine). It + ## accepts calendar-order but echoes back a server-computed position rather + ## than the value that was set, so that property is effectively read-only. + "calendar-order": {"support": "broken", "behaviour": "read-only; server returns its own calendar position rather than the value set"}, ## I'm surprised, I'm quite sure this was passing earlier. reported unsupported with caldav commit a98d50490b872e9b9d8e93e2e401c936ad193003, caldav server checker commit 3cae24cf99da1702b851b5a74a9b88c8e5317dad 2026-02-15 "search.text.category": False, - "search.time-range.event.old-dates": False, - "search.time-range.todo.old-dates": False, + ## old-date time-range search works (probe found, definite-future object + ## correctly excluded); the earlier "False" was an artifact of the old + ## count==1 check, which a next-year open-start DUE-only task inflated. "save-load.journal": {"support": "ungraceful"}, "search.is-not-defined": {"support": "unsupported"}, "search.text.case-sensitive": { @@ -1180,9 +1478,13 @@ def dotted_feature_set_list(self, compact=False): "search.time-range.alarm": { "support": "unsupported" }, - ## was unsupported. reported ungraceful with caldav commit a98d50490b872e9b9d8e93e2e401c936ad193003, caldav server checker commit 3cae24cf99da1702b851b5a74a9b88c8e5317dad 2026-02-15 + ## A comp-type-less query returns nothing - with or without a time-range - + ## so both search.comp-type.optional and search.time-range.comp-type-optional + ## are unsupported (the latter is the default). The previous "ungraceful" was + ## a checker bug where the comp-type.optional probe carried a time-range that + ## SabreDAV-likes reject (https://github.com/python-caldav/caldav/issues/681). "search.comp-type.optional": { - "support": "ungraceful" + "support": "unsupported" }, ## includes-implicit.todo has been observed as both supported and unsupported ## across different test runs. Other includes-implicit children are unsupported. @@ -1260,9 +1562,12 @@ def dotted_feature_set_list(self, compact=False): 'principal-search': {'support': 'ungraceful'}, 'freebusy-query': {'support': 'ungraceful'}, "scheduling": {"support": "unsupported"}, - 'old_flags': [ - 'non_existing_raises_other', ## AuthorizationError instead of NotFoundError - ], + ## Robur answers 403 (AuthorizationError) instead of 404 (NotFoundError) when + ## looking up a non-existing resource - probably to avoid leaking whether a + ## resource exists. (Not re-probed during this migration: the Robur test + ## server was down; value carried over from the old 'non_existing_raises_other' + ## flag.) + 'non-existing-raises-not-found': {'support': 'unsupported', 'behaviour': 'raises AuthorizationError (403) instead of NotFoundError (404)'}, 'save-load.icalendar.related-to': {'support': 'unsupported'}, 'test-calendar': {'cleanup-regime': 'wipe-calendar'}, "sync-token": {"support": "ungraceful"}, @@ -1330,11 +1635,12 @@ def dotted_feature_set_list(self, compact=False): "principal-search.by-name.self": {"support": "unsupported"}, "principal-search": {"support": "ungraceful"}, "save-load.journal.mixed-calendar": {"support": "unsupported"}, - "search.comp-type.optional": {"support": "ungraceful"}, - "old_flags": [ - "calendar_order", - "calendar_color", - ], + ## was 'ungraceful' - that was the checker bug (cnt counted the journal that + ## SabreDAV stores in a separate calendar); confirmed full 2026-06-06. + "search.comp-type.optional": {"support": "full"}, + ## extra properties not specified in RFC4791/RFC5545 + "calendar-color": {"support": "full"}, + "calendar-order": {"support": "full"}, ## I'm surprised, I'm quite sure this was passing earlier. Caldav commit a98d50490b872e9b9d8e93e2e401c936ad193003, caldav server checker commit 3cae24cf99da1702b851b5a74a9b88c8e5317dad 'search.combined-is-logical-and': False } @@ -1356,25 +1662,35 @@ def dotted_feature_set_list(self, compact=False): "save.duplicate-uid.cross-calendar": {"support": "ungraceful"}, # CCS rejects multi-instance VTODOs (thisandfuture recurring completion) "save-load.todo.recurrences.thisandfuture": {"support": "unsupported"}, - "search.comp-type.optional": {"support": "ungraceful"}, - ## "full" observed, 70938dc1cbb6a839978eee4315699746d38ee5f0/3cae24cf99da1702b851b5a74a9b88c8e5317dad, 2026-02-17. - ## However, this may be due to mess with the caldav-server-checker branches. "unsupported" again at be26d42b1ca3ff3b4fd183761b4a9b024ce12b84 / 537a23b145487006bb987dee5ab9e00cdebb0492 + ## was 'ungraceful' - that was the checker bug (cnt mismatch: it counted a + ## journal object that CCS could not store, so the comp-type-less count never + ## matched). Confirmed full 2026-06-06. + ## ("full" had also been observed 2026-02-17, then "unsupported"/"ungraceful" + ## - all that flapping was the same checker bug, now fixed.) + "search.comp-type.optional": {"support": "full"}, "search.text.case-sensitive": {"support": "unsupported"}, "search.time-range.event": {"support": "full"}, "search.time-range.event.old-dates": {"support": "ungraceful"}, "search.time-range.todo": {"support": "full"}, "search.time-range.todo.old-dates": {"support": "ungraceful"}, - "search.time-range.open": {"support": "ungraceful"}, + ## open-ended time-range searches work with the near-future fixtures; CCS only + ## rejected them (ungraceful) for the old year-2000 range, so the leaves default + ## to "full" (a grouping "search.time-range.open: ungraceful" was removed here). "search.time-range.alarm": {"support": "unsupported"}, - "search.recurrences": {"support": "unsupported"}, + ## Recurrence expansion actually works within the (near-future) search window; + ## this was previously reported "unsupported" only because the test fixtures + ## lived in year 2000, which CCS's min-date-time restriction hid. Only infinite + ## scope (far-future) and server-side VTODO expansion remain unsupported. + "search.recurrences.includes-implicit.infinite-scope": {"support": "unsupported"}, + "search.recurrences.expanded.todo": {"support": "unsupported"}, "principal-search": {"support": "unsupported"}, # Ephemeral Docker container: wipe objects (avoids UID conflicts across calendars) "test-calendar": {"cleanup-regime": "wipe-calendar"}, - ## Did pass earlier, ungraceful at be26d42b1ca3ff3b4fd183761b4a9b024ce12b84 / 537a23b145487006bb987dee5ab9e00cdebb0492 - 'freebusy-query': {'support': 'ungraceful'}, - "old_flags": [ - "propfind_allprop_failure", - ], + ## freebusy-query works with the near-future fixtures; CCS rejected the + ## year-2000 range with an error, so this defaults to "full" now. + ## (The old 'propfind_allprop_failure' flag was stale: CCS does return + ## DAV:resourcetype in an allprop PROPFIND, so propfind.allprop.resourcetype + ## is left at the default "full".) } ## Stalwart - all-in-one mail & collaboration server (CalDAV added 2024/2025) @@ -1390,24 +1706,39 @@ def dotted_feature_set_list(self, compact=False): 'create-calendar.auto': True, 'principal-search': {'support': 'ungraceful'}, 'search.time-range.alarm': False, + ## Stalwart accepts comp-type-less queries fully, including the time-range + ## and prop-filter variants (both unsupported on most other servers). + ## Confirmed 2026-06-06. + 'search.time-range.comp-type-optional': {'support': 'full'}, + 'search.text.comp-type-optional': {'support': 'full'}, ## Stalwart supports implicit recurrence for datetime events but not for ## all-day (VALUE=DATE) recurring events in time-range searches. 'search.recurrences.includes-implicit.event': {'support': 'fragile', 'behaviour': 'broken for all-day (VALUE=DATE) events'}, ## Stalwart returns the recurring todo in search results but doesn't return the ## RRULE intact, so client-side expansion can't expand it to specific occurrences. 'search.recurrences.includes-implicit.todo': {'support': 'fragile'}, - ## Stalwart correctly handles exceptions in server-side CALDAV:expand (observed supported). - ## Stalwart stores master+exception VEVENTs as a single resource with 2 VEVENTs. + ## Stalwart stores master+exception VEVENTs as a single resource with 2 VEVENTs, + ## so client-side expand of the recurrence set works. 'save-load.event.recurrences.exception': {'support': 'full'}, + ## ...but server-side CALDAV:expand only suppresses the exception-overridden + ## occurrence when SEQUENCE is absent. With SEQUENCE present (as real clients + ## always emit) it returns both the original occurrence and the override. + ## Detected by the server-tester's csc_monthly_recurring_with_exception_seq fixture. + 'search.recurrences.expanded.exception': { + 'support': 'fragile', + 'behaviour': 'server-side expand fails to suppress the exception-overridden occurrence when SEQUENCE is present', + }, 'search.time-range.open': True, ## Stalwart delivers iTIP notifications to the attendee inbox AND auto-schedules ## into their calendar (verified by running CheckSchedulingInboxDelivery). "scheduling.mailbox.inbox-delivery": True, "scheduling.auto-schedule": True, - 'old_flags': [ - ## Stalwart does not return VTODO items without DTSTART in date searches - 'vtodo_datesearch_nodtstart_task_is_skipped', - ], + ## Stalwart's handling of DTSTART-less VTODOs in date searches is date + ## dependent: a near-future DUE-only task is returned (the server-tester + ## probe sees 'full'), but the old-date fixtures used by testTodoDatesearch + ## are skipped. Marked 'fragile' so the checker skips it and the integration + ## test (is_supported -> False) still treats the old-date task as skipped. + 'search.time-range.todo.no-dtstart': {'support': 'fragile'}, } ## Lots of transient problems with purelymail @@ -1494,12 +1825,24 @@ def dotted_feature_set_list(self, compact=False): ] } -## https://www.open-xchange.com/ +## https://ox.io/ ## OX App Suite CalDAV served at /caldav/ (Apache proxies to /servlet/dav/caldav on port 8009). ## The Docker image must be built locally before use (see tests/docker-test-servers/ox/build.sh). ox = { - ## Renaming a calendar after creation via PROPPATCH is not supported - 'create-calendar.set-displayname': {'support': 'unsupported'}, + ## Renaming a calendar after creation via PROPPATCH is not supported, but + ## setting the display name AT creation time is - and that's what the probe + ## tests. Was 'unsupported' (conflated the two operations, and masked by the + ## checker's display-name-lookup bug). Confirmed full 2026-06-07. + 'create-calendar.set-displayname': {'support': 'full'}, + ## OX gives EVERY calendar an opaque internal 'cal://0/NNN' canonical URL + ## (base64-encoded in the path, e.g. /caldav/Y2FsOi8vMC8xMzYw/), whether or + ## not a display name is set. The requested cal_id does resolve as a usable + ## alias (object GETs under it work, unlike Zimbra), but the canonical URL + ## still differs from the requested URL - so under the URL-stability semantics + ## this is 'unsupported', exactly like Zimbra and with no special-casing: the + ## library discovers and adopts the canonical URL after creation. (The + ## display name itself sticks, so create-calendar.set-displayname is 'full'.) + 'create-calendar.stable-url': {'support': 'unsupported', 'behaviour': "the calendar's canonical URL is an opaque cal://0/NNN (base64 path segment) that differs from the requested cal_id; the cal_id alias is usable but clients should adopt the canonical URL"}, ## VTODOs must be in a dedicated VTODO-only calendar; mixed calendars not supported 'save-load.todo.mixed-calendar': {'support': 'unsupported'}, ## Basic VTODO support works fine; only recurrences are broken @@ -1508,20 +1851,63 @@ def dotted_feature_set_list(self, compact=False): 'save-load.todo.recurrences': {'support': 'ungraceful'}, ## VJOURNAL is not supported 'save-load.journal': {'support': 'unsupported'}, + ## OX exposes the calendar both under its display name and under an internal + ## "cal://0/NNN" id, so objects looked up via REPORT come back under a + ## different calendar URL than the one used to PUT them (GET on the original + ## URL still works via an alias). + 'save-load.stable-url': {'support': 'unsupported'}, + ## OX enforces optimistic concurrency: a no-If-Match overwrite PUT is rejected + ## with 409 Conflict (etag-conditional save() still works). + 'save-load.mutable.if-match-optional': {'support': 'unsupported'}, + ## OX forbids changing an attendee's PARTSTAT via a direct PUT (403 Forbidden + ## even with a matching etag); it must go through iTIP scheduling. + 'save-load.mutable.attendee-partstat': {'support': 'unsupported'}, ## Search limitations 'search.time-range.event.old-dates': {'support': 'unsupported'}, 'search.time-range.todo.old-dates': {'support': 'unsupported'}, 'search.time-range.alarm': {'support': 'unsupported'}, 'search.unlimited-time-range': {'support': 'broken'}, - 'search.comp-type.optional': {'support': 'ungraceful'}, - 'search.text': {'support': 'unsupported'}, + ## was 'ungraceful' - that was the checker bug (cnt mismatch across the + ## separate VTODO calendar); confirmed full 2026-06-06. + 'search.comp-type.optional': {'support': 'full'}, + ## OX silently ignores the CALDAV comp-filter: a calendar-query that + ## specifies a component type returns the calendar's whole contents + ## regardless of the requested type (a VEVENT-calendar answers a VTODO query + ## with its VEVENT, and vice versa). No right-typed objects are dropped, so + ## the library recovers the correct result by post-filtering - hence + ## "unsupported" (silently ignored), not "broken". Confirmed by direct probe + ## 2026-06-09. Contrast bedework, which drops the todos (data loss = broken). + 'search.comp-type': {'support': 'unsupported'}, + ## Text search (case-sensitive, case-insensitive, substring) now works in OX. + ## Confirmed full 2026-06-13. Category search remains unsupported. + 'search.text': {'support': 'full'}, 'search.text.category': {'support': 'unsupported'}, - 'search.text.case-sensitive': {'support': 'unsupported'}, - 'search.text.case-insensitive': {'support': 'unsupported'}, - ## Recurrence searching broken (sliding window + old-dates limitation) - 'search.recurrences.includes-implicit': {'support': 'unsupported'}, + ## Recurrence searching: the sliding window hides far-past/far-future + ## occurrences, but implicit expansion of *datetime* events and server-side + ## expansion of exceptions work within the window (detectable now that the + ## fixtures are in the near future rather than year 2000). VTODO recurrence, + ## datetime-event server-side expansion, and infinite scope remain unsupported. + ## (event and exception expansion are left at the default "full".) + 'search.recurrences.includes-implicit.todo': {'support': 'unsupported'}, 'search.recurrences.includes-implicit.todo.pending': {'support': 'unsupported'}, - 'search.recurrences.expanded': {'support': 'unsupported'}, + 'search.recurrences.includes-implicit.infinite-scope': {'support': 'unsupported'}, + 'search.recurrences.expanded.event': {'support': 'unsupported'}, + 'search.recurrences.expanded.todo': {'support': 'unsupported'}, + ## Rescheduling the whole series (changing the master DTSTART) is rejected with + ## 409 Conflict once detached exceptions exist - even with a matching If-Match + ## etag. Shifting the DTSTART of an exception-free recurring event still works. + ## Confirmed by direct probe 2026-06-14. + 'save-load.event.recurrences.exception.reschedule': {'support': 'unsupported'}, + ## OX ignores the time-range on VTODO queries and returns every task + 'search.time-range.todo.strict': {'support': 'broken'}, + ## OX silently ignores the is-not-defined prop-filter and returns the whole + ## calendar regardless (confirmed by direct probe 2026-06-09: a no_category + ## search still returns the categorised event; a no_class search still + ## returns the CONFIDENTIAL event). Same "filter ignored" behaviour as + ## search.comp-type above - silently ignored, hence unsupported. + 'search.is-not-defined': {'support': 'unsupported'}, + 'search.is-not-defined.category': {'support': 'unsupported'}, + 'search.is-not-defined.class': {'support': 'unsupported'}, ## is-not-defined for DTEND is not supported 'search.is-not-defined.dtend': {'support': 'unsupported'}, ## Freebusy queries are not supported (returns 400) @@ -1538,9 +1924,71 @@ def dotted_feature_set_list(self, compact=False): "scheduling.freebusy-query": "ungraceful", 'search.time-range.open.start': "broken", 'search.time-range.open.end': True, - ## time-range.open is "broken", while time-range.open.start.duration is "unsupported"? - ## this may possibly be some problems with the checker rather than with Ox - 'search.time-range.open.start.duration': "unsupported" + ## DTSTART+DURATION components ARE found by an overlapping time-range search: + ## confirmed by direct probe 2026-06-09 for VEVENT, and the VTODO duration + ## fixture is returned too. The VTODO time-range is not honoured strictly + ## (out-of-range tasks leak in - tracked separately as + ## search.time-range.todo.strict=broken), so the checker now treats the VTODO + ## duration probe as inconclusive rather than a failure and judges this + ## feature from the conclusive VEVENT result. (Previously mis-reported as a + ## VTODO/VEVENT asymmetry; see the old "checker problem" note here.) + 'search.time-range.open.start.duration': {'support': 'full'}, + ## Rate limiting. One may want to override this in production. + 'rate-limit': { + 'enable': True, + 'interval': 300, + 'count': 1500, + 'max_sleep': 350, + 'default_sleep': 20 + } +} + +## Infomaniak (https://www.infomaniak.com/) - kSuite calendar, CalDAV served at +## https://sync.infomaniak.com/ (/.well-known/caldav redirects there). Runs +## SabreDAV 4.3.1. Profiled 2026-06-15 against a freshly created dedicated +## calendar; save-load and most search features work well. +infomaniak = { + ## SabreDAV processes writes asynchronously - MKCALENDAR/PUT/DELETE return + ## before the change is queryable, so an immediate read-back 404s or returns + ## stale data for several seconds. This is server-wide (not just searches), + ## so we sleep after every write rather than only before searches. + 'write-delay': {'behaviour': 'delay', 'delay': 16}, + ## VJOURNAL is not supported. + 'save-load.journal': {'support': 'unsupported'}, + ## Calendar colour/order work once the post-write delay is honoured (the + ## hex form is normalised, e.g. '#FF0000FF' is stored as '#ff0000'). These + ## previously looked 'broken' (read-only): a read-back issued too soon + ## returned the stale value, an artifact of the asynchronous writes above. + ## Set explicitly to 'full' since the feature default is the weaker 'fragile'. + 'calendar-color': {'support': 'full'}, + 'calendar-color.hex': {'support': 'full'}, + 'calendar-order': {'support': 'full'}, + ## The CALDAV comp-filter is silently ignored: a calendar-query that requests + ## one component type returns the calendar's whole contents regardless (a + ## VJOURNAL query returned a VEVENT). No right-typed objects are dropped, so + ## the library recovers by post-filtering - hence "unsupported", not "broken". + 'search.comp-type': {'support': 'unsupported', 'behaviour': 'comp-filter silently ignored - returns the whole calendar regardless of requested component type'}, + ## Because the comp-filter is ignored, omitting it (which the RFC permits) + ## also returns the whole calendar - so the "optional comp-type" behaviour + ## works. The parent is 'unsupported', so this child must say so explicitly, + ## otherwise it inherits 'unsupported' and disagrees with the observation. + 'search.comp-type.optional': {'support': 'full'}, + ## A combined (logical-AND) filter is not honoured. + 'search.combined-is-logical-and': {'support': 'unsupported'}, + ## VTODO recurrence searching is not supported (datetime VEVENT recurrence + ## search, including server-side expand and infinite scope, works fine). + 'search.recurrences.includes-implicit.todo': {'support': 'unsupported'}, + 'search.recurrences.includes-implicit.todo.pending': {'support': 'unsupported'}, + 'search.recurrences.expanded.todo': {'support': 'unsupported'}, + ## Scheduling is advertised and the calendar-user-address-set and scheduling + ## mailbox are present, but the server never returns a Schedule-Tag (neither + ## on GET nor via PROPFIND). + 'scheduling.schedule-tag': {'support': 'unsupported', 'behaviour': 'no Schedule-Tag returned on GET or via PROPFIND'}, + 'scheduling.schedule-tag.stable-partstat': {'support': 'unsupported'}, + ## Principal search is effectively unsupported (lists nothing / errors out). + 'principal-search': {'support': 'ungraceful'}, + 'principal-search.by-name.self': {'support': 'unsupported'}, + 'principal-search.list-all': {'support': 'ungraceful'}, } # fmt: on diff --git a/caldav/config.py b/caldav/config.py index 05c8af72..07fc585a 100644 --- a/caldav/config.py +++ b/caldav/config.py @@ -33,6 +33,8 @@ def expand_config_section(config, section="default", blacklist=None): ## If it's not a glob-pattern ... if set(section).isdisjoint(set("[*?")): + if section not in config: + return [] ## If it's referring to a "meta section" with the "contains" keyword if "contains" in config[section]: results = [] @@ -47,7 +49,7 @@ def expand_config_section(config, section="default", blacklist=None): return results else: ## Disabled sections should be ignored - if config.get("section", {}).get("disable", False): + if config.get(section, {}).get("disable", False): return [] ## NORMAL CASE - return [ section ] @@ -181,7 +183,7 @@ def resolve_features(features): feature_name = features if feature_name.startswith("compatibility_hints."): feature_name = feature_name[len("compatibility_hints.") :] - return getattr(caldav.compatibility_hints, feature_name) + return copy.deepcopy(getattr(caldav.compatibility_hints, feature_name)) if isinstance(features, dict) and "base" in features: base_name = features["base"] if isinstance(base_name, str): @@ -262,16 +264,24 @@ def get_connection_params( Dict with connection parameters (url, username, password, etc.) or None if no configuration found. """ - # 1. Explicit parameters take highest priority - if explicit_params: - # Filter to valid connection keys - conn_params = {k: v for k, v in explicit_params.items() if k in CONNKEYS} - if conn_params.get("url") or conn_params.get("features"): - # Return when URL is given, or when features are given (the - # client constructor resolves URL from auto-connect.url hints - # via _auto_url()). Don't fall through to env vars/config - # files when the caller explicitly provided connection info. - return conn_params + # 1. Explicit parameters take highest priority. + # A kwarg whose value is None counts as "not supplied" rather than + # "unset it" - the common CLI wrapper get_davclient(url=args.url, + # username=args.user, password=args.password) passes None for every + # option the user left out, and overlaying those on the winning source + # would wipe CALDAV_URL and friends. An empty string is kept: it is + # meaningful for servers with no authentication. + explicit_conn = ( + {k: v for k, v in explicit_params.items() if k in CONNKEYS and v is not None} + if explicit_params + else {} + ) + if explicit_conn.get("url") or explicit_conn.get("features"): + # Return when URL is given, or when features are given (the + # client constructor resolves URL from auto-connect.url hints + # via _auto_url()). Don't fall through to env vars/config + # files when the caller explicitly provided connection info. + return explicit_conn # Check for config file path from environment early (needed for test server config too) if environment: @@ -284,6 +294,9 @@ def get_connection_params( if testconfig or (environment and os.environ.get("PYTHON_CALDAV_USE_TEST_SERVER")): conn = _get_test_server_config(name, environment, config_file) if conn is not None: + # Explicit kwargs outrank the discovered test server, same as for + # the environment and config-file sources below. + conn.update(explicit_conn) return conn # In test mode, don't fall through to regular config - return None # This prevents accidentally using personal/production servers for testing @@ -297,14 +310,19 @@ def get_connection_params( if environment: conn_params = _get_env_config() if conn_params: + conn_params.update(explicit_conn) return conn_params # 4. Config file if check_config_file: conn_params = _get_file_config(config_file, config_section) if conn_params: + conn_params.update(explicit_conn) return conn_params + # No env/config source matched. At this point explicit_conn has neither + # 'url' nor 'features' (those return early above), so it cannot produce a + # connectable client — treat it as "no configuration found". return None @@ -334,7 +352,7 @@ def _get_file_config(file_path: str | None, section_name: str | None) -> dict[st return None section_data = config_section(cfg, section_name) - return _extract_conn_params_from_section(section_data) + return extract_conn_params_from_section(section_data) def _get_test_server_config( @@ -496,14 +514,22 @@ def _test_server_to_params(server: Any, was_already_started: bool) -> dict[str, return params -def _extract_conn_params_from_section(section_data: dict[str, Any]) -> dict[str, Any] | None: +def extract_conn_params_from_section(section_data: dict[str, Any]) -> dict[str, Any] | None: """Extract connection parameters from a config section dict. - Returns a dict containing only CONNKEYS entries. Returns ``None`` if no - server URL is present. Calendar filter keys (``calendar_name``, - ``calendar_url``) are intentionally excluded — callers that need them - (e.g. :func:`get_all_file_connection_params`) read ``section_data`` - directly. + Keys prefixed with ``caldav_`` are mapped to client constructor parameters + (with ``caldav_user``/``caldav_pass`` accepted as aliases for + username/password), environment variable references are expanded, and a + ``features`` key is resolved through :func:`resolve_features`. Public so + that downstream tools (e.g. plann) can reuse it on plann-style config + sections. + + Returns a dict containing only CONNKEYS entries. Returns ``None`` if + neither a server URL nor features are present (with features, the client + constructor can resolve the URL from auto-connect.url hints). Calendar + filter keys (``calendar_name``, ``calendar_url``) are intentionally + excluded — callers that need them (e.g. + :func:`get_all_file_connection_params`) read ``section_data`` directly. """ conn_params: dict[str, Any] = {} for k in section_data: @@ -522,7 +548,7 @@ def _extract_conn_params_from_section(section_data: dict[str, Any]) -> dict[str, elif k == "features" and section_data[k]: conn_params["features"] = resolve_features(section_data[k]) - return conn_params if conn_params.get("url") else None + return conn_params if (conn_params.get("url") or conn_params.get("features")) else None def get_all_file_connection_params( @@ -540,7 +566,7 @@ def get_all_file_connection_params( ``calendar_url`` calendar-filter keys read from the config section. Returns an empty list when the config file is absent or the section has - no usable URL. + neither a usable URL nor features to derive one from. """ if not section_name: section_name = "default" @@ -553,7 +579,7 @@ def get_all_file_connection_params( result: list[dict[str, Any]] = [] for s in sections: section_data = config_section(cfg, s) - params = _extract_conn_params_from_section(section_data) + params = extract_conn_params_from_section(section_data) if params: # Add calendar filter keys — these must NOT flow into DAVClient() for k in ("calendar_name", "calendar_url"): @@ -588,7 +614,7 @@ def get_all_test_servers( for section_name in cfg: section_data = config_section(cfg, section_name) if section_data.get("testing_allowed"): - conn_params = _extract_conn_params_from_section(section_data) + conn_params = extract_conn_params_from_section(section_data) if conn_params: # Also copy the raw section data for keys not in CONNKEYS # (e.g., testing_allowed itself, or custom keys) diff --git a/caldav/datastate.py b/caldav/datastate.py index 72c89dc2..9609a10a 100644 --- a/caldav/datastate.py +++ b/caldav/datastate.py @@ -18,6 +18,8 @@ import icalendar +from caldav.lib.vcal import parse_ical + if TYPE_CHECKING: import vobject @@ -64,18 +66,18 @@ def get_uid(self) -> str | None: """ cal = self.get_icalendar_copy() for comp in cal.subcomponents: - if comp.name in ("VEVENT", "VTODO", "VJOURNAL", "FREEBUSY") and "UID" in comp: + if comp.name in ("VEVENT", "VTODO", "VJOURNAL", "VFREEBUSY") and "UID" in comp: return str(comp["UID"]) return None def get_component_type(self) -> str | None: - """Get the component type (VEVENT, VTODO, VJOURNAL, FREEBUSY) without full parsing. + """Get the component type (VEVENT, VTODO, VJOURNAL, VFREEBUSY) without full parsing. Default implementation parses the data, but subclasses can optimize. """ cal = self.get_icalendar_copy() for comp in cal.subcomponents: - if comp.name in ("VEVENT", "VTODO", "VJOURNAL", "FREEBUSY"): + if comp.name in ("VEVENT", "VTODO", "VJOURNAL", "VFREEBUSY"): return comp.name return None @@ -126,7 +128,11 @@ def get_data(self) -> str: return self._data def get_icalendar_copy(self) -> icalendar.Calendar: - return icalendar.Calendar.from_ical(self._data) + ## parse_ical() rather than from_ical(): this is the one parse that gets + ## fed straight from a server response, so an empty or non-iCalendar + ## body has to say so rather than surfacing as a bare ValueError from + ## deep inside icalendar. + return parse_ical(self._data) def get_vobject_copy(self) -> vobject.base.Component: import vobject @@ -149,7 +155,7 @@ def get_component_type(self) -> str | None: return "VTODO" elif "BEGIN:VJOURNAL" in self._data: return "VJOURNAL" - elif "BEGIN:FREEBUSY" in self._data: + elif "BEGIN:VFREEBUSY" in self._data: return "VFREEBUSY" return None @@ -171,7 +177,7 @@ def get_data(self) -> str: def get_icalendar_copy(self) -> icalendar.Calendar: # Parse from serialized form to get a true copy - return icalendar.Calendar.from_ical(self.get_data()) + return parse_ical(self.get_data()) def get_authoritative_icalendar(self) -> icalendar.Calendar: """Returns THE icalendar object (not a copy). @@ -214,7 +220,7 @@ def get_data(self) -> str: return self._vobject.serialize() def get_icalendar_copy(self) -> icalendar.Calendar: - return icalendar.Calendar.from_ical(self.get_data()) + return parse_ical(self.get_data()) def get_vobject_copy(self) -> vobject.base.Component: import vobject diff --git a/caldav/davclient.py b/caldav/davclient.py index 74d37803..ee86009b 100644 --- a/caldav/davclient.py +++ b/caldav/davclient.py @@ -223,7 +223,12 @@ def __init__( preventing DNS-based downgrade attacks where malicious DNS could redirect to unencrypted HTTP. Set to False ONLY if you need to support non-TLS servers and trust your DNS infrastructure. - This parameter has no effect if enable_rfc6764=False. + SCOPE: this only gates the RFC6764 discovery path. It has no + effect when enable_rfc6764=False, and does NOT reject an + explicitly-passed http:// URL (e.g. url="http://your.server.example.com/dav/" + still connects over plaintext despite require_tls=True). + Making enforcement global is deferred to 4.0 — see + https://github.com/python-caldav/caldav/issues/687 rate_limit_handle: boolean, whether to automatically sleep and retry when the server responds with 429 Too Many Requests or 503 Service Unavailable. Default: False (raise RateLimitError immediately). @@ -297,9 +302,16 @@ def __init__( } ) self.headers.update(headers or CaseInsensitiveDict()) - if self.url.username is not None: + ## An explicit username discards the URL credentials wholesale: they + ## belong to a different account, and merging them field by field + ## would let DAVClient(url="https://bob:hunter2@cal.example.com/", + ## username="alice") ship alice's login with bob's password. + ## Overriding only the password is a different thing - the username + ## still comes from the URL, so the pair stays coherent. + if self.url.username is not None and username is None: username = unquote(self.url.username) - password = unquote(self.url.password) + if password is None and self.url.password is not None: + password = unquote(self.url.password) # Use discovered username if no explicit username was provided if username is None and discovered_username is not None: @@ -331,19 +343,9 @@ def __init__( self._principal = None - rate_limit = self.features.is_supported("rate-limit", dict) - if rate_limit_handle is None: - if rate_limit and rate_limit.get("enable"): - rate_limit_handle = True - if "default_sleep" in rate_limit: - rate_limit_default_sleep = rate_limit["default_sleep"] - if "max_sleep" in rate_limit: - rate_limit_max_sleep = rate_limit["max_sleep"] - else: - rate_limit_handle = False - self.rate_limit_handle = rate_limit_handle - self.rate_limit_default_sleep = rate_limit_default_sleep - self.rate_limit_max_sleep = rate_limit_max_sleep + self._init_rate_limit_config( + rate_limit_handle, rate_limit_default_sleep, rate_limit_max_sleep + ) def __enter__(self) -> Self: ## Used for tests, to set up a temporarily test server @@ -466,13 +468,6 @@ def get_calendars(self, principal: Principal | None = None) -> list[Calendar]: for cal in calendars: print(f"Calendar: {cal.get_display_name()}") """ - from caldav.collection import ( - _extract_calendar_home_set_from_results as extract_home_set, - ) - from caldav.collection import ( - _extract_calendars_from_propfind_results as extract_calendars, - ) - if principal is None: principal = self.principal() @@ -482,14 +477,7 @@ def get_calendars(self, principal: Principal | None = None) -> list[Calendar]: props=self.CALENDAR_HOME_SET_PROPS, depth=0, ) - calendar_home_url = extract_home_set(response.results) - if not calendar_home_url: - # Fall back to the principal URL as calendar home - # (some servers like GMX don't support calendar-home-set) - calendar_home_url = str(principal.url) - - # Make URL absolute if relative - calendar_home_url = self._make_absolute_url(calendar_home_url) + calendar_home_url = self._calendar_home_url(response, principal) # Fetch calendars via PROPFIND response = self.propfind( @@ -498,14 +486,7 @@ def get_calendars(self, principal: Principal | None = None) -> list[Calendar]: depth=1, ) - # Process results using shared helper - calendar_infos = extract_calendars(response.results) - - # Convert CalendarInfo objects to Calendar objects - return [ - Calendar(client=self, url=info.url, name=info.name, id=info.cal_id) - for info in calendar_infos - ] + return self._build_calendars_from_propfind(response) def search_calendar( self, @@ -825,20 +806,7 @@ def request( try: return self._sync_request(url, method, body, headers) except error.RateLimitError as e: - if not self.rate_limit_handle: - raise - sleep_seconds = error.compute_sleep_seconds( - e.retry_after_seconds, - self.rate_limit_default_sleep, - self.rate_limit_max_sleep, - ) - if rate_limit_time_slept: - sleep_seconds += rate_limit_time_slept / 2 - if sleep_seconds is None or ( - self.rate_limit_max_sleep is not None - and rate_limit_time_slept > self.rate_limit_max_sleep - ): - raise + sleep_seconds = self._rate_limit_sleep_seconds(e, rate_limit_time_slept) time.sleep(sleep_seconds) return self.request(url, method, body, headers, rate_limit_time_slept + sleep_seconds) diff --git a/caldav/davobject.py b/caldav/davobject.py index f2a2383b..e25e5de2 100644 --- a/caldav/davobject.py +++ b/caldav/davobject.py @@ -407,6 +407,13 @@ def _post_get_properties(self, response, props, parse_response_xml, parse_props) if not parse_response_xml: return response + ## A 207 whose responses are all bare 404s means the resource we + ## asked about is not there. Without this the propstat-oriented + ## parsing below finds nothing and hands the caller a dict of None + ## values instead - see DAVResponse.all_responses_not_found(). + if response.all_responses_not_found(): + raise error.NotFoundError(f"{self.url} not found on the server") + # Use protocol layer results when available and parse_props=True if parse_props and response.results: # Convert results to the expected {href: {tag: value}} format diff --git a/caldav/discovery.py b/caldav/discovery.py index c240858b..08c74387 100644 --- a/caldav/discovery.py +++ b/caldav/discovery.py @@ -393,6 +393,10 @@ def discover_service( DNS-based downgrade attacks to plaintext HTTP. Set to False only if you explicitly need to support non-TLS servers and trust your DNS infrastructure. + NOTE: this gates the discovery path only; the client does not + enforce TLS on explicitly-passed URLs. Global enforcement is + deferred to 4.0 — see + https://github.com/python-caldav/caldav/issues/687 Returns: ServiceInfo object with discovered service details, or None if discovery fails @@ -481,10 +485,16 @@ def discover_service( well_known_info = _well_known_lookup(domain, service_type, timeout, ssl_verify_cert) if well_known_info: - # Preserve username from email address - well_known_info.username = username - log.info(f"Discovered {service_type} service via well-known URI: {well_known_info.url}") - return well_known_info + if require_tls and not well_known_info.tls: + log.warning( + f"require_tls=True: Rejecting well-known redirect to non-TLS URL " + f"{well_known_info.url!r} — possible misconfiguration or downgrade attack" + ) + else: + # Preserve username from email address + well_known_info.username = username + log.info(f"Discovered {service_type} service via well-known URI: {well_known_info.url}") + return well_known_info # All discovery methods failed log.warning(f"Failed to discover {service_type} service for {domain}") diff --git a/caldav/jmap/async_client.py b/caldav/jmap/async_client.py index 07f64e57..bcbd2dfe 100644 --- a/caldav/jmap/async_client.py +++ b/caldav/jmap/async_client.py @@ -12,6 +12,7 @@ import logging import uuid +import warnings from niquests import AsyncSession @@ -71,14 +72,50 @@ def _get_http_session(self) -> AsyncSession: self._http_session = sess return self._http_session + async def aclose(self) -> None: + """Release the persistent HTTP session and its connection pool. + + Only needed when the client was not used as an async context manager + -- the documented Quick Start builds one directly. Idempotent; the + session is recreated on the next request. + """ + if self._http_session is not None: + await self._http_session.close() + self._http_session = None + async def __aenter__(self) -> AsyncJMAPClient: self._get_http_session() return self async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: - if self._http_session is not None: - await self._http_session.close() - self._http_session = None + await self.aclose() + + def __del__(self) -> None: + ## Closing an async session needs an event loop, which is long gone by the + ## time __del__ runs, so all we can do is say so. getattr() rather than + ## attribute access: __init__ may have raised before setting it, and an + ## AttributeError here would be reported as "exception ignored in __del__" + ## on top of whatever actually went wrong. + if getattr(self, "_http_session", None) is None: + return + try: + warnings.warn( + f"{type(self).__name__} was garbage collected with an open HTTP " + "session; use 'async with' or await aclose()", + ResourceWarning, + ## stacklevel=1 on purpose, and explicitly because ruff's B028 + ## wants it stated: pointing any higher is a lie in __del__, where + ## the caller is the garbage collector. source= is what makes the + ## warning actionable instead - with `python -X tracemalloc` it + ## reports where the leaked client was allocated. + stacklevel=1, + source=self, + ) + except Exception: + ## __del__ must not raise. At interpreter shutdown the warnings + ## machinery may already be torn down and stderr may be closed, and + ## there is neither anything to recover nor anywhere to report it. + pass async def _get_session(self) -> Session: """Return the cached Session, fetching it on first call.""" diff --git a/caldav/jmap/client.py b/caldav/jmap/client.py index ebbba237..89e60452 100644 --- a/caldav/jmap/client.py +++ b/caldav/jmap/client.py @@ -441,14 +441,33 @@ def _get_http_session(self): self._http_session = sess return self._http_session + def close(self) -> None: + """Release the persistent HTTP session and its connection pool. + + Only needed when the client was not used as a context manager -- the + documented Quick Start builds one directly, and without this there + was no way to hand the sockets back. Idempotent; the session is + recreated on the next request. + """ + if self._http_session is not None: + self._http_session.close() + self._http_session = None + def __enter__(self) -> JMAPClient: self._get_http_session() return self def __exit__(self, exc_type, exc_val, exc_tb) -> None: - if self._http_session is not None: - self._http_session.close() - self._http_session = None + self.close() + + def __del__(self) -> None: + ## Last-resort net for a client that was neither closed nor used as a + ## context manager. Interpreter shutdown can have torn down enough + ## for this to fail, and an exception here is unraisable noise. + try: + self.close() + except Exception: + pass def _get_session(self) -> Session: """Return the cached Session, fetching it on first call.""" diff --git a/caldav/jmap/convert/_utils.py b/caldav/jmap/convert/_utils.py index 475eca85..8f19b493 100644 --- a/caldav/jmap/convert/_utils.py +++ b/caldav/jmap/convert/_utils.py @@ -5,6 +5,7 @@ from __future__ import annotations from datetime import date, datetime, timedelta +from datetime import tzinfo as tzinfo_t def _timedelta_to_duration(td: timedelta) -> str: @@ -110,22 +111,31 @@ def _duration_to_timedelta(duration_str: str) -> timedelta: return sign * td -def _format_local_dt(dt: datetime | date) -> str: +def _format_local_dt(dt: datetime | date, tzinfo: tzinfo_t | None = None) -> str: """Format a datetime or date as a JSCalendar LocalDateTime string. RFC 8984 requires LocalDateTime (no Z suffix) for override keys and RRULE - ``until`` values. Timezone information is stripped — callers must convert - UTC datetimes to the event's local timezone before calling if the event uses - TZID; for floating or all-day events the naive value is already correct. + ``until`` values, and those are expressed in the *event's* timezone. An + aware datetime is therefore converted into ``tzinfo`` before the offset is + dropped; merely stripping it would shift the value by the UTC offset, and + a floating ``UNTIL`` against a TZID ``DTSTART`` is forbidden outright by + RFC 5545 3.3.10. + + ``tzinfo`` is the event's timezone, normally ``DTSTART.dt.tzinfo``. When + it is None the event is floating or all-day: there is nothing to convert + into, so the value is passed through as-is. For date objects (all-day), uses T00:00:00 suffix. Args: dt: A datetime (with or without tzinfo) or a date. + tzinfo: The event's timezone, or None for a floating/all-day event. Returns: Formatted string suitable for use as a JSCalendar override key or RRULE until. """ if isinstance(dt, datetime): + if tzinfo is not None and dt.tzinfo is not None: + dt = dt.astimezone(tzinfo) return dt.strftime("%Y-%m-%dT%H:%M:%S") return f"{dt.isoformat()}T00:00:00" diff --git a/caldav/jmap/convert/ical_to_jscal.py b/caldav/jmap/convert/ical_to_jscal.py index 07b358c4..6bd50e3d 100644 --- a/caldav/jmap/convert/ical_to_jscal.py +++ b/caldav/jmap/convert/ical_to_jscal.py @@ -12,7 +12,6 @@ import uuid from datetime import date, datetime, timedelta -from zoneinfo import ZoneInfo, ZoneInfoNotFoundError import icalendar @@ -26,26 +25,6 @@ "CANCELLED": "cancelled", } - -def _to_event_local(dt, time_zone): - """Shift a tz-aware datetime to naive wall-clock in the event's timeZone. - - RFC 8984 LocalDateTime values (RRULE ``until``, EXDATE override keys) are - expressed in the recurring event's own time zone. A UTC UNTIL/EXDATE on a - TZID event must therefore be converted to that zone before being formatted - as a (suffix-less) LocalDateTime, otherwise the series ends at the wrong - wall-clock time and the round-trip emits a Z-less UNTIL that RFC 5545 - §3.3.10 forbids for TZID events. Floating / all-day / naive values (and - unknown time zones) pass through unchanged. - """ - if time_zone and isinstance(dt, datetime) and dt.tzinfo is not None: - try: - return dt.astimezone(ZoneInfo(time_zone)).replace(tzinfo=None) - except ZoneInfoNotFoundError: - return dt - return dt - - _CLASS_MAP = { "PRIVATE": "private", "CONFIDENTIAL": "secret", @@ -96,14 +75,14 @@ def _dtstart_to_jscal(dtstart_prop) -> tuple[str, str | None, bool]: return dt.strftime("%Y-%m-%dT%H:%M:%S"), None, False -def _rrule_to_jscal(rrule_prop, time_zone=None) -> dict: +def _rrule_to_jscal(rrule_prop, tzinfo=None) -> dict: """Convert an iCalendar RRULE property to a JSCalendar RecurrenceRule dict. Always emits @type, interval, rscale, skip, firstDayOfWeek to match the fields Cyrus returns — makes round-trip comparison predictable. - ``time_zone`` is the event's IANA time zone; a UTC ``UNTIL`` is shifted to - it so the LocalDateTime value matches the event frame (see _to_event_local). + ``tzinfo`` is the event's timezone; ``until`` is a LocalDateTime in that + zone, so a UTC ``UNTIL`` off the wire has to be converted, not truncated. """ rule: dict = { "@type": "RecurrenceRule", @@ -128,7 +107,7 @@ def _rrule_to_jscal(rrule_prop, time_zone=None) -> dict: until_list = rrule_prop.get("UNTIL", []) if until_list: - rule["until"] = _format_local_dt(_to_event_local(until_list[0], time_zone)) + rule["until"] = _format_local_dt(until_list[0], tzinfo) byday_list = rrule_prop.get("BYDAY", []) if byday_list: @@ -176,11 +155,10 @@ def _rrule_to_jscal(rrule_prop, time_zone=None) -> dict: return rule -def _exdate_to_overrides(exdate_prop, time_zone=None) -> dict: +def _exdate_to_overrides(exdate_prop, tzinfo=None) -> dict: """Convert an EXDATE property (single or list) to recurrenceOverrides entries. - ``time_zone`` is the event's IANA time zone; a UTC EXDATE is shifted to it - so the override key matches the occurrence key (see _to_event_local). + ``tzinfo`` is the event's timezone — see :func:`_format_local_dt`. Returns: Dict mapping LocalDateTime/UTCDateTime string → {"excluded": True} @@ -194,7 +172,7 @@ def _exdate_to_overrides(exdate_prop, time_zone=None) -> dict: dts = getattr(ex, "dts", [ex]) for dt_prop in dts: dt = getattr(dt_prop, "dt", dt_prop) - overrides[_format_local_dt(_to_event_local(dt, time_zone))] = {"excluded": True} + overrides[_format_local_dt(dt, tzinfo)] = {"excluded": True} return overrides @@ -336,15 +314,13 @@ def ical_to_jscal(ical_str: str, calendar_id: str | None = None) -> dict: # Split subcomponents into master VEVENTs and override VEVENTs master: icalendar.Event | None = None - overrides_by_recurrence_id: dict[str, icalendar.Event] = {} + override_components: list[icalendar.Event] = [] for component in cal.subcomponents: if not isinstance(component, icalendar.Event): continue if component.get("RECURRENCE-ID") is not None: - # Override instance — key by its recurrence-id datetime - rid = _format_local_dt(component["RECURRENCE-ID"].dt) - overrides_by_recurrence_id[rid] = component + override_components.append(component) elif master is None: master = component @@ -357,6 +333,17 @@ def ical_to_jscal(ical_str: str, calendar_id: str | None = None) -> dict: dtstart_prop = master["DTSTART"] start, time_zone, show_without_time = _dtstart_to_jscal(dtstart_prop) + ## The event's own timezone. Every LocalDateTime slot below (RRULE + ## until, EXDATE keys, RECURRENCE-ID keys) is expressed in it, so it has + ## to be known before any of them can be formatted — which is why the + ## override keys cannot be built in the loop above. + event_tzinfo = getattr(getattr(dtstart_prop, "dt", None), "tzinfo", None) + + overrides_by_recurrence_id: dict[str, icalendar.Event] = { + _format_local_dt(component["RECURRENCE-ID"].dt, event_tzinfo): component + for component in override_components + } + if master.get("DURATION"): duration = _timedelta_to_duration(master["DURATION"].dt) elif master.get("DTEND"): @@ -451,19 +438,19 @@ def ical_to_jscal(ical_str: str, calendar_id: str | None = None) -> dict: if rrules is not None: if not isinstance(rrules, list): rrules = [rrules] - jscal["recurrenceRules"] = [_rrule_to_jscal(r, time_zone) for r in rrules] + jscal["recurrenceRules"] = [_rrule_to_jscal(r, event_tzinfo) for r in rrules] exrules = master.get("EXRULE") if exrules is not None: if not isinstance(exrules, list): exrules = [exrules] - jscal["excludedRecurrenceRules"] = [_rrule_to_jscal(r, time_zone) for r in exrules] + jscal["excludedRecurrenceRules"] = [_rrule_to_jscal(r, event_tzinfo) for r in exrules] recurrence_overrides: dict = {} exdate = master.get("EXDATE") if exdate is not None: - recurrence_overrides.update(_exdate_to_overrides(exdate, time_zone)) + recurrence_overrides.update(_exdate_to_overrides(exdate, event_tzinfo)) for rid_key, child in overrides_by_recurrence_id.items(): # Build a patch: only fields that differ from the master diff --git a/caldav/jmap/convert/jscal_to_ical.py b/caldav/jmap/convert/jscal_to_ical.py index 8267e25b..f947f1dd 100644 --- a/caldav/jmap/convert/jscal_to_ical.py +++ b/caldav/jmap/convert/jscal_to_ical.py @@ -43,7 +43,7 @@ "resource": "RESOURCE", "room": "ROOM", } -# RFC 8984 status -> RFC 5545 STATUS +# RFC 8984 status -> RFC 5545 STATUS; module-level constant (cf. _KIND_TO_CUTYPE etc.) _STATUS_JSCAL_TO_ICAL = { "confirmed": "CONFIRMED", "tentative": "TENTATIVE", diff --git a/caldav/lib/auth.py b/caldav/lib/auth.py index fa4d351e..c15b9ef1 100644 --- a/caldav/lib/auth.py +++ b/caldav/lib/auth.py @@ -28,7 +28,7 @@ def extract_auth_types(header: str) -> set[str]: Reference: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/WWW-Authenticate#syntax """ - return {h.split()[0] for h in header.lower().split(",")} + return {h.split()[0] for h in header.lower().split(",") if h.strip()} def select_auth_type( diff --git a/caldav/lib/error.py b/caldav/lib/error.py index 16d79883..2822c0b7 100644 --- a/caldav/lib/error.py +++ b/caldav/lib/error.py @@ -13,6 +13,12 @@ ## Environmental variables prepended with "PYTHON_CALDAV" are used for debug purposes, ## environmental variables prepended with "CALDAV_" are for connection parameters debug_dump_communication = os.environ.get("PYTHON_CALDAV_COMMDUMP", False) + if debug_dump_communication: + logging.getLogger("caldav").warning( + "PYTHON_CALDAV_COMMDUMP is set: request/response bodies and headers " + "(including credentials and calendar PII) will be written to uniquely-named " + "files under /tmp. These files accumulate indefinitely — remove them when done." + ) ## one of DEBUG_PDB, DEBUG, DEVELOPMENT, PRODUCTION debugmode = os.environ["PYTHON_CALDAV_DEBUGMODE"] except KeyError: diff --git a/caldav/lib/url.py b/caldav/lib/url.py index c2b426e0..3390371b 100644 --- a/caldav/lib/url.py +++ b/caldav/lib/url.py @@ -140,7 +140,13 @@ def canonical(self) -> "URL": """ url = self.unauth() - arr = list(cast(urllib.parse.ParseResult, self.url_parsed)) + # Use url's parsed form (credentials already stripped), not self's. + # Also always build a fresh URL so self is never mutated — unauth() + # returns self when there are no credentials, and the old code then + # overwrote url.url_raw/url_parsed which are the same object as self. + if url.url_parsed is None: + url.url_parsed = cast(urllib.parse.ParseResult, urlparse(str(url))) + arr = list(url.url_parsed) ## quoting path and removing double slashes arr[2] = quote(unquote(url.path.replace("//", "/"))) ## sensible defaults @@ -155,11 +161,7 @@ def canonical(self) -> "URL": portpart = "" arr[1] += portpart - # make sure to delete the string version - url.url_raw = urlunparse(arr) - url.url_parsed = None - - return url + return URL(urlunparse(arr)) def join(self, path: Any) -> "URL": """ diff --git a/caldav/lib/vcal.py b/caldav/lib/vcal.py index fb29f7cd..9bfeaaf2 100644 --- a/caldav/lib/vcal.py +++ b/caldav/lib/vcal.py @@ -6,11 +6,43 @@ import icalendar +from caldav.lib import error from caldav.lib.python_utilities import to_normal_str ## Global counter. We don't want to be too verbose on the users, ref https://github.com/home-assistant/core/issues/86938 fixup_error_loggings = 0 + +def parse_ical(data, context: str | None = None) -> icalendar.Calendar: + """icalendar.Calendar.from_ical(), with a usable error for a body that + holds no iCalendar at all. + + Servers do send such bodies: an empty object in a scheduling inbox, an HTML + error page delivered with a 200, a notification carrying only headers. + from_ical() answers with `ValueError: Found no components where exactly one + is required`, which names neither the object the data came from nor the fact + that it came off the wire - so the traceback tells you almost nothing. + + The guard is deliberately narrow. Data that does contain a component is + handed to icalendar unchanged, whatever it then makes of it: reclassifying + every parse failure as a server error would hide client-side bugs. + + Args: + data: the body, str or bytes. + context: where it came from, typically a URL. Included in the error. + + Raises: + error.ResponseError: when the body contains no iCalendar component. + """ + text = to_normal_str(data) if data is not None else "" + if "BEGIN:" not in text.upper(): + where = f" from {context}" if context else "" + raise error.ResponseError( + f"no iCalendar data{where}: the body holds no BEGIN: line, got {text.strip()[:200]!r}" + ) + return icalendar.Calendar.from_ical(data) + + ## Fixups to the icalendar data to work around compatibility issues. ## TODO: @@ -77,20 +109,28 @@ def fix(event): ## TODO: add ^ before COMPLETED and CREATED? ## 1) Add an arbitrary time if completed is given as date - fixed = re.sub(r"COMPLETED(?:;VALUE=DATE)?:(\d+)\s", r"COMPLETED:\g<1>T120000Z", event) + fixed = re.sub(r"COMPLETED(?:;VALUE=DATE)?:(\d+)(?=\s)", r"COMPLETED:\g<1>T120000Z", event) ## 2) CREATED timestamps prior to epoch does not make sense, ## change from year 0001 to epoch. fixed = re.sub("CREATED:00001231T000000Z", "CREATED:19700101T000000Z", fixed) - fixed = re.sub(r"\\+('\")", r"\1", fixed) + fixed = re.sub(r"\\+(['\"])", r"\1", fixed) - ## 4) trailing whitespace probably never makes sense - fixed = re.sub(" *$", "", fixed) + ## 4) trailing whitespace probably never makes sense -- but only on a + ## line that is not continued by a folded line. RFC 5545 3.1 folds + ## blind at 75 octets, so the fold may land right after a space that is + ## part of the value; stripping it would join two words together. The + ## negative lookahead is what keeps that whitespace alone. + fixed = re.sub(r"[ \t]+$(?!\n[ \t])", "", fixed, flags=re.MULTILINE) ## 6) add DTSTAMP if not given ## (corner case that DTSTAMP is given in one but not all the recurrences is ignored) if "\nDTSTAMP:" not in fixed: - assert "\nEND" in fixed + if "\nEND" not in fixed: + logging.getLogger(__name__).warning( + "vcal.fix(): truncated iCalendar data (no END: line) — skipping DTSTAMP fixup" + ) + return fixed dtstamp = datetime.datetime.now(tz=datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ") fixed = re.sub("(\nEND:(VTODO|VEVENT|VJOURNAL))", f"\nDTSTAMP:{dtstamp}\\1", fixed) @@ -240,8 +280,8 @@ def create_ical(ical_fragment=None, objtype=None, language="en_DK", **props): ret = to_normal_str(my_instance.to_ical()) if ical_fragment and ical_fragment.strip(): ret = re.sub( - "^END:V", - ical_fragment.strip() + "\nEND:V", + "^(END:V(?:EVENT|TODO|JOURNAL))", + ical_fragment.strip() + "\n\\1", ret, flags=re.MULTILINE, count=1, diff --git a/caldav/response.py b/caldav/response.py index 0b27d5b9..a8b1205e 100644 --- a/caldav/response.py +++ b/caldav/response.py @@ -116,22 +116,38 @@ def _strip_to_multistatus(tree: _Element) -> "_Element | list[_Element]": return [tree] -def _extract_properties(propstats: "list[_Element]") -> "dict[str, Any]": - """Extract properties from propstat elements into a flat dict.""" - properties: dict[str, Any] = {} +def _collect_prop_elements(propstats: "list[_Element]") -> "dict[str, _Element]": + """Collect ``{proptag: element}`` from a list of propstat elements. + + Each propstat status is validated first: anything but 200/201/207/404 + raises :class:`error.ResponseError`, since a ``500``/``403``/``507`` + propstat means the server failed to answer, not that the property is + unset. Swallowing it would hand the caller a silently-empty value. + + Propstats whose status reports 404 are then skipped — that is the single, + shared expression of the "a 404 propstat means the property is absent on + the resource" quirk. This helper is the one place both the dataclass + parsers (via :func:`_extract_properties`) and the legacy + :meth:`DAVResponse._find_objects_and_props` collect prop children, so both + the validation and the quirk stop having to be maintained in two parallel + loops (code-review §5.7). + """ + collected: dict[str, _Element] = {} for propstat in propstats: status_elem = propstat.find(dav.Status.tag) - if status_elem is not None and status_elem.text and " 404 " in status_elem.text: - continue - prop = propstat.find(dav.Prop.tag) - if prop is None: - continue - for child in prop: - if len(child) == 0: - properties[child.tag] = child.text - else: - properties[child.tag] = _element_to_value(child) - return properties + if status_elem is not None and status_elem.text: + _validate_status(status_elem.text) + if " 404 " in status_elem.text: + continue + for prop in propstat.iterfind(dav.Prop.tag): + for child in prop: + collected[child.tag] = child + return collected + + +def _extract_properties(propstats: "list[_Element]") -> "dict[str, Any]": + """Extract properties from propstat elements into a flat dict of parsed values.""" + return {tag: _element_to_value(el) for tag, el in _collect_prop_elements(propstats).items()} def _element_to_value(elem: _Element) -> Any: @@ -274,7 +290,12 @@ def _init_from_response(self, response: "Response", davclient: Any = None) -> No # We'll try to parse the content as XML no matter the content type. self.tree = etree.XML( self._raw, - parser=etree.XMLParser(remove_blank_text=True, huge_tree=self.huge_tree), + parser=etree.XMLParser( + remove_blank_text=True, + huge_tree=self.huge_tree, + resolve_entities=False, + no_network=True, + ), ) except Exception: # Content wasn't XML. What does the content-type say? @@ -474,28 +495,23 @@ def _parse_response(self, response: _Element) -> tuple[str, list[_Element], Any href = _normalize_href(elem.text or "") elif elem.tag == dav.PropStat.tag: propstats.append(elem) - elif elem.tag == "{DAV:}responsedescription": - ## This happens with Stalwart on a 404. - ## This code is mostly moot, but in debug - ## mode I want to be sure we do not toss away any data - error.assert_(elem.text == "No resources found") - check_404 = True - elif elem.tag == "{DAV:}error": - ## This happens with purelymail on a 404. - ## This code is mostly moot, but in debug - ## mode I want to be sure we do not toss away any data - children = elem.getchildren() - error.assert_(len(children) == 1) - error.assert_(children[0].tag == "{https://purelymail.com}does-not-exist") + elif elem.tag in ("{DAV:}responsedescription", "{DAV:}error"): + ## Both are optional children of per RFC 4918 + ## and carry server-defined content. We've seen them on + ## 404s (Stalwart sends No resources + ## found, purelymail sends + ## <…:does-not-exist/>). check_404 = True else: - ## i.e. purelymail may contain one more tag, ... - ## This is probably not a breach of the standard. It may - ## probably be ignored. But it's something we may want to - ## know. + ## A tag we don't recognise at all (e.g. a server inventing + ## an element). Not necessarily a standards + ## breach and probably ignorable, but worth surfacing. error.weirdness("unexpected element found in response", elem) error.assert_(href) - if check_404: + if check_404 and status: + ## We've only ever observed / on + ## 404s; flag it in debug mode if a server pairs them with some + ## other status so we notice and revisit this handling. error.assert_("404" in status) return (cast(str, href), propstats, status) @@ -579,6 +595,35 @@ def sync_token(self): ## protocol.xml_parsers layer is a better approach. Look for more ## cases of old code that was is still remaining after the ## protocol layer refactoring + def all_responses_not_found(self) -> bool: + """True if the multistatus consists solely of response-level 404s. + + RFC 4918 §14.24 lets a ```` carry a bare ```` + instead of one or more ```` elements, so a server may + report "this resource does not exist" inside a 207 Multi-Status + rather than as a transport-level 404. Xandikos answers PROPFIND on + a missing collection that way (while answering REPORT on the very + same URL with a plain 404). + + A 404 for one href among several is normal on ``Depth: 1`` and must + not be treated as "the resource is gone", hence the requirement that + *every* response reports 404 and none carries properties. + """ + if self.tree is None: + return False + responses = [r for r in self._strip_to_multistatus() if r.tag == dav.Response.tag] + if not responses: + return False + for response in responses: + ## a direct-child ; the ones nested inside + ## are a different thing and handled by _collect_prop_elements + if response.find(dav.PropStat.tag) is not None: + return False + status = response.find(dav.Status.tag) + if status is None or "404" not in (status.text or ""): + return False + return True + def _find_objects_and_props(self) -> dict[str, dict[str, _Element]]: """Internal implementation of find_objects_and_props without deprecation warning.""" self.objects: dict[str, dict[str, _Element]] = {} @@ -606,27 +651,11 @@ def _find_objects_and_props(self) -> dict[str, dict[str, _Element]]: self.objects[href] = {} self.statuses[href] = status - ## The properties may be delivered either in one - ## propstat with multiple props or in multiple - ## propstat - for propstat in propstats: - cnt = 0 - status = propstat.find(dav.Status.tag) - error.assert_(status is not None) - if status is not None and status.text is not None: - error.assert_(len(status) == 0) - cnt += 1 - self.validate_status(status.text) - ## if a prop was not found, ignore it - if " 404 " in status.text: - continue - for prop in propstat.iterfind(dav.Prop.tag): - cnt += 1 - for theprop in prop: - self.objects[href][theprop.tag] = theprop - - ## there shouldn't be any more elements except for status and prop - error.assert_(cnt == len(propstat)) + ## The properties may be delivered either in one propstat + ## with multiple props or in multiple propstats; the 404-skip + ## quirk is shared with the dataclass parsers via + ## _collect_prop_elements (code-review §5.7). + self.objects[href].update(_collect_prop_elements(propstats)) return self.objects diff --git a/caldav/search.py b/caldav/search.py index 5b853382..061a336c 100644 --- a/caldav/search.py +++ b/caldav/search.py @@ -17,6 +17,8 @@ from .lib import error if TYPE_CHECKING: + from collections.abc import Generator + from .calendarobjectresource import ( CalendarObjectResource as AsyncCalendarObjectResource, ) @@ -190,7 +192,8 @@ def _build_search_xml_query( for property in searcher._property_operator: if searcher._property_operator[property] == "undef": match = cdav.NotDefined() - filters.append(cdav.PropFilter(property.upper()) + match) + prop_name = "CATEGORIES" if property.lower() == "category" else property.upper() + filters.append(cdav.PropFilter(prop_name) + match) else: value = searcher._property_filters[property] property_ = property.upper() @@ -232,6 +235,23 @@ def _build_search_xml_query( return (root, comp_class) +def _dedup_by_url(matches: list) -> list: + """Drop repeated resources, keeping the first occurrence and the order. + + A search that is split into several server queries can return the same + resource more than once: the include-completed split issues overlapping + queries, and in a comp-type split a resource that legally holds both a + VEVENT and a VTODO matches two of the three queries. + """ + objects = [] + seen = set() + for item in matches: + if item.url not in seen: + seen.add(item.url) + objects.append(item) + return objects + + def _is_not_defined_supported(features: Any, prop: str) -> bool: """Check if is-not-defined search is supported for a specific property. @@ -267,9 +287,39 @@ class SearchAction(Enum): SEARCH_WITH_COMPTYPES = auto() # (args) -> search with all comp types REQUEST_REPORT = auto() # (xml, comp_class, props) -> make CalDAV request LOAD_OBJECT = auto() # (obj) -> load object data + LOAD_OBJECTS_BATCH = ( + auto() + ) # (calendar, objects) -> batch-load via calendar._batch_load_objects RETURN = auto() # (result) -> return this value +def _advance_search_gen( + gen: "Generator[tuple[SearchAction, Any], Any, None]", + result: Any = None, + exc: BaseException | None = None, +) -> "tuple[SearchAction, Any] | None": + """Phase 2 of the search driver protocol, shared by sync and async drivers. + + Feed the Phase-1 ``result`` (or the ``exc`` raised while executing the + yielded action) back into the search generator. Feeding an exception via + ``gen.throw()`` lets the search logic's own try/except blocks act on it + (the issue #681 time-range fallback, per-object load error handling, ...); + if the generator does not handle it, ``gen.throw()`` re-raises it out of + here, which is the correct propagation. + + Passing ``result=None, exc=None`` on a fresh generator primes it. + + :return: the next ``(action, data)`` to execute, or ``None`` when the + generator is exhausted (StopIteration → the driver returns ``[]``). + """ + try: + if exc is not None: + return gen.throw(exc) + return gen.send(result) + except StopIteration: + return None + + @dataclass class CalDAVSearcher(Searcher): """The baseclass (which is generic, and not CalDAV-specific) @@ -316,6 +366,12 @@ class CalDAVSearcher(Searcher): comp_class: Optional["CalendarObjectResource"] = None _explicit_operators: set = field(default_factory=set) _calendar: Optional["Calendar"] = field(default=None, repr=False) + ## When False, all server-compatibility workarounds in _search_impl are + ## disabled and the query the searcher describes is sent verbatim (a single + ## REPORT, no comp-type splitting, no filter rewriting, no fallback retries). + ## Used by the server-compatibility checker to observe raw server behaviour. + ## Propagates to clones automatically via dataclasses.replace(). + _compatibility_workarounds: bool = True def add_property_filter( self, @@ -455,12 +511,18 @@ def _search_impl( "create the searcher via calendar.searcher()" ) + ## When disabled, every server-compatibility workaround below is skipped + ## and the query is sent verbatim (used by the compatibility checker to + ## observe raw server behaviour). + cw = self._compatibility_workarounds + ## Workaround for servers where REPORT without a time range only returns ## objects within a sliding window (search.unlimited-time-range: broken). ## Inject a wide time range covering 1970–2126 so that year-2000 test ## objects and other old data are returned. if ( - not self.start + cw + and not self.start and not self.end and not (self.expand or server_expand) and not calendar.client.features.is_supported("search.unlimited-time-range") @@ -480,7 +542,8 @@ def _search_impl( ## Handle servers with broken component-type filtering (e.g., Bedework) comp_type_support = calendar.client.features.is_supported("search.comp-type", str) no_comp_filter = ( - (self.comp_class or self.todo or self.event or self.journal) + cw + and (self.comp_class or self.todo or self.event or self.journal) and comp_type_support == "broken" and post_filter is not False ) @@ -490,13 +553,18 @@ def _search_impl( post_filter = True ## Setting default value for post_filter - if post_filter is None and ( - (self.todo and not self.include_completed) - or self.expand - or "categories" in self._property_filters - or "category" in self._property_filters - or not calendar.client.features.is_supported("search.text.case-sensitive") - or not calendar.client.features.is_supported("search.time-range.accurate") + if ( + cw + and post_filter is None + and ( + (self.todo and not self.include_completed) + or self.expand + or "categories" in self._property_filters + or "category" in self._property_filters + or any(op == "==" for op in self._property_operator.values()) + or not calendar.client.features.is_supported("search.text.case-sensitive") + or not calendar.client.features.is_supported("search.time-range.accurate") + ) ): post_filter = True @@ -508,7 +576,8 @@ def _search_impl( ## expansion is unreliable (the master expands without knowing its exceptions, yielding ## duplicate occurrences). Fall back to server-side expansion when it handles exceptions. if ( - self.expand + cw + and self.expand and not server_expand and not calendar.client.features.is_supported("save-load.event.recurrences.exception") and calendar.client.features.is_supported("search.recurrences.expanded.exception") @@ -523,7 +592,8 @@ def _search_impl( ## (e.g. purelymail where both i;octet and i;ascii-casemap collations are unsupported). ## Remove all text-value filters and rely on client-side post_filter instead. if ( - not calendar.client.features.is_supported("search.text") + cw + and not calendar.client.features.is_supported("search.text") and self._property_filters and post_filter is not False ): @@ -545,7 +615,8 @@ def _search_impl( ## special compatbility-case for servers that does not ## support category search properly if ( - not calendar.client.features.is_supported("search.text.category") + cw + and not calendar.client.features.is_supported("search.text.category") and ("categories" in self._property_filters or "category" in self._property_filters) and post_filter is not False ): @@ -562,7 +633,7 @@ def _search_impl( ## special compatibility-case for servers that do not support is-not-defined ## for specific properties (e.g. search.is-not-defined.category or .dtend) - if post_filter is not False: + if cw and post_filter is not False: undef_props_without_support = [ prop for prop, op in self._property_operator.items() @@ -587,7 +658,8 @@ def _search_impl( ## special compatibility-case for servers that do not support substring search if ( - not calendar.client.features.is_supported("search.text.substring") + cw + and not calendar.client.features.is_supported("search.text.substring") and post_filter is not False ): explicit_contains = [ @@ -614,7 +686,7 @@ def _search_impl( ## special compatibility-case for servers that does not ## support combined searches very well - if not calendar.client.features.is_supported("search.combined-is-logical-and"): + if cw and not calendar.client.features.is_supported("search.combined-is-logical-and"): if self.start or self.end: if self._property_filters: clone = self._clone_without_filters(clear_all_filters=True) @@ -624,7 +696,7 @@ def _search_impl( ) yield ( SearchAction.RETURN, - self.filter(objects, post_filter, split_expanded, server_expand), + self.filter(objects, True, split_expanded, server_expand), ) return @@ -657,7 +729,7 @@ def _search_impl( ## TODO: consider if not ignore_completed3 is sufficient, ## then the recursive part of the query here is moot, and ## we wouldn't waste so much time on repeated queries - if self.todo and self.include_completed is False: + if cw and self.todo and self.include_completed is False: clone = replace(self, include_completed=True) clone.include_completed = True ## Why? Isn't this redundant? clone.expand = False @@ -688,13 +760,7 @@ def _search_impl( (clone, calendar, server_expand, False, props, xml, None, _hacks), ) - # Deduplicate by URL - objects = [] - match_set = set() - for item in matches: - if item.url not in match_set: - match_set.add(item.url) - objects.append(item) + objects = _dedup_by_url(matches) else: orig_xml = xml @@ -703,12 +769,47 @@ def _search_impl( server_expand, props=props, filters=xml, _hacks=_hacks ) - if not self.comp_class and not calendar.client.features.is_supported( - "search.comp-type.optional" - ): - if self.include_completed is None: - self.include_completed = True - + ## A CALDAV:time-range (and VALARM) filter is a component-level filter: + ## RFC4791 section 9.7 only allows it inside a comp-filter for + ## VEVENT/VTODO/VJOURNAL/VFREEBUSY/VALARM, never directly under VCALENDAR. + ## So when no component type is given we cannot place such a filter in an + ## RFC-legal way - we must split the search into one query per component + ## type (search.time-range.comp-type-optional). This is independent of + ## search.comp-type.optional, which only governs comp-type-less queries + ## WITHOUT any filter. + ## The same applies to a prop-filter (CATEGORIES, SUMMARY, ...): under + ## VCALENDAR it would filter on VCALENDAR's own properties (which lack + ## component properties), so servers match nothing + ## (search.text.comp-type-optional). + ## See https://github.com/python-caldav/caldav/issues/681 + has_component_level_filter = bool( + self.start or self.end or self.alarm_start or self.alarm_end + ) + has_property_filter = bool(self._property_filters) + needs_comptype_split = ( + cw + and not self.comp_class + and ( + not calendar.client.features.is_supported("search.comp-type.optional") + or ( + has_component_level_filter + and not calendar.client.features.is_supported( + "search.time-range.comp-type-optional" + ) + ) + or ( + has_property_filter + and not calendar.client.features.is_supported( + "search.text.comp-type-optional" + ) + ) + ) + ) + if needs_comptype_split: + ## The include_completed default for the split is resolved + ## inside _search_with_comptypes, on a clone. Setting it on + ## self here would permanently change the meaning of the + ## caller's searcher - the issue-#650 class of bug. result = yield ( SearchAction.SEARCH_WITH_COMPTYPES, (calendar, server_expand, split_expanded, props, orig_xml, _hacks, post_filter), @@ -722,8 +823,37 @@ def _search_impl( (calendar, xml, self.comp_class, props), ) except error.ReportError as err: + ## Reactive workaround for https://github.com/python-caldav/caldav/issues/681: + ## if the server was (optimistically) configured as supporting + ## search.time-range.comp-type-optional but actually rejects the + ## comp-type-less time-range query (e.g. SabreDAV's HTTP 400 "You cannot + ## add time-range filters on the VCALENDAR component"), retry by splitting + ## into one query per component type. Also covers prop-filters + ## (search.text.comp-type-optional). orig_xml must be empty - if the + ## caller passed a full calendar-query we cannot rebuild it per comp-type. if ( - calendar.client.features.backward_compatibility_mode + cw + and not self.comp_class + and not orig_xml + and (has_component_level_filter or has_property_filter) + ): + result = yield ( + SearchAction.SEARCH_WITH_COMPTYPES, + ( + calendar, + server_expand, + split_expanded, + props, + orig_xml, + _hacks, + post_filter, + ), + ) + yield (SearchAction.RETURN, result) + return + if ( + cw + and calendar.client.features.backward_compatibility_mode and not self.comp_class and "400" not in err.reason ): @@ -780,29 +910,17 @@ def _search_impl( ) return - # Post-process: load objects - obj2 = [] - for o in objects: - try: - yield (SearchAction.LOAD_OBJECT, o) - obj2.append(o) - except Exception: - logging.error( - "Server does not want to reveal details about the calendar object", - exc_info=True, - ) - objects = obj2 + # Post-process: batch-load unloaded objects in one REPORT instead of N GETs + yield (SearchAction.LOAD_OBJECTS_BATCH, (calendar, objects)) + objects = [o for o in objects if o.is_loaded() or o.has_component()] # Google sometimes returns empty objects objects = [o for o in objects if o.has_component()] objects = self.filter(objects, post_filter, split_expanded, server_expand) # Partial workaround for https://github.com/python-caldav/caldav/issues/201 - for obj in objects: - try: - yield (SearchAction.LOAD_OBJECT, obj) - except Exception: - pass + # Re-issue a batch load in case any objects need a second fetch + yield (SearchAction.LOAD_OBJECTS_BATCH, (calendar, objects)) yield (SearchAction.RETURN, self.sort(objects)) @@ -815,6 +933,7 @@ def search( xml: str = None, post_filter=None, _hacks: str = None, + compatibility_workarounds: bool | None = None, ) -> list[CalendarObjectResource]: """Do the search on a CalDAV calendar. @@ -831,6 +950,13 @@ def search( :param xml: XML query to be sent to the server (string or elements) :param post_filter: Do client-side filtering after querying the server :param _hacks: Please don't ask! + :param compatibility_workarounds: When ``False``, all server-compatibility + workarounds are disabled and the query is sent verbatim + (single REPORT, no comp-type splitting, no filter + rewriting, no fallback retries). Mainly for the + server-compatibility checker, to observe raw server + behaviour. ``None`` (the default) leaves the searcher's + current setting unchanged. Make sure not to confuse he CalDAV properties with iCalendar properties. @@ -851,36 +977,52 @@ def search( flag on. """ + if compatibility_workarounds is not None: + self._compatibility_workarounds = compatibility_workarounds gen = self._search_impl( calendar, server_expand, split_expanded, props, xml, post_filter, _hacks ) - result = None - try: - action, data = gen.send(result) - except StopIteration: - return [] - - while True: + ## The driver alternates Phase 1 (execute the yielded action, here) and + ## Phase 2 (feed the result/exception back, in _advance_search_gen). Only + ## Phase 1 differs between sync and async; the generator protocol is shared. + step = _advance_search_gen(gen) # prime the generator + while step is not None: + action, data = step + if action == SearchAction.RETURN: + return data + result = exc = None try: - if action == SearchAction.RECURSIVE_SEARCH: - clone, cal, srv_exp, spl_exp, prp, xm, pf, hk = data - result = clone.search(cal, srv_exp, spl_exp, prp, xm, pf, hk) - elif action == SearchAction.SEARCH_WITH_COMPTYPES: - cal, srv_exp, spl_exp, prp, xm, hk, pf = data - result = self._search_with_comptypes(cal, srv_exp, spl_exp, prp, xm, hk, pf) - elif action == SearchAction.REQUEST_REPORT: - cal, xm, comp_cls, prp = data - result = cal._request_report_build_resultlist(xm, comp_cls, props=prp) - elif action == SearchAction.LOAD_OBJECT: - data.load(only_if_unloaded=True) - result = None - elif action == SearchAction.RETURN: - return data - - action, data = gen.send(result) - except StopIteration: - return [] + result = self._dispatch_search_action(action, data) + except Exception as e: + exc = e + step = _advance_search_gen(gen, result, exc) + return [] + + def _dispatch_search_action(self, action: SearchAction, data: Any) -> Any: + """Phase 1 of the sync search driver: execute one yielded SearchAction. + + Returns the value to feed back into the generator (``None`` for actions + whose effect is a side effect). ``RETURN`` is handled by the driver + loop itself. Sync twin of :meth:`_async_dispatch_search_action`. + """ + if action == SearchAction.RECURSIVE_SEARCH: + clone, cal, srv_exp, spl_exp, prp, xm, pf, hk = data + return clone.search(cal, srv_exp, spl_exp, prp, xm, pf, hk) + if action == SearchAction.SEARCH_WITH_COMPTYPES: + cal, srv_exp, spl_exp, prp, xm, hk, pf = data + return self._search_with_comptypes(cal, srv_exp, spl_exp, prp, xm, hk, pf) + if action == SearchAction.REQUEST_REPORT: + cal, xm, comp_cls, prp = data + return cal._request_report_build_resultlist(xm, comp_cls, props=prp) + if action == SearchAction.LOAD_OBJECT: + data.load(only_if_unloaded=True) + return None + if action == SearchAction.LOAD_OBJECTS_BATCH: + cal, objs = data + cal._batch_load_objects(objs) + return None + raise AssertionError(f"unhandled search action {action!r}") def _search_with_comptypes( self, @@ -894,6 +1036,10 @@ def _search_with_comptypes( ) -> list[CalendarObjectResource]: """ Internal method - does three searches, one for each comp class (event, journal, todo). + + Note that the results come back grouped by component type rather than + in server order; three queries cannot preserve an order that only one + query ever had. Pass a sort key if the order matters. """ if xml and (isinstance(xml, str) or "calendar-query" in xml.tag): # Full XML provided – cannot inject a comp-type filter into it. @@ -904,19 +1050,19 @@ def _search_with_comptypes( return self.sort(objects) objects = [] - assert self.event is None and self.todo is None and self.journal is None + base = self._comptype_split_base() for comp_class in (Event, Todo, Journal): if not calendar.client.features.is_supported( f"save-load.{comp_class.__name__.lower()}" ): continue - clone = replace(self) + clone = replace(base) clone.comp_class = comp_class objects += clone.search( calendar, server_expand, split_expanded, props, xml, post_filter, _hacks ) - return self.sort(objects) + return self.sort(_dedup_by_url(objects)) async def async_search( self, @@ -927,6 +1073,7 @@ async def async_search( xml: str = None, post_filter=None, _hacks: str = None, + compatibility_workarounds: bool | None = None, ) -> list["AsyncCalendarObjectResource"]: """Async version of search() - does the search on an AsyncCalendar. @@ -935,40 +1082,52 @@ async def async_search( See the sync search() method for full documentation. """ + if compatibility_workarounds is not None: + self._compatibility_workarounds = compatibility_workarounds gen = self._search_impl( calendar, server_expand, split_expanded, props, xml, post_filter, _hacks ) - result = None - - try: - action, data = gen.send(result) - except StopIteration: - return [] - while True: + ## See the sync search() driver: only Phase 1 (the action execution) is + ## awaited here; the Phase-2 generator protocol is shared via + ## _advance_search_gen. + step = _advance_search_gen(gen) # prime the generator + while step is not None: + action, data = step + if action == SearchAction.RETURN: + return data + result = exc = None try: - if action == SearchAction.RECURSIVE_SEARCH: - clone, cal, srv_exp, spl_exp, prp, xm, pf, hk = data - result = await clone.async_search(cal, srv_exp, spl_exp, prp, xm, pf, hk) - elif action == SearchAction.SEARCH_WITH_COMPTYPES: - cal, srv_exp, spl_exp, prp, xm, hk, pf = data - result = await self._async_search_with_comptypes( - cal, srv_exp, spl_exp, prp, xm, hk, pf - ) - elif action == SearchAction.REQUEST_REPORT: - cal, xm, comp_cls, prp = data - result = await cal._request_report_build_resultlist(xm, comp_cls, props=prp) - elif action == SearchAction.LOAD_OBJECT: - load_result = data.load(only_if_unloaded=True) - if inspect.isawaitable(load_result): - await load_result - result = None - elif action == SearchAction.RETURN: - return data - - action, data = gen.send(result) - except StopIteration: - return [] + result = await self._async_dispatch_search_action(action, data) + except Exception as e: + exc = e + step = _advance_search_gen(gen, result, exc) + return [] + + async def _async_dispatch_search_action(self, action: SearchAction, data: Any) -> Any: + """Phase 1 of the async search driver: execute one yielded SearchAction. + + Async twin of :meth:`_dispatch_search_action`; see it for semantics. + """ + if action == SearchAction.RECURSIVE_SEARCH: + clone, cal, srv_exp, spl_exp, prp, xm, pf, hk = data + return await clone.async_search(cal, srv_exp, spl_exp, prp, xm, pf, hk) + if action == SearchAction.SEARCH_WITH_COMPTYPES: + cal, srv_exp, spl_exp, prp, xm, hk, pf = data + return await self._async_search_with_comptypes(cal, srv_exp, spl_exp, prp, xm, hk, pf) + if action == SearchAction.REQUEST_REPORT: + cal, xm, comp_cls, prp = data + return await cal._request_report_build_resultlist(xm, comp_cls, props=prp) + if action == SearchAction.LOAD_OBJECT: + load_result = data.load(only_if_unloaded=True) + if inspect.isawaitable(load_result): + await load_result + return None + if action == SearchAction.LOAD_OBJECTS_BATCH: + cal, objs = data + await cal._async_batch_load_objects(objs) + return None + raise AssertionError(f"unhandled search action {action!r}") async def _async_search_with_comptypes( self, @@ -990,20 +1149,20 @@ async def _async_search_with_comptypes( return self.sort(objects) objects: list[AsyncCalendarObjectResource] = [] - assert self.event is None and self.todo is None and self.journal is None + base = self._comptype_split_base() for comp_class in (Event, Todo, Journal): if not calendar.client.features.is_supported( f"save-load.{comp_class.__name__.lower()}" ): continue - clone = replace(self) + clone = replace(base) clone.comp_class = comp_class results = await clone.async_search( calendar, server_expand, split_expanded, props, xml, post_filter, _hacks ) objects.extend(results) - return self.sort(objects) + return self.sort(_dedup_by_url(objects)) def filter( self, @@ -1039,6 +1198,27 @@ def filter( server_expand=server_expand, ) + def _comptype_split_base(self) -> "CalDAVSearcher": + """Return the searcher the per-comp-type clones are built from. + + A truthy ``event``/``todo``/``journal`` flag would have produced a + ``comp_class``, and the split only happens when there is none — so + reaching here with one set means the caller and the driver disagree. + ``event=False`` is *not* such a case: it is a legal argument to the + public ``search()``, and testing it with ``is None`` used to trip a + bare, message-less ``AssertionError``. + + ``include_completed`` defaults to True for the split (otherwise the + VTODO sub-search would quietly drop completed tasks), but that is + resolved on a copy: writing it back to ``self`` would change the + meaning of the caller's searcher for every later call — the + issue-#650 class of bug. + """ + error.assert_(not (self.event or self.todo or self.journal)) + if self.include_completed is None: + return replace(self, include_completed=True) + return self + def build_search_xml_query(self, server_expand=False, props=None, filters=None, _hacks=None): """Build a CalDAV calendar-query XML request. diff --git a/caldav/testing.py b/caldav/testing.py index 1211f30b..f87c2730 100644 --- a/caldav/testing.py +++ b/caldav/testing.py @@ -9,6 +9,7 @@ Docker and external server support lives in tests/test_servers/ (source only). """ +import copy import socket import tempfile import threading @@ -123,7 +124,7 @@ def __init__(self, config: dict[str, Any] | None = None) -> None: if "features" not in config: from caldav import compatibility_hints - features = compatibility_hints.xandikos.copy() + features = copy.deepcopy(compatibility_hints.xandikos) features["auto-connect.url"]["domain"] = f"{config['host']}:{config['port']}" config["features"] = features super().__init__(config) @@ -265,7 +266,7 @@ def __init__(self, config: dict[str, Any] | None = None) -> None: if "features" not in config: from caldav import compatibility_hints - features = compatibility_hints.radicale.copy() + features = copy.deepcopy(compatibility_hints.radicale) features["auto-connect.url"]["domain"] = f"{config['host']}:{config['port']}" config["features"] = features super().__init__(config) diff --git a/docs/design/FEATURE_COMPLETE_ROADMAP.md b/docs/design/FEATURE_COMPLETE_ROADMAP.md index 2a9e185d..bd60f54d 100644 --- a/docs/design/FEATURE_COMPLETE_ROADMAP.md +++ b/docs/design/FEATURE_COMPLETE_ROADMAP.md @@ -1,13 +1,12 @@ # Feature-Complete CalDAV Library Roadmap -**Created:** 2026-01-28 -**Author:** AI-generated based on RFC analysis and open issues -**Status:** Planning document for work after issue #599 completion -**Branch:** v3.0-dev +- **Created:** 2026-01-28, **updated** 2026-08-20 +- **Author:** AI-generated and human-edited based on RFC analysis and open issues +- **Status:** Planning document for work after issue [#599](https://github.com/python-caldav/caldav/issues/599) completion ## Overview -This document outlines the work needed to make the caldav library a **feature-complete CalDAV client** per the relevant IETF RFCs. It is intended as a continuation of the roadmap in issue #599, covering features beyond the v3.0 and v3.2 releases. +This document outlines the work needed to make the caldav library a **feature-complete CalDAV client** per the relevant IETF RFCs. It is intended as a continuation of the roadmap in issue [#599](https://github.com/python-caldav/caldav/issues/599), covering features beyond v3.3. ### Scope @@ -18,18 +17,22 @@ The caldav library already implements: - WebDAV sync (RFC 6578) - Extensive search capabilities - Async support +- JMAP (`caldav/jmap/`) — not a CalDAV RFC, and not covered by this roadmap This roadmap covers the **remaining gaps** to achieve full RFC compliance and addresses open feature requests. +Items are ordered by phase and priority; which release they land in is decided when the release is planned. Work that has to break the API is marked as v4.0 material where it appears. + --- ## Phase 1: RFC Compliance - Core Features ### 1.1 WebDAV Access Control (RFC 3744) - ACL Support -**Priority:** High -**Estimated effort:** 40-60 hours -**RFC:** [RFC 3744](https://datatracker.ietf.org/doc/html/rfc3744) +- **Priority:** High +- **Estimated effort:** 40-60 hours +- **RFC:** [RFC 3744](https://datatracker.ietf.org/doc/html/rfc3744) +- **Related issues:** [#699](https://github.com/python-caldav/caldav/issues/699), [#701](https://github.com/python-caldav/caldav/issues/701) (3.2 — the sharing alternative to the same problem) Current state: The library has basic principal support but lacks ACL manipulation. @@ -42,37 +45,34 @@ Current state: The library has basic principal support but lacks ACL manipulatio - [ ] Implement inherited ACL support - [ ] Add helper methods for common permission patterns (read-only, read-write, owner) -**Related issues:** None currently open - --- ### 1.2 Improved Scheduling (RFC 6638) -**Priority:** High -**Estimated effort:** 40 hours (partially covered in #599 for v3.2) -**RFC:** [RFC 6638](https://datatracker.ietf.org/doc/html/rfc6638) +- **Priority:** High +- **Estimated effort:** 40 hours (partially covered in [#599](https://github.com/python-caldav/caldav/issues/599) for v3.2) +- **RFC:** [RFC 6638](https://datatracker.ietf.org/doc/html/rfc6638) +- **Related issues:** [#524](https://github.com/python-caldav/caldav/issues/524), [#399](https://github.com/python-caldav/caldav/issues/399), [#596](https://github.com/python-caldav/caldav/issues/596), [#544](https://github.com/python-caldav/caldav/issues/544) — all four are now closed; what remains here is the iTIP/`SCHEDULE-AGENT`/delegation work, which has no issue of its own The v3.2 roadmap covers basic scheduling improvements. Additional work for full compliance: **Tasks:** -- [ ] Complete Schedule-Tag header support (`If-Schedule-Tag-Match`) +- [x] Complete Schedule-Tag header support (`If-Schedule-Tag-Match`) — **done in v3.2.0**, see [#660](https://github.com/python-caldav/caldav/issues/660) - [ ] Full iTIP method support: REQUEST, REPLY, CANCEL, ADD, REFRESH, COUNTER, DECLINECOUNTER - [ ] Implicit scheduling with `SCHEDULE-AGENT` parameter handling -- [ ] `SEQUENCE` property management per iTIP requirements +- [x] `SEQUENCE` property management per iTIP requirements — **done in v3.2.0**: absent SEQUENCE is treated as 0 (RFC 5546 2.1.4), and `save(increase_seqno=False)` opts out - [ ] Better conflict detection and resolution - [ ] Delegation support for scheduling -- [ ] Add `organizer.change_status()` and similar convenience methods - -**Related issues:** #524, #399, #596, #544 +- [x] Add `organizer.change_status()` and similar convenience methods — **done**: `change_attendee_status()`, `accept_invite()`, `decline_invite()`, `tentatively_accept_invite()`, `add_organizer()` --- ### 1.3 Calendar Availability (RFC 7953) -**Priority:** Medium -**Estimated effort:** 16-24 hours -**RFC:** [RFC 7953](https://datatracker.ietf.org/doc/html/rfc7953) -**Related issue:** #425 +- **Priority:** Medium +- **Estimated effort:** 16-24 hours +- **RFC:** [RFC 7953](https://datatracker.ietf.org/doc/html/rfc7953) +- **Related issue:** [#425](https://github.com/python-caldav/caldav/issues/425) **Tasks:** - [ ] Implement `VAVAILABILITY` component support @@ -86,9 +86,10 @@ The v3.2 roadmap covers basic scheduling improvements. Additional work for full ### 1.4 Extended iCalendar Properties (RFC 7986) -**Priority:** Medium -**Estimated effort:** 8-12 hours -**RFC:** [RFC 7986](https://datatracker.ietf.org/doc/html/rfc7986) +- **Priority:** Medium +- **Estimated effort:** 8-12 hours +- **RFC:** [RFC 7986](https://datatracker.ietf.org/doc/html/rfc7986) +- **Note:** We may stop short doing only some research, estimated at 2 hours effort **Tasks:** - [ ] Support calendar-level properties: `NAME`, `DESCRIPTION`, `COLOR`, `REFRESH-INTERVAL`, `SOURCE` @@ -102,12 +103,12 @@ The v3.2 roadmap covers basic scheduling improvements. Additional work for full ### 2.1 Negated Searches -**Priority:** Medium -**Estimated effort:** 12-16 hours -**Related issue:** #568 +- **Priority:** Medium +- **Estimated effort:** 12-16 hours +- **Related issue:** [#568](https://github.com/python-caldav/caldav/issues/568) **Tasks:** -- [ ] Add `negate="yes"` attribute support in text-match filters +- [x] Add `negate="yes"` attribute support in text-match filters — the `cdav.TextMatch(..., negate=True)` element exists and is used internally (`search.py` `vNotCompleted`/`vNotCancelled`); what is missing is exposing it through the public search API - [ ] Update `CalDAVSearcher` to support `!=` operator - [ ] Add server compatibility detection - [ ] Implement client-side fallback filtering for non-supporting servers @@ -117,13 +118,13 @@ The v3.2 roadmap covers basic scheduling improvements. Additional work for full ### 2.2 Improved Collation Support -**Priority:** Low -**Estimated effort:** 8-12 hours -**Related issue:** #567 +- **Priority:** Low +- **Estimated effort:** 8-12 hours +- **Related issue:** [#567](https://github.com/python-caldav/caldav/issues/567) **Tasks:** -- [ ] Better support for `i;unicode-casemap` collation -- [ ] Locale-aware case-insensitive matching +- [x] Better support for `i;unicode-casemap` collation — `_collation_to_caldav()` in `search.py` maps the `Collation` enum to `i;octet` / `i;ascii-casemap` / `i;unicode-casemap`, selectable per property +- [ ] Locale-aware case-insensitive matching — `Collation.LOCALE` currently falls back to `i;ascii-casemap`, so this is a stub - [ ] Server capability detection for collation support - [ ] Documentation of collation behavior per server @@ -131,13 +132,13 @@ The v3.2 roadmap covers basic scheduling improvements. Additional work for full ### 2.3 Multiget Optimization -**Priority:** Medium -**Estimated effort:** 8 hours -**Related issue:** #487 +- **Priority:** Medium +- **Estimated effort:** 8 hours +- **Related issue:** [#487](https://github.com/python-caldav/caldav/issues/487) **Tasks:** - [ ] Use `calendar-multiget` REPORT when server doesn't return object data in search -- [ ] Batch retrieval of multiple objects +- [x] Batch retrieval of multiple objects — **done**: `Collection.multiget()`, `AsyncDAVClient.calendar_multiget()`, shared body builder `_build_calendar_multiget_body()`. The remaining gap is the [#487](https://github.com/python-caldav/caldav/issues/487) ask: using it *automatically* when a search response carried no object data - [ ] Configurable batch sizes --- @@ -146,9 +147,10 @@ The v3.2 roadmap covers basic scheduling improvements. Additional work for full ### 3.1 Managed Attachments (RFC 8607) -**Priority:** Low -**Estimated effort:** 24-32 hours -**RFC:** [RFC 8607](https://datatracker.ietf.org/doc/html/rfc8607) +- **Priority:** Low +- **Estimated effort:** 24-32 hours +- **RFC:** [RFC 8607](https://datatracker.ietf.org/doc/html/rfc8607) +- **Related issue:** [#700](https://github.com/python-caldav/caldav/issues/700) **Tasks:** - [ ] Detect server support for `calendar-managed-attachments` @@ -161,9 +163,10 @@ The v3.2 roadmap covers basic scheduling improvements. Additional work for full ### 3.2 Calendar Sharing -**Priority:** Medium -**Estimated effort:** 32-40 hours -**Spec:** [draft-pot-caldav-sharing](https://datatracker.ietf.org/doc/html/draft-pot-caldav-sharing) +- **Priority:** Medium +- **Estimated effort:** 32-40 hours +- **Spec:** [draft-pot-caldav-sharing](https://datatracker.ietf.org/doc/html/draft-pot-caldav-sharing) +- **Related issues:** [#701](https://github.com/python-caldav/caldav/issues/701), [#699](https://github.com/python-caldav/caldav/issues/699) (the ACL alternative to the same problem) Note: This is a draft standard but widely implemented by major servers. @@ -179,22 +182,24 @@ Note: This is a draft standard but widely implemented by major servers. ### 3.3 Extended MKCOL (RFC 5689) -**Priority:** Low -**Estimated effort:** 4-8 hours -**RFC:** [RFC 5689](https://datatracker.ietf.org/doc/html/rfc5689) +- **Priority:** Low +- **Estimated effort:** 4-8 hours +- **RFC:** [RFC 5689](https://datatracker.ietf.org/doc/html/rfc5689) +- **Related issue:** [#702](https://github.com/python-caldav/caldav/issues/702) **Tasks:** -- [ ] Support extended MKCOL as alternative to MKCALENDAR -- [ ] Set calendar properties atomically during creation -- [ ] Detect server support +- [x] Support extended MKCOL as alternative to MKCALENDAR — **done**: `Calendar._create()` builds a `DAV:mkcol` with `resourcetype` = collection + calendar, and uses it when the server declares `create-calendar: {support: quirk, behaviour: mkcol-required}` +- [x] Set calendar properties atomically during creation — display name and `supported-calendar-component-set` go into the creation request; PROPPATCH is only a fallback +- [ ] Detect server support — today the MKCOL path is only taken when a server profile declares `mkcol-required` (only `baikal_old` does); nothing probes for it and nothing tests it +- [ ] Handle a `207 Multi-Status` reply (RFC 5689 section 3, a property that could not be set): `expected_return_value=201` makes it raise instead --- ### 3.4 Quota Support (RFC 4331) -**Priority:** Low -**Estimated effort:** 4-8 hours -**RFC:** [RFC 4331](https://datatracker.ietf.org/doc/html/rfc4331) +- **Priority:** Low +- **Estimated effort:** 4-8 hours +- **RFC:** [RFC 4331](https://datatracker.ietf.org/doc/html/rfc4331) **Tasks:** - [ ] Add `calendar.get_quota()` method @@ -203,17 +208,39 @@ Note: This is a draft standard but widely implemented by major servers. --- +### 3.5 WebDAV Push + +- **Priority:** Medium +- **Estimated effort:** 24-40 hours (rough) +- **Spec:** [draft-bitfire-webdav-push](https://bitfireat.github.io/webdav-push/draft-bitfire-webdav-push-00.html) +- **Related issue:** [#674](https://github.com/python-caldav/caldav/issues/674) + +Not an IETF standard — a proposal from the bitfire team (DAVx⁵), already +implemented as a Nextcloud extension. Replaces polling with server-pushed +change notifications. Requested by the proposal authors themselves. + +**Tasks:** +- [ ] Decide whether to support a non-IETF draft at all, and how prominently +- [ ] Detect push support on the collection +- [ ] Subscribe / refresh / unsubscribe +- [ ] Some way of receiving notifications that makes sense for a client library + (this is the hard part: the library does not own an event loop or a + public endpoint) +- [ ] Server feature detection + +--- + ## Phase 4: Robustness and Edge Cases ### 4.1 Collision Avoidance -**Priority:** High -**Estimated effort:** 16-24 hours -**Related issue:** #152 +- **Priority:** High +- **Estimated effort:** 16-24 hours +- **Related issue:** [#152](https://github.com/python-caldav/caldav/issues/152) **Tasks:** -- [ ] Robust ETag-based collision detection -- [ ] Proper `If-Match` / `If-None-Match` header usage +- [x] Robust ETag-based collision detection — **done in v3.2.0**: the ETag from PUT/GET responses is cached in `self.props` and `ETagMismatchError` is raised on 412 +- [ ] Proper `If-Match` / `If-None-Match` header usage — half done: `If-Match` is sent when an ETag is cached (and `If-Schedule-Tag-Match` takes precedence when a Schedule-Tag is), but `If-None-Match` is not used for create-only semantics - [ ] Handle UID vs path name mismatches - [ ] Race condition mitigation - [ ] Clear error messages for conflicts @@ -222,28 +249,40 @@ Note: This is a draft standard but widely implemented by major servers. ### 4.2 Recurrence Handling Improvements -**Priority:** High -**Estimated effort:** 24-32 hours -**Related issues:** #398, #597, #598 +- **Priority:** High +- **Estimated effort:** 24-32 hours (much of it now spent — see the ticked items) +- **Related issues:** [#398](https://github.com/python-caldav/caldav/issues/398), [#597](https://github.com/python-caldav/caldav/issues/597), [#598](https://github.com/python-caldav/caldav/issues/598) **Tasks:** -- [ ] Helper methods for identifying recurrence states -- [ ] Intelligent deletion of single recurrences -- [ ] Better RECURRENCE-ID handling -- [ ] Documentation and examples for recurrence editing -- [ ] Timezone-aware recurrence expansion -- [ ] Tests for complex recurrence scenarios +- [ ] Helper methods for identifying recurrence states ([#597](https://github.com/python-caldav/caldav/issues/597)) — still nothing; there is + no `is_recurring()` / `is_recurrence_instance()` / "find my master" on the object API +- [ ] Intelligent deletion of single recurrences ([#598](https://github.com/python-caldav/caldav/issues/598)) — still nothing; the library never + writes `EXDATE`, so cancelling one occurrence is left entirely to the caller +- [x] Better RECURRENCE-ID handling — **done**: `save()` grew `only_this_recurrence` (a tristate: + merge into the master, merge-or-PUT-as-is, or PUT as-is) and `all_recurrences`, + `_incorporate_recurrence_into_parent()` does the merge, orphaned recurrences no longer crash, + and `RANGE=THISANDFUTURE` is handled when completing recurring tasks +- [x] Documentation and examples for recurrence editing ([#398](https://github.com/python-caldav/caldav/issues/398)) — largely done in + `docs/source/tutorial.rst`: the "big caveat" section explaining that one recurring event is + several components, and a worked example editing a single recurrence +- [x] Timezone-aware recurrence expansion — **done**, but elsewhere: expansion moved to the + `icalendar_searcher` package (built on `recurring_ical_events`), and `expand_rrule()` is now + deprecated in this library +- [x] Tests for complex recurrence scenarios — **done**: `testEditSingleRecurrence`, + `testAddOrphanedRecurrence`, the recurring-todo completion tests including THISANDFUTURE, and + their async twins. Server quirks are captured as flags + (`save-load.event.recurrences.exception.reschedule`, `search.recurrences.expanded.exception`) --- ### 4.3 PROPFIND Redirect Handling -**Priority:** Low -**Estimated effort:** 4-8 hours -**Related issue:** #552 +- **Priority:** Low +- **Estimated effort:** 4-8 hours +- **Related issue:** [#552](https://github.com/python-caldav/caldav/issues/552) **Tasks:** -- [ ] Follow 3xx redirects on PROPFIND +- [ ] Follow 3xx redirects on PROPFIND — still open, and there is a comment in `davclient.py` marking the spot - [ ] Update internal URLs after redirect - [ ] Prevent redirect loops @@ -251,26 +290,47 @@ Note: This is a draft standard but widely implemented by major servers. ### 4.4 Alarm Support -**Priority:** Medium -**Estimated effort:** 12-16 hours -**Related issue:** #132 +- **Priority:** Medium +- **Estimated effort:** 12-16 hours +- **Related issue:** [#132](https://github.com/python-caldav/caldav/issues/132) **Tasks:** -- [ ] Add `event.add_alarm()`, `event.remove_alarm()` methods +- [ ] Add `event.add_alarm()`, `event.remove_alarm()` methods — nothing on the object API; VALARM only appears in search filters and as `alarm_*` arguments to `vcal.create_ical()` - [ ] Support VALARM with ACTION (DISPLAY, AUDIO, EMAIL) - [ ] Trigger types: relative (before/after) and absolute - [ ] Snooze/dismiss support where servers allow --- +### 4.5 Transport Robustness: Retries and Rate Limiting + +- **Priority:** Medium +- **Estimated effort:** 16-24 hours +- **Related issues:** [#695](https://github.com/python-caldav/caldav/issues/695), [#620](https://github.com/python-caldav/caldav/issues/620), [#697](https://github.com/python-caldav/caldav/issues/697) + +Partly in place already: 429/503 `Retry-After` handling with `RateLimitError` +and the `rate-limit` server peculiarity exist. What is missing: + +**Tasks:** +- [ ] Retry on connection failures, configurable ([#695](https://github.com/python-caldav/caldav/issues/695)) +- [ ] Retry by default when an idle keep-alive connection was closed by the + server — observed against Stalwart in the async suite ([#695](https://github.com/python-caldav/caldav/issues/695)) +- [ ] General opt-in sleep-and-retry on transient errors, configured through the + feature subsystem ([#620](https://github.com/python-caldav/caldav/issues/620)) +- [ ] Smarter rate-limit budgeting than "sleep a fixed slice between every + request": either burst-then-sleep-out-the-window, or a progressively + growing delay ([#697](https://github.com/python-caldav/caldav/issues/697)) + +--- + ## Phase 5: Service Discovery and Security ### 5.1 DNSSEC Validation -**Priority:** Medium -**Estimated effort:** 16-24 hours -**Related issue:** #571 -**RFC:** [RFC 6764 Section 8](https://datatracker.ietf.org/doc/html/rfc6764#section-8) +- **Priority:** Medium +- **Estimated effort:** 16-24 hours +- **Related issue:** [#571](https://github.com/python-caldav/caldav/issues/571) +- **RFC:** [RFC 6764 Section 8](https://datatracker.ietf.org/doc/html/rfc6764#section-8) **Tasks:** - [ ] Add optional DNSSEC validation for SRV/TXT lookups @@ -283,15 +343,30 @@ Note: This is a draft standard but widely implemented by major servers. ### 5.2 Server Auto-Detection Improvements -**Priority:** Medium -**Estimated effort:** 16-24 hours -**Related issue:** #600 +- **Priority:** Medium +- **Estimated effort:** 16-24 hours +- **Related issues:** [#600](https://github.com/python-caldav/caldav/issues/600), [#592](https://github.com/python-caldav/caldav/issues/592) **Tasks:** - [ ] Auto-detect server quirks on first connection - [ ] Cache detected quirks - [ ] Improve feature detection heuristics - [ ] Better handling of unknown servers +- [ ] Composable feature configuration — `include` and/or `extra_features`, so a + deployment can start from a named profile and override individual + features ([#592](https://github.com/python-caldav/caldav/issues/592)) + +--- + +### 5.3 TLS Enforcement + +- **Priority:** Medium +- **Estimated effort:** 2-4 hours +- **Related issue:** [#687](https://github.com/python-caldav/caldav/issues/687) + +**Tasks:** +- [ ] `require_tls` is only enforced on RFC 6764 discovery, not on an explicitly + passed URL — a plain `http://` URL is accepted despite the setting --- @@ -299,9 +374,9 @@ Note: This is a draft standard but widely implemented by major servers. ### 6.1 jCal Support (RFC 7265) -**Priority:** Low -**Estimated effort:** 16-24 hours -**RFC:** [RFC 7265](https://datatracker.ietf.org/doc/html/rfc7265) +- **Priority:** Low +- **Estimated effort:** 16-24 hours +- **RFC:** [RFC 7265](https://datatracker.ietf.org/doc/html/rfc7265) **Tasks:** - [ ] Accept `application/calendar+json` responses @@ -312,9 +387,9 @@ Note: This is a draft standard but widely implemented by major servers. ### 6.2 xCal Support (RFC 6321) -**Priority:** Low -**Estimated effort:** 16-24 hours -**RFC:** [RFC 6321](https://datatracker.ietf.org/doc/html/rfc6321) +- **Priority:** Low +- **Estimated effort:** 16-24 hours +- **RFC:** [RFC 6321](https://datatracker.ietf.org/doc/html/rfc6321) **Tasks:** - [ ] Accept `application/calendar+xml` responses @@ -327,24 +402,27 @@ Note: This is a draft standard but widely implemented by major servers. ### 7.1 Test Coverage Expansion -**Priority:** High -**Estimated effort:** 40+ hours (ongoing) -**Related issues:** #93, #45, #595 +- **Priority:** High +- **Estimated effort:** 40+ hours (ongoing) +- **Related issues:** [#93](https://github.com/python-caldav/caldav/issues/93), [#45](https://github.com/python-caldav/caldav/issues/45), [#595](https://github.com/python-caldav/caldav/issues/595), [#667](https://github.com/python-caldav/caldav/issues/667) **Tasks:** - [ ] Increase unit test coverage to 90%+ -- [ ] Add DAViCal docker container for testing (#595) -- [ ] Add more server docker containers +- [x] Add DAViCal docker container for testing ([#595](https://github.com/python-caldav/caldav/issues/595)) — **done**, `tests/docker-test-servers/davical/` +- [x] Add more server docker containers — **done**: baikal, bedework, ccs, cyrus, davical, davis, nextcloud, ox, sogo, stalwart and zimbra all have docker test setups - [ ] Edge case testing for all RFCs +- [x] Make the async test suite symmetric with the sync one ([#667](https://github.com/python-caldav/caldav/issues/667)) — + substantially done; the issue is still open for the remaining gap in + `change_attendee_status()` ([#678](https://github.com/python-caldav/caldav/issues/678)) - [ ] Performance regression tests --- ### 7.2 Server Documentation -**Priority:** Medium -**Estimated effort:** 24-40 hours -**Related issue:** #120 +- **Priority:** Medium +- **Estimated effort:** 24-40 hours +- **Related issue:** [#120](https://github.com/python-caldav/caldav/issues/120) **Tasks:** - [ ] Document setup and quirks for each major server: @@ -366,17 +444,17 @@ Note: This is a draft standard but widely implemented by major servers. ### 7.3 Example Code and Tutorials -**Priority:** Medium -**Estimated effort:** 16-24 hours -**Related issue:** #513, #541 +- **Priority:** Medium +- **Estimated effort:** 16-24 hours +- **Related issue:** [#513](https://github.com/python-caldav/caldav/issues/513), [#541](https://github.com/python-caldav/caldav/issues/541) **Tasks:** - [ ] Update all examples to use icalendar `.new()` method -- [ ] Add howto guides for common tasks +- [x] Add howto guides for common tasks — `docs/source/howtos.rst` exists ([#513](https://github.com/python-caldav/caldav/issues/513) is still open, so presumably not considered complete) - [ ] Scheduling example code - [ ] Recurrence editing examples - [ ] Service discovery examples -- [ ] Migration guide from v2.x to v3.x +- [x] Migration guide from v2.x to v3.x — `docs/source/v3-migration.rst` --- @@ -384,44 +462,105 @@ Note: This is a draft standard but widely implemented by major servers. ### 8.1 Deprecation Cleanup -**Priority:** Medium -**Estimated effort:** 8-16 hours -**Related issues:** #585, #482, #128 +- **Priority:** Medium +- **Estimated effort:** 8-16 hours +- **Related issues:** [#585](https://github.com/python-caldav/caldav/issues/585), [#482](https://github.com/python-caldav/caldav/issues/482), [#128](https://github.com/python-caldav/caldav/issues/128), [#619](https://github.com/python-caldav/caldav/issues/619), [#515](https://github.com/python-caldav/caldav/issues/515) **Tasks:** -- [ ] Remove old incompatibility flags (#585) -- [ ] Obsolete `get_duration`, `get_due`, `get_dtend` (#482) -- [ ] Review `DAVObject.name` removal (#128) +- [ ] Remove old incompatibility flags ([#585](https://github.com/python-caldav/caldav/issues/585)) +- [ ] Obsolete `get_duration`, `get_due`, `get_dtend` ([#482](https://github.com/python-caldav/caldav/issues/482)) +- [ ] v4.0: remove everything that raises a `DeprecationWarning`, and add the + warning to methods deprecated without one ([#619](https://github.com/python-caldav/caldav/issues/619)) +- [ ] Find and kill remaining uses of `event.component['uid']` and friends ([#515](https://github.com/python-caldav/caldav/issues/515)) +- [x] Review `DAVObject.name` removal ([#128](https://github.com/python-caldav/caldav/issues/128)) — **done**: [#128](https://github.com/python-caldav/caldav/issues/128) is closed and `name` is a property raising `DeprecationWarning`, pointing at `get_display_name()`. Actual removal is a 4.0 matter --- ### 8.2 Test Infrastructure -**Priority:** Medium -**Estimated effort:** 16-24 hours -**Related issues:** #577, #509, #593, #518 +- **Priority:** Medium +- **Estimated effort:** 16-24 hours +- **Related issues:** [#577](https://github.com/python-caldav/caldav/issues/577), [#593](https://github.com/python-caldav/caldav/issues/593) ([#509](https://github.com/python-caldav/caldav/issues/509) and [#518](https://github.com/python-caldav/caldav/issues/518) are closed) **Tasks:** -- [ ] Clean up `tests/conf.py` (#577) -- [ ] Refactor test configuration (#509) -- [ ] Refactor setup/teardown methods (#593) -- [ ] Mute expected error logging, break on unexpected (#518) +- [ ] Clean up `tests/conf.py` ([#577](https://github.com/python-caldav/caldav/issues/577)) +- [x] Refactor test configuration ([#509](https://github.com/python-caldav/caldav/issues/509)) — issue closed +- [ ] Refactor setup/teardown methods ([#593](https://github.com/python-caldav/caldav/issues/593)) +- [x] Mute expected error logging, break on unexpected ([#518](https://github.com/python-caldav/caldav/issues/518)) — issue closed --- ### 8.3 Search Module Refactoring -**Priority:** Low -**Estimated effort:** 16-24 hours -**Related issue:** #580 +- **Priority:** Low +- **Estimated effort:** 16-24 hours +- **Related issue:** [#580](https://github.com/python-caldav/caldav/issues/580) + +**Status: largely done** — [#580](https://github.com/python-caldav/caldav/issues/580) is closed, and the matching logic now lives in the +external `icalendar_searcher` library, which `search.py` imports. **Tasks:** -- [ ] Refactor `search.py` for better maintainability -- [ ] Separate concerns more cleanly +- [x] Refactor `search.py` for better maintainability +- [x] Separate concerns more cleanly - [ ] Improve documentation --- +### 8.4 Internal Refactoring Backlog + +- **Priority:** Medium +- **Estimated effort:** 24-40 hours +- **Related issues:** [#659](https://github.com/python-caldav/caldav/issues/659), [#664](https://github.com/python-caldav/caldav/issues/664), [#665](https://github.com/python-caldav/caldav/issues/665), [#698](https://github.com/python-caldav/caldav/issues/698), [#634](https://github.com/python-caldav/caldav/issues/634), [#94](https://github.com/python-caldav/caldav/issues/94) + +Housekeeping that does not change what the library can do, but that the code +needs. Collected here so the roadmap does not pretend the backlog is only +features. + +**Tasks:** +- [ ] `FeatureSet` cleanup — simplify the over-complex type-system remnants in + `compatibility_hints.py` ([#659](https://github.com/python-caldav/caldav/issues/659)) +- [ ] Decide the fate of the sans-I/O "protocol layer": XML parsing lives both in + `caldav/protocol/xml*.py` and in the `Response` class, and the two overlap + ([#664](https://github.com/python-caldav/caldav/issues/664)) +- [ ] Reduce the remaining sync/async code duplication ([#665](https://github.com/python-caldav/caldav/issues/665)) +- [ ] Parse all server iCalendar through `vcal.parse_ical()` consistently ([#698](https://github.com/python-caldav/caldav/issues/698)) +- [ ] Shrink the ruff ignore list ([#634](https://github.com/python-caldav/caldav/issues/634)) +- [ ] `object.id` should always work ([#94](https://github.com/python-caldav/caldav/issues/94)) +- [ ] v4.0: a major API review — no issue for this, and it is the kind of thing + that needs one before it means anything + +--- + +### 8.5 Packaging and HTTP Dependencies + +- **Priority:** Medium +- **Estimated effort:** 8-16 hours +- **Related issues:** [#690](https://github.com/python-caldav/caldav/issues/690), [#611](https://github.com/python-caldav/caldav/issues/611), [#696](https://github.com/python-caldav/caldav/issues/696) + +**Tasks:** +- [ ] Make the HTTP transport an extra, so `caldav` can be installed without + `niquests` ([#690](https://github.com/python-caldav/caldav/issues/690)) +- [ ] Settle the v4.0 HTTP-library question ([#611](https://github.com/python-caldav/caldav/issues/611)) +- [ ] Sync-mode support for the httpx family — async already has it ([#696](https://github.com/python-caldav/caldav/issues/696)) + +--- + +## Not Covered Here + +These open issues are bug reports, support questions or automated noise rather +than roadmap items, and are deliberately left out: +[#71](https://github.com/python-caldav/caldav/issues/71) (`add_event` can update as well), +[#545](https://github.com/python-caldav/caldav/issues/545) (searches return full-day events of adjacent days), +[#612](https://github.com/python-caldav/caldav/issues/612) (support question), +[#624](https://github.com/python-caldav/caldav/issues/624) (GMX calendar creation), +[#678](https://github.com/python-caldav/caldav/issues/678) (`change_attendee_status()` async safety — see 7.1), +[#680](https://github.com/python-caldav/caldav/issues/680), [#681](https://github.com/python-caldav/caldav/issues/681), [#684](https://github.com/python-caldav/caldav/issues/684) (server-specific breakage reports), +[#685](https://github.com/python-caldav/caldav/issues/685) (automated link-checker report). + +Bugs get fixed when they get fixed; they do not need a phase. + +--- + ## Summary: Effort Estimates by Priority | Priority | Phase | Estimated Hours | @@ -437,6 +576,10 @@ Note: This is a draft standard but widely implemented by major servers. | Medium | Alarm Support (4.4) | 12-16 | | Medium | DNSSEC (5.1) | 16-24 | | Medium | Server Auto-Detection (5.2) | 16-24 | +| Medium | WebDAV Push (3.5) | 24-40 | +| Medium | Transport Robustness (4.5) | 16-24 | +| Medium | Internal Refactoring Backlog (8.4) | 24-40 | +| Medium | Packaging / HTTP Dependencies (8.5) | 8-16 | | Medium | Server Documentation (7.2) | 24-40 | | Medium | Examples/Tutorials (7.3) | 16-24 | | Medium | Deprecation Cleanup (8.1) | 8-16 | @@ -449,39 +592,12 @@ Note: This is a draft standard but widely implemented by major servers. | Low | PROPFIND Redirects (4.3) | 4-8 | | Low | jCal/xCal (6.1-6.2) | 32-48 | | Low | Search Refactoring (8.3) | 16-24 | +| Low | TLS Enforcement (5.3) | 2-4 | -**Total estimated effort:** 380-560 hours (depending on scope and depth) - ---- - -## Version Planning Suggestion - -Based on the roadmap, suggested version milestones after v3.2: - -### v3.3 - Robustness Release -- Collision avoidance (#152) -- Recurrence handling improvements (#398, #597, #598) -- PROPFIND redirect handling (#552) - -### v3.4 - Search & Sync Release -- Negated searches (#568) -- Multiget optimization (#487) -- Improved collation (#567) - -### v3.5 - Extended Features Release -- Alarm support (#132) -- RFC 7986 iCalendar properties -- RFC 7953 Availability (#425) - -### v4.0 - ACL & Sharing Release -- Full ACL support (RFC 3744) -- Calendar sharing (draft-pot-caldav-sharing) -- Managed attachments (RFC 8607) -- Major API review +**Total estimated effort:** 455-685 hours (depending on scope and depth). -### v4.1 - Security & Discovery Release -- DNSSEC validation (#571) -- Server auto-detection improvements (#600) +Note that this total is *not* adjusted for the items ticked off as already done +in the 2026-08-20 QA pass, so the real remaining figure is lower. --- diff --git a/docs/design/FULL_CODE_REVIEW_2026-06.md b/docs/design/FULL_CODE_REVIEW_2026-06.md index fa412342..61fac23f 100644 --- a/docs/design/FULL_CODE_REVIEW_2026-06.md +++ b/docs/design/FULL_CODE_REVIEW_2026-06.md @@ -51,7 +51,7 @@ real bugs, clustering around four themes: ## 1. Crash bugs (realistic trigger → unhandled exception) -### 1.1 `calendarobjectresource.py:1167` + `:1187` — 302 handling iterates headers as tuples `[repro]` +### 1.1 `calendarobjectresource.py:1167` + `:1187` — 302 handling iterates headers as tuples `[repro]` ✅ FIXED (commit 22b9cc66+1) `[x[1] for x in r.headers if x[0] == "location"][0]` — iterating a dict-like `Headers` object (niquests `CaseInsensitiveDict` sync, `httpx.Headers` async) yields key *strings*, so `x[0]` is the first character of each header name. @@ -60,7 +60,7 @@ instead of following the redirect. The same broken pattern appears twice because the whole block is pasted twice (see §6.2; the second copy is partly dead code). Fix: `r.headers.get("location")`. -### 1.2 `davclient.py:302` — URL with username but no password → TypeError `[repro]` +### 1.2 `davclient.py:302` — URL with username but no password → TypeError `[repro]` ✅ FIXED `DAVClient(url='https://user@example.com/dav/', password='secret')`: `self.url.username` is set, so `unquote(self.url.password)` runs with `password=None` → TypeError inside `urllib.parse.unquote`. The async client @@ -68,7 +68,7 @@ dead code). Fix: `r.headers.get("location")`. also gives **explicit kwargs precedence over URL credentials, while sync does the opposite**. Pick one precedence (kwargs should win) and share the code. -### 1.3 `davclient.py:836` / `async_davclient.py:376` — rate-limit retry: `None + float` `[code]` +### 1.3 `davclient.py:836` / `async_davclient.py:376` — rate-limit retry: `None + float` `[code]` ✅ FIXED `sleep_seconds += rate_limit_time_slept / 2` executes *before* the `sleep_seconds is None` check. With `rate_limit_handle=True` and `rate_limit_default_sleep=None`: first 429 has `Retry-After: 5` → retried; @@ -76,7 +76,7 @@ second 429 has no usable Retry-After (`compute_sleep_seconds` returns None, e.g. `Retry-After: 0`) → `None += 2.5` → TypeError instead of the documented `RateLimitError`. Same bug copy-pasted in both clients. -### 1.4 `async_davclient.py:1272` — `aio.get_calendars(calendar_name=...)` can never work `[code]` +### 1.4 `async_davclient.py:1272` — `aio.get_calendars(calendar_name=...)` can never work `[code]` ✅ FIXED The async module-level helper awaits the *synchronous* `Principal.calendar()`, which has no async dispatch (`collection.py:448–475`): `calendar_home_set` → `get_property` returns a coroutine for async clients, and @@ -85,7 +85,7 @@ coroutine → TypeError (swallowed into an empty result when `raise_errors=False`). Name-based calendar lookup via `caldav.aio` is broken end-to-end. -### 1.5 `collection.py:601` — async `freebusy_request` with Principal attendees → AttributeError `[code]` +### 1.5 `collection.py:601` — async `freebusy_request` with Principal attendees → AttributeError `[code]` ✅ FIXED `add_attendee(attendee)` is called *before* the `is_async_client` branch at line 604. For a `Principal` attendee on an async client, `get_vcal_address()` returns a coroutine, and `add_attendee` then does @@ -93,13 +93,13 @@ line 604. For a `Principal` attendee on an async client, `_async_save_with_invites` (`collection.py:983–984`) already does the awaited conversion correctly — the same dance is missing here. -### 1.6 `calendarobjectresource.py:727` — `add_attendee("MAILTO:user@example.com")` → UnboundLocalError `[code]` +### 1.6 `calendarobjectresource.py:727` — `add_attendee("MAILTO:user@example.com")` → UnboundLocalError `[code]` ✅ FIXED The string-branch chain is case-sensitive: uppercase `MAILTO:` (common in real-world iCalendar; RFC 3986 schemes are case-insensitive) fails `startswith("mailto:")` and fails the `":" not in attendee` branch, so `attendee_obj` is never assigned and line 742 raises UnboundLocalError. -### 1.7 `calendarobjectresource.py:1272` — `change_attendee_status` raises bare KeyError; `:1284` literal `%s` `[repro]` +### 1.7 `calendarobjectresource.py:1272` — `change_attendee_status` raises bare KeyError; `:1284` literal `%s` `[repro]` ✅ FIXED When the component has no ATTENDEE property at all, `ical_obj['attendee']` raises `KeyError('ATTENDEE')` — not `error.NotFoundError`, which is the only thing the principal-address loops catch — so the "Principal is not invited" @@ -107,32 +107,32 @@ fallback is unreachable. Additionally the genuine not-found raise is `error.NotFoundError("Participant %s not found in attendee list")` with no `% attendee`: the user literally sees `%s`. -### 1.8 `lib/auth.py:31` — IndexError on malformed WWW-Authenticate `[repro]` +### 1.8 `lib/auth.py:31` — IndexError on malformed WWW-Authenticate `[repro]` ✅ FIXED `extract_auth_types('Basic realm="x",')` (trailing comma — seen in the wild) → the empty segment makes `h.split()[0]` raise IndexError, aborting the auth negotiation with an unrelated traceback. Guard with `for h in header.split(",") if h.strip()`. -### 1.9 `config.py:37` — missing section raises KeyError instead of returning empty `[repro]` +### 1.9 `config.py:37` — missing section raises KeyError instead of returning empty `[repro]` ✅ FIXED `expand_config_section` does `config[section]` for non-glob names. A config file with only named sections (no `default`) makes plain `caldav.get_calendars()` crash with `KeyError: 'default'` instead of falling through to "no configuration found". -### 1.10 `compatibility_hints.py:611` — `copyFeatureSet` crashes merging plain-string features `[repro]` +### 1.10 `compatibility_hints.py:611` — `copyFeatureSet` crashes merging plain-string features `[repro]` ✅ FIXED `FeatureSet({'scheduling': 'unsupported'}).copyFeatureSet({'scheduling': 'fragile'})` → bare AssertionError: the `'support' not in server_node` guard makes string-valued updates of an existing feature fall through to the final `else: raise AssertionError`. Plain strings are the dominant style in the hint dicts, so any two-layer merge expressing the same feature crashes. -### 1.11 `compatibility_hints.py:605` — unknown feature names: warn now, crash later `[repro]` +### 1.11 `compatibility_hints.py:605` — unknown feature names: warn now, crash later `[repro]` ✅ FIXED A typoed feature name in a user's config produces only a UserWarning at set time, but the bad key is still stored — a later `collapse()` / `is_supported()` hits a message-less AssertionError in `find_feature`, far from the config that caused it. Reject (or drop) the key at intake instead. -### 1.12 `lib/vcal.py:93` — bare `assert` on server-supplied data `[repro]` +### 1.12 `lib/vcal.py:93` — bare `assert` on server-supplied data `[repro]` ✅ FIXED Truncated/garbage iCalendar without DTSTAMP and without an `END:` line makes `fix()` raise a bare AssertionError. Under `python -O` the assert (and thus the DTSTAMP fixup logic it guards) is silently skipped. Should be @@ -148,7 +148,7 @@ hierarchy callers are told to catch. Copy-paste gap in both clients. ## 2. Silent wrong results / data corruption -### 2.1 `lib/vcal.py:80` — COMPLETED fixup merges the next line into the property ⚠ data corruption `[repro]` +### 2.1 `lib/vcal.py:80` — COMPLETED fixup merges the next line into the property ⚠ data corruption `[repro]` ✅ FIXED (commit 22b9cc66) `fix()` normalizes CRLF→LF first, then the COMPLETED date-to-datetime regex `(\d+)\s` *consumes the newline without restoring it*: `COMPLETED:20240101\nSUMMARY:hello` becomes @@ -156,24 +156,24 @@ hierarchy callers are told to catch. Copy-paste gap in both clients. destroyed and the object parses with corrupted data. This runs on every inbound object. -### 2.2 `lib/vcal.py:242` — `create_ical(ical_fragment=...)` injects the fragment inside VALARM `[repro]` +### 2.2 `lib/vcal.py:242` — `create_ical(ical_fragment=...)` injects the fragment inside VALARM `[repro]` ✅ FIXED The fragment is re-inserted before the first `^END:V` line — which is `END:VALARM` when any `alarm_*` props were given. `ical_fragment='RRULE:...'` plus an alarm produces an event *without* recurrence and with an invalid RRULE inside the alarm. Should target `END:VEVENT|VTODO|VJOURNAL`. -### 2.3 `lib/vcal.py:88` — trailing-whitespace fixup is dead code `[repro]` +### 2.3 `lib/vcal.py:88` — trailing-whitespace fixup is dead code `[repro]` ✅ FIXED `re.sub(" *$", "", fixed)` without `re.MULTILINE` only touches the document end, never the per-line trailing spaces (iCloud X-APPLE-STRUCTURED-EVENT) that docstring fix #4 targets. The vobject traceback it was written to prevent still occurs. -### 2.4 `lib/vcal.py:85` — backslash-unescape regex is a no-op `[repro]` +### 2.4 `lib/vcal.py:85` — backslash-unescape regex is a no-op `[repro]` ✅ FIXED `re.sub(r"\\+('\")", r"\1", fixed)` matches only the literal two-character sequence `'"`; the group should be a character class `['\"]`. Harmless for compliant data, but the fix does nothing. -### 2.5 `lib/url.py:143` + `:159` — `canonical()` keeps credentials and mutates self `[repro]` +### 2.5 `lib/url.py:143` + `:159` — `canonical()` keeps credentials and mutates self `[repro]` ✅ FIXED Two related bugs: (a) `canonical()` builds its result from `self.url_parsed` instead of the `unauth()`'ed URL, so `URL('https://user:pass@example.com/cal/').canonical()` **retains the @@ -184,7 +184,7 @@ then overwrites `url_raw`/`url_parsed` **in place** — a mere `==` comparison silently rewrites the URL (port added, path re-quoted; a literal `+` becomes `%2B`), so subsequent requests can go to a different resource. -### 2.6 `search.py:648` — `combined-is-logical-and` workaround silently drops property filters `[code]` +### 2.6 `search.py:648` — `combined-is-logical-and` workaround silently drops property filters `[code]` ✅ FIXED The workaround strips property filters from the server query but passes the *ambient* `post_filter` (still `None` on otherwise-capable servers — e.g. Nextcloud, whose only relevant flag is `search.combined-is-logical-and: @@ -194,58 +194,58 @@ range. The sibling workarounds at 597–604 and 625–632 correctly force `post_filter=True`; this branch also uniquely lacks the `post_filter is not False` guard. -### 2.7 `search.py:193` — `undef` operator misses the category→CATEGORIES alias `[code]` +### 2.7 `search.py:193` — `undef` operator misses the category→CATEGORIES alias `[code]` ✅ FIXED The `undef` branch emits `PropFilter(property.upper())` without the alias mapping the non-undef branch applies, so `add_property_filter('category', '', operator='undef')` queries the nonexistent property `CATEGORY` — `is-not-defined` on it matches *every* object, returning events that do have categories. -### 2.8 `search.py:362`/`:506` — documented `'=='` exact-match is never enforced `[code]` +### 2.8 `search.py:362`/`:506` — documented `'=='` exact-match is never enforced `[code]` ✅ FIXED The docstring promises "`==` — exact match required, enforced client-side", but no code path inspects the `==` operator (only `'contains'` is checked at line 617) and the post-filter default block ignores it. On a fully-capable server, RFC 4791 substring `text-match` semantics leak through: `'=='` `'rain'` matches "Training". -### 2.9 `calendarobjectresource.py:1570` — `_set_data` leaves a stale `DataState` cache `[repro]` +### 2.9 `calendarobjectresource.py:1570` — `_set_data` leaves a stale `DataState` cache `[repro]` ✅ FIXED The raw-string branch clears the legacy instance attributes but never resets `self._state`. Sequence: fetch event → touch `event.id` / `is_loaded()` (caches state v1) → `event.load()` assigns `self.data = r.raw` → afterwards `get_data()` / `get_icalendar_instance()` / `id` still serve the **pre-reload content** while `.data` returns the new content. -### 2.10 `datastate.py:152` (+ `:67`, `:78`) — `BEGIN:FREEBUSY` never matches `VFREEBUSY` `[repro]` +### 2.10 `datastate.py:152` (+ `:67`, `:78`) — `BEGIN:FREEBUSY` never matches `VFREEBUSY` `[repro]` ✅ FIXED The component-type sniffing tests for `BEGIN:FREEBUSY`; real data says `BEGIN:VFREEBUSY`. A `FreeBusy` object holding raw data gets `get_component_type() → None`, so `is_loaded()`/`has_component()` are False, `save()` **silently no-ops** at the early return, and `load(only_if_unloaded=True)` reloads spuriously. -### 2.11 `calendarobjectresource.py:1943` — `_get_duration` isinstance check on the wrapper, not `.dt` `[repro]` +### 2.11 `calendarobjectresource.py:1943` — `_get_duration` isinstance check on the wrapper, not `.dt` `[repro]` ✅ FIXED `isinstance(i["DTSTART"], datetime)` tests the icalendar `vDDDTypes` wrapper (never a datetime), so the date-vs-datetime branch always takes the date path: a VTODO with a timed DTSTART and no DUE/DURATION gets duration **1 day instead of 0**. Completing a recurring task then sets the next DUE a full day late, and `Todo._next` shifts the recurrence. -### 2.12 `calendarobjectresource.py:2140` — sync safe-mode completion ignores `completion_timestamp` `[code]` +### 2.12 `calendarobjectresource.py:2140` — sync safe-mode completion ignores `completion_timestamp` `[code]` ✅ FIXED `_complete_recurring_safe` calls `completed.complete()` (defaults to *now*) while the async twin passes the caller's timestamp through. Sync/async divergence with user-visible effect on the recorded COMPLETED time. -### 2.13 `base_client.py:689` — calendar with displayname `""` dropped from results `[code]` +### 2.13 `base_client.py:689` — calendar with displayname `""` dropped from results `[code]` ✅ FIXED `if _try(calendar.get_display_name, ...)` is a truthiness check, so a calendar explicitly requested by URL whose displayname is the empty string is silently omitted. The async counterpart (`async_davclient.py:1262`) correctly uses `is not None`. -### 2.14 `async_davclient.py:957` — async `get_calendars()` lacks the GMX principal-URL fallback `[code]` +### 2.14 `async_davclient.py:957` — async `get_calendars()` lacks the GMX principal-URL fallback `[code]` ✅ FIXED Sync `get_calendars()` (`davclient.py:486–489`) falls back to the principal URL when `calendar-home-set` is missing; async returns `[]` for the same server. Parity gap. -### 2.15 `async_davclient.py:487` — issue-#158 workaround can return the probe response as the real one `[code]` +### 2.15 `async_davclient.py:487` — issue-#158 workaround can return the probe response as the real one `[code]` ✅ FIXED When the original request dies with a connection abort, the workaround sends a probe GET; if that GET is *not* 401+WWW-Authenticate (e.g. 200 with a login page), the code falls through and returns the **probe GET's response as the @@ -253,25 +253,25 @@ original request's response** — the caller sees status 200 for a PUT that never happened, and the real connection error is lost. Also: the sync client has no #158 workaround at all (parity gap in the other direction). -### 2.16 `async_davclient.py:435` — HTML-on-401 hint checks the wrong headers `[code]` +### 2.16 `async_davclient.py:435` — HTML-on-401 hint checks the wrong headers `[code]` ✅ FIXED The diagnostic checks `self.headers` (the client's own request headers) for `text/html` instead of `r.headers`, so the intended "server returned HTML, maybe set auth_type" hint can never fire. -### 2.17 `config.py:50` — `disable: true` ignored for named sections `[repro]` +### 2.17 `config.py:50` — `disable: true` ignored for named sections `[repro]` ✅ FIXED `expand_config_section` checks `config.get("section", ...)` with the string literal `"section"` instead of the variable. `disable` only works under `section='*'`; sections pulled in via a meta-section's `contains` list (or by name) connect to servers the user explicitly disabled. -### 2.18 `config.py:265` — explicit params without url/features silently discarded `[code]` +### 2.18 `config.py:265` — explicit params without url/features silently discarded `[code]` ✅ FIXED `get_connection_params` honors `explicit_params` only when `url` or `features` is present, and never merges them with the env/file source that wins: `get_davclient(password='secret')` with `CALDAV_URL`/`CALDAV_USERNAME` in env returns a config **without the password**, contradicting the docstring's "explicit parameters take highest priority". -### 2.19 `config.py:180` + `testing.py:127`/`:263` — shared module-level hint dicts get mutated `[repro]` +### 2.19 `config.py:180` + `testing.py:127`/`:263` — shared module-level hint dicts get mutated `[repro]` ✅ FIXED `resolve_features` with a string name returns the module-level `compatibility_hints` dict itself (the `base` branch deepcopies; this branch doesn't). `XandikosServer`/`RadicaleServer` then do a *shallow* `.copy()` and @@ -285,7 +285,7 @@ permanently polluted for the whole process, redirecting any later ## 3. Security -### 3.1 `discovery.py:329` — `require_tls=True` not enforced on well-known redirect target `[code]` +### 3.1 `discovery.py:329` — `require_tls=True` not enforced on well-known redirect target `[code]` ✅ FIXED `_well_known_lookup` never receives `require_tls`; a same-domain `Location: http://...` passes the `_is_subdomain_or_same` check and is returned as `ServiceInfo(tls=False)`, which `discover_service` returns unchecked. A @@ -294,7 +294,7 @@ guarantee to plaintext, and credentials follow. (Otherwise the discovery module's security posture is good: require_tls defaults True, same-domain redirect validation, single manual redirect hop.) -### 3.2 `response.py:277` — XML parser for untrusted server data lacks entity hardening `[code]` +### 3.2 `response.py:277` — XML parser for untrusted server data lacks entity hardening `[code]` ✅ FIXED `etree.XMLParser(remove_blank_text=True, huge_tree=self.huge_tree)` relies on libxml2 defaults for entity resolution. Current libxml2 blocks the classic XXE paths, but the library makes no guarantee across the unpinned dependency @@ -303,7 +303,9 @@ of server data in the package — one line fixes it: add `resolve_entities=False` (and consider `no_network=True`, `dtd_validation=False` explicitly). -### 3.3 `lib/error.py:51` — `PYTHON_CALDAV_COMMDUMP` persists bodies/headers in /tmp (low) `[code]` +**Fixed**: added `resolve_entities=False, no_network=True` to the `etree.XMLParser` call in `response.py:277`. `dtd_validation=False` is lxml's default so was not added explicitly. + +### 3.3 `lib/error.py:51` — `PYTHON_CALDAV_COMMDUMP` persists bodies/headers in /tmp (low) `[code]` ✅ FIXED `NamedTemporaryFile(delete=False)` dumps full request/response headers and bodies (calendar PII, custom auth headers) to files that accumulate indefinitely. Files are 0600, and the niquests-applied Authorization header @@ -350,12 +352,12 @@ only includes keys conditionally, so a property deleted client-side (e.g. LOCATION, VALARM) is simply *absent* from the patch and **persists on the server**. Clearing requires explicit `null` entries. -### 4.6 `jmap/objects/calendar.py:113` — search `after`/`before` are not UTCDate `[code]` +### 4.6 `jmap/objects/calendar.py:113` — search `after`/`before` are not UTCDate `[code]` ✅ FIXED `datetime.isoformat()` is passed straight through (naive → no `Z`, aware → `+02:00` offset, plus microseconds); JMAP requires `...Z` UTCDate. Strict servers reject the query; lenient ones interpret the window inconsistently. -### 4.7 `jmap/client.py:462` / `async_client.py:347` — `newState` from `/changes` discarded `[code]` +### 4.7 `jmap/client.py:462` / `async_client.py:347` — `newState` from `/changes` discarded `[code]` ✅ FIXED `get_objects_by_sync_token` unpacks `new_state` into `_`. Callers' only option for a new baseline is a separate `get_sync_token()` call — changes landing in between are silently skipped on the next sync. @@ -378,15 +380,26 @@ producing drift bugs. in sequence**; the second `elif r.status not in (204, 201)` is unreachable. Also factor the Etag/Schedule-Tag header→props snippet repeated in `load`/`_async_load` (the code itself carries a "consider - refactoring - this is repeated many places now" comment). + refactoring - this is repeated many places now" comment). ✅ FIXED — the + dead second copy in `_post_put` was removed and the Etag/Schedule-Tag + capture extracted into a shared `_update_tag_props()` helper now used by + `_post_put`, `load`, and `_async_load`. 3. **`async_davclient.py` re-implements ~200 lines of `DAVClient`** (init tail, get_calendars, rate-limit retry loop — byte-identical except `time.sleep` vs `asyncio.sleep`). The §2.14 GMX gap and §1.3 retry bug are direct drift products. Move into `BaseDAVClient` / `lib/error.py`. + ✅ FIXED — the byte-identical pure logic is now shared via + `BaseDAVClient`: `_init_rate_limit_config()` (the rate-limit init tail), + `_rate_limit_sleep_seconds()` (the retry sleep-decision — where §1.3 + lived), and `_calendar_home_url()` / `_build_calendars_from_propfind()` + (the get_calendars post-processing — where §2.14 lived). Only the + irreducible per-twin parts remain duplicated: the actual `time.sleep` vs + `await asyncio.sleep`, the awaited PROPFIND/principal I/O, and the + library-specific session/header setup in `__init__`. 4. **`search.py:869` — post-processing loads unloaded results one GET at a time**; `Calendar._multiget` can fetch them in a single REPORT. On the issue-#201 workaround path a 200-event search costs ~200 extra - round-trips. + round-trips. ✅ FIXED 5. **JMAP clients open a fresh HTTP connection per request** (async: `async with AsyncSession()` per `_request`; sync: module-level `requests.post`). `__exit__`/`__aexit__` already exist but do nothing — @@ -395,28 +408,66 @@ producing drift bugs. sync twin** (the file says "TERRIBLY much code duplication here"), and the async safe-variant has drifted: it PUTs the completed copy twice. Extract a pure icalendar-mutation helper; keep 5-line sync/async - wrappers. + wrappers. ✅ FIXED — the icalendar mutation now lives once in the pure + (no-I/O) `_prepare_recurring_thisandfuture()` and + `_build_recurring_safe_completed()`; each sync/async twin is reduced to + a thin wrapper that does only the `await`-able save(s). The double-PUT of + the completed copy is gone (the copy is now completed in memory and PUT + once). New offline unit tests in `TestRecurringCompleteHelpers` cover the + mutation and the single-PUT invariant. 7. **`response.py` carries two parallel multistatus-parsing stacks** — legacy `_find_objects_and_props`/`expand_simple_props` (still load-bearing for `_multiget`, report-result building, `search_principals`) vs the newer dataclass parsers. Every parsing quirk (Confluence %2540, purelymail 404) must be maintained twice; the TODO at line 577 already - acknowledges this. + acknowledges this. ✅ FIXED (the duplicated structural parsing) — the + *named* quirks were already shared: Confluence `%2540` and absolute-URL + normalization live in `_normalize_href`, and the purelymail/stalwart 404 + response shapes in `_parse_response`, both of which both stacks call. The + remaining genuinely-duplicated piece — the propstat iteration plus the + "a 404 propstat means the property is absent" skip — is now in the single + `_collect_prop_elements()` helper, used by both `_extract_properties` + (dataclass stack) and `_find_objects_and_props` (legacy stack). As a side + effect the legacy path dropped its over-strict per-propstat asserts + (status-present / `cnt == len(propstat)` / non-404-status validation) that + the dataclass path never had, so the two stacks now treat odd-shaped + propstats identically. `TestParserStackEquivalence` guards the agreement. + What is *not* collapsed (and was not, to avoid touching the slow + integration-tested `_multiget`/report/`search_principals` paths): the two + value-conversion APIs — `_element_to_value` (pre-parsed values for the + dataclass results) vs `_expand_simple_prop` (caller-directed text + expansion). Those are two output formats, not a duplicated quirk; fully + migrating the `expand_simple_props` consumers onto the dataclass results + remains a larger follow-on. 8. **`search.py` sync/async driver loops duplicated** (~80 lines including the hase-1/Phase-2 exception-rethrow protocol and `_search_with_comptypes`). A small executor object with sync/async - implementations would leave one driver. + implementations would leave one driver. ✅ FIXED — the drift-prone + Phase-2 generator protocol (`gen.throw`/`gen.send`/`StopIteration`) now + lives once in the shared module-level `_advance_search_gen()`; each + driver loop is reduced to priming + a `while` that defers Phase-1 to a + `_dispatch_search_action` / `_async_dispatch_search_action` method. Only + the irreducible `await` and the per-action one-liners remain duplicated. --- ## 6. Altitude / design notes 1. **`response.py:464` — server fingerprints hardcoded in the generic - parser.** purelymail's `{https://purelymail.com}does-not-exist` tag, - Stalwart's "No resources found" string, and SOGo status notes live in the - core multistatus path instead of going through the compatibility-hints - feature matrix. Adding the next server's 404 shape means editing generic + parser.** ✅ FIXED. purelymail's `{https://purelymail.com}does-not-exist` + tag, Stalwart's "No resources found" string, and SOGo status notes lived in + the core multistatus path instead of going through the compatibility-hints + feature matrix. Adding the next server's 404 shape meant editing generic parsing — the exact inversion the hints mechanism exists to avoid. + + **Resolution:** `` and `` are optional children + of `` per RFC 4918 with server-defined content, so no per-server + config (nor content fingerprint) is warranted at all — `_parse_response` + now accepts either element generically. A genuinely novel tag still hits + `error.weirdness()`. The `check_404` debug guard was made `None`-safe (a + server may legally send these without a response-level status). Tests: + `test_parse_sync_collection_generic_responsedescription` / + `test_parse_sync_collection_generic_error` in `tests/test_protocol.py`. 2. **`vcal.fix()` is a regex-rewriting layer applied to every inbound object.** §2.1–§2.4 show the current fixups are individually broken in four different ways; the module's own TODOs flag the approach. Worth diff --git a/docs/design/README.md b/docs/design/README.md index cc65c090..472392c0 100644 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -60,6 +60,12 @@ How to use the protocol layer for testing and low-level access. ### [GET_DAVCLIENT_ANALYSIS.md](GET_DAVCLIENT_ANALYSIS.md) Analysis of `get_davclient()` factory function vs direct `DAVClient()` instantiation. +### [RETRY_AND_RESILIENCE_DESIGN.md](RETRY_AND_RESILIENCE_DESIGN.md) +**Proposal** - how the library should retry failed requests. What the five +supported HTTP libraries already offer (researched, with versions), which of the +three failure classes belongs at which layer, the configuration surface, and why +PR #648 was closed. Consolidates issues #620, #647, #695 and PR #648. + ### [TODO.md](TODO.md) Known issues and remaining work items. diff --git a/docs/design/RELEASE-HOWTO.md b/docs/design/RELEASE-HOWTO.md index f8764cab..e5cb4fa2 100644 --- a/docs/design/RELEASE-HOWTO.md +++ b/docs/design/RELEASE-HOWTO.md @@ -27,16 +27,24 @@ I have no clue on the proper procedures for doing releases, and I keep on doing * Run tests (particularly the style check): `pytest` and `tox -e style`. TODO: is `tox -e style` still relevant? * Push the code to github: `cd ~/caldav ; git push ; git push --tags` * Some people relies on the github release system for finding releases - go to https://github.com/python-caldav/caldav/releases/new, choose the new tag, copy the version number and the release notes in. Remember to check the box to make it the latest release. -* The most important part - push to pypi: +* The most important part - push to pypi. Note that the virtualenv is created + *outside* the release clone: `python -m build` packages the directory it is + pointed at, and a venv sitting inside it goes straight into the tarball. + That is how `caldav-3.2.1.tar.gz` came to contain 1755 files under `venv/`. ``` + python3 -m venv ~/caldav-release-venv + . ~/caldav-release-venv/bin/activate + pip install -U pip build twine tox cd ~/caldav-release - python3 -m venv venv - . venv/bin/activate - pip install -U pip build twine + tox -e package # builds sdist+wheel and fails if anything untracked is in them python -m build python -m twine upload dist/* ``` -* Remove the release dir: `rm -r caldav-release` + `tox -e package` is the safety net for this whole class of mistake: it + compares the sdist file list against `git ls-files` and refuses anything git + does not track. It runs in CI too, but run it here as well - CI checks the + *repository*, this checks the *tree you are about to upload*. +* Remove the release dir and its venv: `rm -r ~/caldav-release ~/caldav-release-venv` ## List of mistakes to be avoided @@ -48,5 +56,5 @@ This is most likely not complete, but should explain some of the "silly" steps a * Forgetting to add new files to the git repo * Having checked out a branch or tag or something, and tagging that as the new release rather than the latest HEAD. * Forgetting to push to pypi, or pushing something else than the tagged revision to pypi -* Pushing out junk files in the pypi-release (i.e. .pyc-files, log files, temp files, `tests/conf_private.py`, `tests/caldav_test_servers.yaml`, etc +* Pushing out junk files in the pypi-release (i.e. .pyc-files, log files, temp files, `tests/conf_private.py`, `tests/caldav_test_servers.yaml`, an entire `venv/`, etc). `tox -e package` now catches this - see the build step above * Not adding the release to the "github releases" (I don't care much about this feature, but apparently some people check there to find the latest release version) diff --git a/docs/design/RETRY_AND_RESILIENCE_DESIGN.md b/docs/design/RETRY_AND_RESILIENCE_DESIGN.md new file mode 100644 index 00000000..30605152 --- /dev/null +++ b/docs/design/RETRY_AND_RESILIENCE_DESIGN.md @@ -0,0 +1,624 @@ +This document was AI-generated, but has been reviewed and edited by a human. + +# Retry and resilience design + +- **Status:** proposal, not implemented. +- **Supersedes:** issue #620 (`sleep-retry-logic`), PR #648 (`issue647` branch). +- **Tracking issue:** https://github.com/python-caldav/caldav/issues/695 +- **Milestone:** v3.x (after 3.3). + +This document is the single place where retry behaviour for the CalDAV library is +specified. It exists because three tickets turned out to describe the same +feature from three different angles: + +| ticket | angle | +| --- | --- | +| https://github.com/python-caldav/caldav/issues/620 | opt-in, configurable *sleep-and-retry on transient server errors*, driven from the `features` config | +| https://github.com/python-caldav/caldav/issues/695 | retry on *connection failures*, particularly a keep-alive socket the server closed while idle | +| https://github.com/python-caldav/caldav/pull/648 | a concrete implementation: catch `ConnectionError`/`Timeout` from niquests and retry once for idempotent methods | +| https://github.com/python-caldav/caldav/issues/647 | the user-visible bug that triggered #648 - `c.events()` intermittently dies with `ConnectionError: None: Read timed out` | + +Everything below about the HTTP libraries was verified by reading the installed +source, not from memory. Versions checked: `niquests` 3.15.2, +`urllib3-future` 2.15.901, `requests` 2.34.2, `urllib3` 2.7.0, `httpx` 0.28.1, +`httpx2` 2.3.0, `httpcore` 1.0.9 (August 2026). + +## 1. Three failure classes, not one + +Retry logic gets muddled when "transient error" is treated as one thing. It is +three, and they need different handling: + +**(A) The request never reached the server.** DNS failure, TCP connect +refused/timed out, TLS handshake failure, or - the #695 case - a pooled +keep-alive connection that the server closed while idle and that the client +picked for the next request. A retry here cannot duplicate a write, because +nothing was written. Safe for *every* method, including POST. + +**(B) The request reached the server but the response did not come +back.** Read timeout, connection reset mid-response, HTTP/2 stream +error. The server may or may not have processed the request. Safe +for methods that are idempotent by definition. Not safe for POST - +but POST is not used in the caldav standard. DELETE may result in an +error as the resource was already deleted, but that is already caught: +`DAVObject._post_delete()` accepts 404 as success today. Of the ten +methods the library actually sends, that leaves only `MKCOL` and +`MKCALENDAR` with a real wrinkle (a retry after a lost success gives +405 or 409) - see §4.1 for the full method-by-method answer. + +**(C) The server answered, with a status saying "not now".** 429, 503, and +arguably 500/502/504. This is #620's original subject. Retrying is safe (the +server declined to act), but it needs a *sleep*, ideally honouring +`Retry-After`. + +The library already handles a subset of (C): `raise_if_rate_limited()` raises +`RateLimitError` on 429/503, and `DAVClient.request()` / +`AsyncDAVClient._async_request()` sleep and retry via +`error.compute_sleep_seconds()` when `rate_limit_handle` is set, with +`rate_limit_default_sleep` / `rate_limit_max_sleep` as bounds and a `rate-limit` +client-feature in `compatibility_hints.py` carrying `interval`, `count`, +`max_sleep`, `default_sleep`. **Nothing new is built on top of that machinery, +and nothing new duplicates it.** Whether it should eventually be deleted in +favour of `Retry(status_forcelist=[429, 503])` is taken up in §4.5 - the answer +is "yes, in 4.0, once urllib3-future can cap `Retry-After`". + +Nothing at all handles (A) or (B) today: there is no `except ConnectionError` +anywhere under `caldav/`, and no adapter is mounted, so we run on library +defaults - and the library defaults are "retry nothing" in every one of the five +libraries we support. + +## 2. What the HTTP libraries already give us + +We support five libraries: niquests (sync + async, default), requests (sync +fallback), and httpx2 / httpxyz / httpx (async fallbacks). See +`docs/source/http-libraries.rst`. + +niquests is the supported and recommended http library - the priority is to solve things for niquests. + +### Summary + +| | niquests / urllib3-future | requests / urllib3 | httpx family / httpcore | +| --- | --- | --- | --- | +| retries on by default | no (`DEFAULT_RETRIES = 0`) | no (`HTTPAdapter(max_retries=0)`) | no (`retries=0`) | +| how to configure | `Session(retries=...)` / `AsyncSession(retries=...)` - no adapter mounting needed | `session.mount(scheme, HTTPAdapter(max_retries=Retry(...)))` | `Client(transport=HTTPTransport(retries=int))` | +| class (A) connect errors | yes, `Retry(connect=N)` | yes, `Retry(connect=N)` | yes, but *only* connect - `retries=int` | +| class (B) read errors | yes, `Retry(read=N)`, method-filtered | yes, `Retry(read=N)`, method-filtered | **no** | +| class (C) status retries | yes, `status_forcelist` + `Retry(status=N)` | yes, `status_forcelist` + `Retry(status=N)` | **no** | +| honours `Retry-After` | yes, `respect_retry_after_header=True` | yes (+ `retry_after_max=21600`) | **no** | +| backoff control | `backoff_factor`, `backoff_max=120`, `backoff_jitter` | same | hardcoded `0.5 * 2**n`, not configurable | +| method allow-list | `allowed_methods` | `allowed_methods` | **no** | +| proactive rate limiting | yes - `LeakyBucketLimiter`, `TokenBucketLimiter` (+ async variants) | no | no | +| keep-alive liveness | HTTP/2+ PING: `keepalive_delay=600`, `keepalive_idle_window=60` | no | no | + +### niquests / urllib3-future - batteries fully included + +`niquests.Session` and `niquests.AsyncSession` both take `retries: RetryType` +directly in the constructor, defaulting to `DEFAULT_RETRIES = 0`. It accepts an +int or a `Retry` object; `niquests.RetryConfiguration` is a re-export of +`urllib3_future.util.retry.Retry`, so there is a public, documented name for it +that does not require importing `urllib3_future`. The session pushes the value +into every adapter it builds, so **no `mount()` call is needed** - which matters, +because caldav never mounts anything today. + +`Retry.__init__` (identical parameter list in `urllib3` 2.7.0 and +`urllib3_future` 2.15.901, except that urllib3 additionally has +`retry_after_max=21600`): + +```python +Retry(total=10, connect=None, read=None, redirect=None, status=None, other=None, + allowed_methods=frozenset({'OPTIONS','TRACE','DELETE','GET','HEAD','PUT'}), + status_forcelist=None, backoff_factor=0, backoff_max=120, + raise_on_redirect=True, raise_on_status=True, history=None, + respect_retry_after_header=True, + remove_headers_on_redirect=frozenset({'Proxy-Authorization','Cookie','Authorization'}), + backoff_jitter=0.0) +``` + +Two details from the classification code that decide our design: + +```python +def _is_connection_error(self, err): # -> counts against `connect` + return isinstance(err, ConnectTimeoutError) # NewConnectionError subclasses this + +def _is_read_error(self, err): # -> counts against `read` + return isinstance(err, (ReadTimeoutError, ProtocolError)) +``` + +and `increment()` retries a *read* error only when +`self._is_method_retryable(method)`, i.e. when the method is in +`allowed_methods`. + +**Consequence, and this is the single most important finding in this document:** +"Remote end closed connection without response" surfaces as a `ProtocolError`, +so urllib3 classifies it as a **read** error, not a connect error - even though +in this particular case nothing was ever delivered. It is therefore +method-filtered, and the default `allowed_methods` contains neither `PROPFIND` +nor `REPORT`. Handing the library a plain `Retry(3)` would leave the exact +failure #695 is about **unretried**, because CalDAV's two most-used methods are +not on urllib3's allow-list. `allowed_methods` must be set explicitly. + +Also worth knowing: `HTTPConnectionPool._get_conn()` already discards a dead +pooled connection - `if conn and is_connection_dropped(conn): conn.close()`. +That is a `select()` on the socket, so it closes the common window but not the +race: the server can close between the check and the write. This is why #695 +still happens even though the pool "already handles" stale connections, and why +a retry, not a liveness check, is the fix. + +Related knobs that reduce how often we get there in the first place: +`keepalive_delay` (default 600s) and `keepalive_idle_window` (default 60s) make +niquests send HTTP/2+ PING frames on idle connections. They do nothing for +HTTP/1.1, and CalDAV servers behind nginx with digest auth are on HTTP/1.1 for +us today (multiplexing is off by default - see the `http.multiplexing` feature). + +niquests also ships *proactive* rate limiters (`LeakyBucketLimiter`, +`TokenBucketLimiter`, and async variants). Those map onto the `interval`/`count` +half of our existing `rate-limit` client-feature, which we currently implement in +test code. Out of scope here, but noted: there is a wheel we could stop +reinventing. + +### requests / urllib3 - same batteries, one extra step + +`HTTPAdapter.__init__(pool_connections=10, pool_maxsize=10, max_retries=0, +pool_block=False)` - retries off. `requests.Session.__init__` takes no +arguments at all, so configuring retries means constructing an adapter and +mounting it on both `http://` and `https://`. Same `Retry` class, same +semantics, so one `Retry`-building helper serves both libraries. + +### httpx family - nearly no batteries + +`httpx.HTTPTransport` / `AsyncHTTPTransport` (identical signature in httpx +0.28.1 and httpx2 2.3.0 - httpx2 brings *nothing* extra here) accept +`retries: int = 0`, forwarded to `httpcore.ConnectionPool(retries=...)`. In +httpcore that value is consumed in exactly one place, `HTTPConnection._connect`: + +```python +retries_left = self._retries +... +except (ConnectError, ConnectTimeout): + if retries_left <= 0: + raise + retries_left -= 1 +``` + +So: connection *establishment* only. No read retries, no status retries, no +`Retry-After`, no method filter, no configurable backoff (httpcore sleeps +`0.5 * 2**n` internally). For the httpx family, classes (B) and (C) can only be +handled by caldav itself, or by a third-party transport such as `httpx-retries` +(`RetryTransport`), which we should *not* take on as a dependency. + +## 3. Layering decision + +``` + ┌──────────────────────────────────────────────────────────────────┐ + │ caldav: DAVClient.request() / AsyncDAVClient._async_request() │ + │ │ + │ Layer B - status retries (class C) │ + │ 429/503 today, opt-in 5xx tomorrow, reuses compute_sleep_..() │ + │ Layer C - the niquests lazy-gather catch (class B, special) │ + │ wrap ConnectionError/Timeout raised at attribute access │ + ├──────────────────────────────────────────────────────────────────┤ + │ Layer A - transport retries (classes A and B) │ + │ niquests: Session(retries=Retry(...)) │ + │ requests: mount(HTTPAdapter(max_retries=Retry(...))) │ + │ httpx*: HTTPTransport(retries=n) [connect only, degraded] │ + └──────────────────────────────────────────────────────────────────┘ +``` + +**Layer A handles what caldav cannot see.** A socket that dies before or during +the write, a DNS hiccup, a TLS renegotiation - by the time an exception reaches +`DAVClient.request()`, the connection is gone, the response body is +unrecoverable, and all caldav can do is send the whole request again from +scratch. urllib3 can do better: it retries inside the pool, reuses the +connection machinery, tracks the budget across redirects, and honours +`Retry-After`. It is also far better tested than anything we would write. +**Do not hand-roll this.** + +**Layer B is where the existing 429/503 handling already lives, and it gets no +new code.** `status_forcelist` in `Retry` does the job for niquests and requests, +and duplicating it in caldav would mean two mechanisms racing over the same +responses. So for 3.x the existing `rate_limit_handle` loop stays as it is, and +429/503 deliberately stay *out* of `status_forcelist` so the two never overlap. +Whether that loop can be **deleted** and the job handed to niquests entirely is a +separate and attractive question - see §4.5, where the answer turns out to be +"yes, but not yet, and not for free". + +**Layer C is the only part that genuinely needs new caldav-side code for class +(B),** and it is the part PR #648 got right. niquests gathers HTTP/2 responses +lazily: the request returns a `Response` whose `status_code`/`headers`/`content` +are resolved on first access, and that resolution can raise +`niquests.exceptions.ConnectionError`. #647's traceback shows it firing from +`log.debug("server responded with %i %s" % (r.status_code, r.reason))` inside +`request()` - i.e. **after** `adapter.send()` returned, therefore **outside** +urllib3's retry scope. No `Retry` configuration can cover this. It needs a +`try`/`except` around the point where caldav first touches response attributes. + +### 3.1 Scope: niquests first - and what the others actually cost + +niquests is the supported and recommended library, so it sets the schedule. The +question worth asking is whether to stop there. Measured against the real code, +the answer is "no, but nearly" - because the expensive part of this feature is +not the per-library plumbing, it is deciding the policy (defaults, +`allowed_methods`, the config surface), and that is shared by all of them. + +**niquests: one keyword argument, twice.** Both session constructors already sit +behind a `try`/`except TypeError` that exists to tell niquests and the fallback +apart: + +```python +## caldav/davclient.py:261-265, as it stands today +try: + multiplexed = self.features.is_supported("http.multiplexing") + self.session = requests.Session(multiplexed=multiplexed) +except TypeError: + self.session = requests.Session() +``` + +The niquests fix is `retries=retry_cfg` added to that call, and the same in +`AsyncDAVClient._create_session()` for `AsyncSession`. Two lines of diff. +Everything else is the shared `_build_retry()` helper and the config plumbing. + +**requests: four more lines, same `Retry` object.** `requests.Session` takes no +constructor arguments, so it needs a mounted adapter - but the `Retry` class has +an identical parameter list in `urllib3` 2.7.0 and `urllib3_future` 2.15.901, and +`niquests.Session.__init__` even converts a vanilla urllib3 `Retry` for you +(`urllib3_ensure_type()` when `"urllib3_future" not in str(type(self.retries))`). +So one builder serves both; only the import differs +(`niquests.RetryConfiguration` when niquests is in use, `urllib3.util.retry.Retry` +otherwise - and the module already has `_USE_NIQUESTS` to branch on): + +```python +except TypeError: + self.session = requests.Session() + if retry_cfg is not None: + adapter = requests.adapters.HTTPAdapter(max_retries=retry_cfg) + self.session.mount("http://", adapter) + self.session.mount("https://", adapter) +``` + +Excluding requests would save four lines and one conditional import. That is a +false economy, and the audience argues against it too: the requests fallback +exists because a distro packager objected to niquests, and +`docs/source/http-libraries.rst` recommends requests for "a very sharp production +environment" - which is exactly the deployment that needs retries most. **Do +requests in the same change.** + +**httpx family: cheap-looking, actually a refactor - defer it.** `retries=` is +an int on the *transport*, not on the client, and `httpx.AsyncClient(transport=...)` +**ignores** the `verify`, `cert`, `http2` and `limits` keywords when a transport +is supplied - they only configure the default transport it would otherwise build. +`_create_session()` passes exactly those keywords today, so adding a transport +means moving TLS configuration into the transport constructor. The failure mode +if that is done carelessly is silently disabled certificate verification. For +that risk we would buy connect-retries only: no read retries, so **not even the +failure this design is named after**. Defer, and document that the httpx +fallbacks get no retries. + +**Layer C cannot be delegated to any library.** Worth being explicit, because it +is the one place where "this belongs in the HTTP library" does not get us out of +writing code: the lazy-gather `ConnectionError` is raised after `adapter.send()` +has returned, so it is outside every retry mechanism niquests has. If we do +Layer A only, https://github.com/python-caldav/caldav/issues/647 - the bug that +started this - stays open. It should *also* be reported upstream to +https://github.com/jawah/niquests, with the traceback and the observation that +the session's retry budget does not extend to lazy response gathering; that is +where the fix belongs long-term. Note that a previous report in this area +(jawah/niquests#352) was closed as "Not Planned" because the analysis handed to +the maintainer was AI-hallucinated nonsense, so this one needs to be precise, +minimal and reproducible, or not filed at all. + +**Total.** The niquests-only version is roughly an hour of work: two keyword +arguments, a ~25-line `_build_retry()`, the `retry` config key, and unit tests +asserting the constructed `Retry` (no server needed). Including requests adds +minutes. Including httpx adds a TLS-adjacent refactor and buys the least. So: +niquests and requests now, httpx when someone asks for it. + +The cost of divergence is a documentation cost, not a code cost - the §2 table is +written for users as much as for us, and `docs/source/http-libraries.rst` needs +the one-liner "the httpx fallbacks do not retry". A `retry` setting that cannot +be honoured should say so rather than pretend, so constructing an httpx-backed +client with an explicit `retry` config should warn. + +## 4. Proposed behaviour + +### 4.1 Method classification + +The library sends exactly ten HTTP methods - grepped from `davclient.py`, +`async_davclient.py` and `base_client.py`, not assumed: `GET`, `OPTIONS`, +`PROPFIND`, `PROPPATCH`, `REPORT`, `PUT`, `DELETE`, `MKCOL`, `MKCALENDAR`, and +`POST`. No `MOVE`, no `COPY` (`CalendarObjectResource.copy()` builds a new +object client-side rather than issuing an HTTP `COPY`), no `HEAD`. + +One shared frozenset, defined once in `caldav/base_client.py` and used both for +`Retry(allowed_methods=...)` and for the Layer C decision: + +```python +IDEMPOTENT_METHODS = frozenset({ + "GET", "HEAD", "OPTIONS", "PROPFIND", "REPORT", # reads + "PUT", "DELETE", "PROPPATCH", "MKCOL", "MKCALENDAR", +}) +# POST is deliberately absent and must never be retried. +# HEAD is listed although the library never sends one - it costs nothing and +# keeps the set readable as "everything except POST". +``` + +`PUT` and `DELETE` belong in there. This is not a compromise, it is what the +protocol says: a CalDAV `PUT` writes a complete object at a known URL and a +second identical `PUT` produces the same end state; `DELETE` on an +already-deleted URL is a 404, not a second deletion. urllib3's own default +`allowed_methods` includes both, for the same reason. `POST` is the only method +where a duplicate can produce a duplicate side effect, and CalDAV barely uses +it. + +The DELETE objection raised in the #648 review - "a retried DELETE returns 404 +and the user gets `NotFoundError` from `event.delete()`, which reads as *it was +never there*" - **does not reproduce.** `DAVObject._post_delete()` is: + +```python +if r.status not in (200, 204, 404): + raise error.DeleteError(errmsg(r)) +``` + +404 is already accepted, and `DAVResponse` does not raise on a 404 for a plain +DELETE (only 401/403 raise, in `_raise_authorization_error`). So a +retried-and-404 DELETE already succeeds silently at the object layer. A user +calling `client.delete(url)` directly gets a 404 response object, which is +correct and unambiguous. No special handling is needed; the concern is +withdrawn. + +`MKCALENDAR`/`MKCOL` are the genuinely awkward ones: if the first attempt +succeeded and its response was lost, the retry gets 405 or 409. See open +question Q3. + +### 4.2 Configuration + +One new client-feature in `compatibility_hints.py`, named `retry` - not #620's +suggested `high-availability`, which promises load balancing and failover that +we do not provide, and not `retry-on-transient-error`, which is a mouthful and +misdescribes class (A): + +```python +"retry": { + "type": "client-feature", + "description": "The client retries requests that failed for transport " + "reasons, and optionally requests answered with a " + "server-error status. The retrying itself is done by the " + "HTTP library, not by the CalDAV library: these keys are " + "mapped onto urllib3's Retry for niquests and requests. " + "The httpx fallbacks can only retry connection setup, and " + "ignore the rest. 429/503 are NOT covered here - they " + "belong to the 'rate-limit' feature.", + "extra_keys": { + "connect": "retries when the connection could not be established (default 2)", + "read": "retries when the connection broke after the request was sent (default 1)", + "status": "retries on a status in status_forcelist (default 0 - off)", + "status_forcelist": "statuses to retry, e.g. [500, 502, 504]. Empty by default.", + "initial_delay": "seconds to sleep before the first retry (default 0)", + "backoff_factor": "exponential backoff multiplier (default 0.5)", + "max_delay": "cap on the sleep between retries (default 30)", + "jitter": "add randomness to the backoff (default true)", + }, +}, +``` + +Defaults, chosen to satisfy #695 ("by default at least one retry whenever a +keep-alive connection has been closed") without turning fast failures into slow +ones: + +* `connect=2`, `read=1`, `status=0`, `status_forcelist=[]` - **on by default** +* everything in class (C) beyond 429/503 - **opt-in**, as #620 asked +* `backoff_factor=0.5`, `max_delay=30`, `jitter=true`, `initial_delay=0` + +Rationale for `read=1` being on by default: the #695 case is a *read* error in +urllib3's taxonomy (see section 2) even though nothing was delivered, so a +default of `read=0` would leave the reported bug unfixed. One retry is enough +for a closed idle socket - the second attempt gets a fresh connection - and a +budget of one bounds the worst case at two timeouts rather than N. + +`retry` also becomes a `DAVClient`/`AsyncDAVClient` keyword accepting a dict (or +`None`, or `False` to disable), plumbed through `CONNKEYS` like the other +connection parameters, so it can come from a config file or environment as well +as from the `features` config. + +### 4.3 Exceptions + +Add `DAVNetworkError(DAVError)` as PR #648 proposed, raised when a transport +failure survives all retries, wrapping the underlying library exception with +`raise ... from e`. This is the only user-visible API addition, and it is +worth it: today the caller has to catch +`niquests.exceptions.ConnectionError` *or* `requests.exceptions.ConnectionError` +*or* `httpx.TransportError` depending on which library happens to be installed, +which defeats the whole point of the fallback chain. Export it from +`caldav.lib.error` and document it in the release notes. + +### 4.4 Interaction with timeouts + +A retry converts a fast failure into a slow one. With `timeout=None` and +`read=1`, a hung server produces an unbounded wait, twice. The documentation +must state the worst case plainly: **wall-clock ≈ (retries + 1) × timeout + +accumulated backoff**, and repeat the existing advice to set an explicit +`timeout`. Consider warning when `retry` is enabled and `timeout is None`. + +### 4.5 Can the 429/503 handling be deleted? + +If the HTTP library is the right home for retry logic, then the ~40 lines of +rate-limit handling in caldav - `raise_if_rate_limited()`, +`compute_sleep_seconds()`, `RateLimitError.retry_after_seconds`, the recursive +sleep-and-retry in `request()` and `_async_request()`, three constructor +arguments - are complexity we are hosting for no good reason. `Retry` can do it: +`status_forcelist=[429, 503]`, `status=N`, `respect_retry_after_header=True` +(the default), and `sleep_for_retry()` sleeps for exactly the header value. + +The direction is right. Three things stand in the way of doing it now, and one +of them is a genuine functional loss: + +1. **niquests has no `rate_limit_max_sleep` equivalent.** Verified in the + source: `urllib3` 2.7.0 clamps the header in `parse_retry_after()` - + "*Check the seconds do not exceed the specified maximum*", `retry_after_max`, + default 21600 - but **`urllib3_future` 2.15.901 has no such clamp and no such + parameter**. A server answering `Retry-After: 86400`, or an HTTP-date a week + out, parks the client in `time.sleep()` for that long, with no way to bound it. + `rate_limit_max_sleep` exists precisely to prevent that. So the *unbounded* + implementation is the one in our recommended library, and this is the piece + that cannot simply be deleted. It is also an easy upstream contribution: + port `retry_after_max` from urllib3 2.7 to urllib3-future. **File that first** + - if it lands, this whole objection evaporates. +2. **It is an API break, so it belongs in 4.0.** `rate_limit_handle`, + `rate_limit_default_sleep` and `rate_limit_max_sleep` are documented + constructor parameters, and `RateLimitError` is a documented exception with a + `retry_after_seconds` attribute that callers can act on. Delegating means + either dropping them (breaking) or keeping them as a mapping onto `Retry` + (which keeps most of the code we wanted to delete). 4.0 is already the + release that reworks the HTTP-library relationship, so that is where this + belongs. +3. **Behaviour changes in ways worth choosing deliberately.** Today the default + is *not* to sleep: `rate_limit_handle=None` auto-detects from the `rate-limit` + feature, and with no such feature the client raises `RateLimitError` and lets + the caller decide. Under `status_forcelist` the client sleeps silently + instead, and when the budget runs out the caller gets a niquests `RetryError` + rather than our typed exception - so a `except RateLimitError` in downstream + code (plann, for instance) stops firing. Also: the httpx fallbacks lose 429 + handling completely, since httpcore has no status retries at all. + +Recommended sequence, therefore: propose `retry_after_max` upstream in +urllib3-future; keep the caldav loop untouched through 3.x; and in 4.0 delete it +in favour of `status_forcelist=[429, 503]`, deprecating the three constructor +arguments in favour of the `retry` config and keeping `RateLimitError` only for +the "raise, do not sleep" default. The `interval`/`count` half of the +`rate-limit` feature - proactive throttling, currently implemented in test code - +is unaffected by all of this, and niquests' `LeakyBucketLimiter` / +`TokenBucketLimiter` are the obvious home for it if it ever moves into the +library. + +## 5. Consequences for PR #648 + +PR #648 is closed rather than merged. It is not wrong about the bug - it is at +the wrong layer, and it is now `CONFLICTING` against the branch anyway: + +* its `_SAFE_METHODS` frozenset is superseded by `IDEMPOTENT_METHODS` (§4.1), + which additionally has to be handed to `Retry(allowed_methods=...)` - the PR + cannot do that, because it never touches session construction; +* its hand-rolled retry loop for classes (A) and (B) duplicates urllib3's, with + no backoff, no jitter, no `Retry-After`, and a one-shot `_connection_retried` + flag instead of a budget; +* the 105-line async restructuring it needs is what the PR's own review flagged + as the riskiest part of it, and Layer A makes it unnecessary: the async fix is + one constructor argument; +* its `except Exception` interaction in the async auth-workaround (network + errors reaching the auth-detection branch when `password` is set but `auth` is + not) is a real pre-existing bug, but it is a bug about auth negotiation and + should be fixed on its own, not inside a retry PR. + +What survives from it, and should be re-submitted as a small PR: `DAVNetworkError` +(§4.3) and the Layer C lazy-gather catch (§3), which is roughly 15 lines per +client instead of 250 across four files. + +## 6. Implementation plan + +Ordered so that each step is independently shippable, and so that the two steps +that fix a reported bug come first. Effort figures are for the code plus its +unit tests, and assume no surprises. + +1. **`DAVNetworkError`** in `caldav/lib/error.py`, plus `IDEMPOTENT_METHODS` in + `caldav/base_client.py`. Unit tests only. *Small.* +2. **Layer C**, the niquests lazy-gather catch, in both clients - roughly 15 + lines each, around the first access to response attributes. This is what + actually fixes https://github.com/python-caldav/caldav/issues/647. + Unit-testable with a `Response` stub whose `status_code` property raises. + *Small.* Report the same thing upstream at + https://github.com/jawah/niquests as a separate, non-blocking action. +3. **Layer A for niquests**: a `_build_retry()` helper in `base_client.py` + mapping the `retry` config dict onto a `Retry`, and `retries=` passed to + `Session(...)` in `DAVClient.__init__` and to `AsyncSession(...)` in + `AsyncDAVClient._create_session()`. Two lines of session diff plus the + helper. *~1 hour including tests.* This is the step that closes + https://github.com/python-caldav/caldav/issues/695 for the default, + recommended configuration. +4. **Layer A for requests**: the mounted-adapter branch in the existing + `except TypeError:` path, reusing the same `Retry` object. *Minutes.* Do it + in the same change as step 3 - splitting it costs more in review than in code. +5. **The `retry` feature** in `compatibility_hints.py`, the `retry` kwarg, + `CONNKEYS`, and config-file/env plumbing. Until this lands, step 3 can ship + with hardcoded defaults. *Small.* +6. **Docs**: the guarantees-per-library table from §2 into + `docs/source/http-libraries.rst`, the timeout warning from §4.4, and a + CHANGELOG entry. +7. **Layer A for the httpx family** - deferred, not scheduled. Needs the + transport refactor described in §3.1 (moving `verify`/`cert`/`http2` into the + transport constructor) and buys connect-retries only. Do it when someone + asks, or when `_create_session()` is being touched anyway for another reason. + +Steps 1-2 are the bug fix and are worth doing on their own. Steps 3-5 are the +feature. Step 7 is optional and step 6 is not. + +## 7. Testing + +* Unit tests, per layer, with no server: a session stub that raises + `ProtocolError` on the first call and succeeds on the second; a `Response` + stub whose `status_code` raises (Layer C); assertions that POST is *not* + retried and that `PROPFIND`/`REPORT` *are* in `allowed_methods` on the + constructed `Retry` object. +* Assertions on the constructed `Retry`/transport rather than on network + behaviour, so the tests run identically under all five libraries and in CI + where only some are installed. +* No mocking in the integration tests. If a server-side scenario cannot be + provoked, skip. +* The `retries` attribute on the urllib3 response (`Retry` with a `.history` of + `RequestHistory(method, url, error, status, redirect_location)`, present in + both urllib3 and urllib3-future) lets an integration test assert *that* a + retry happened, without mocking. Useful for a `tests-behaviour` check if we + ever want one. +* A regression test for #647 is not possible without a server that reproduces + stale HTTP/2 gathering; the unit test on the stub is the substitute. + +## 8. Open questions + +* **Q1.** Should `read` retries default to 1 (as proposed) or 0? 1 fixes #695 + out of the box but means a hung server is waited on twice. Decision needed + before implementation, since it is the one default that can surprise people. +* **Q2.** For the httpx family, classes (B) and (C) are unreachable at Layer A. + Proposal (§3.1): accept the degradation, document it in the §2 table and in + `http-libraries.rst`, warn if a `retry` config is given to an httpx-backed + client, and do not schedule httpx support. niquests is the recommended + library; httpx is a fallback for people with a specific objection, and they can + live with a specific limitation. +* **Q3.** `MKCALENDAR`/`MKCOL` retried after a lost response gives 405/409. + Tolerate it as "the calendar exists, fine" the way `_post_delete` tolerates + 404, or drop them from `IDEMPOTENT_METHODS`? +* **Q4.** #620's original phrasing put the whole thing in the *server* + configuration. Retry policy is a property of the client's environment (a + flaky VPN is not a server peculiarity), which is why §4.2 makes it a + `client-feature` *and* a constructor argument. Confirm that is the intent. +* **Q5.** Should the niquests-only `keepalive_idle_window` / `keepalive_delay` + be exposed? They prevent class (A) failures rather than retrying them, but + only over HTTP/2+, which we mostly do not use (multiplexing is off by + default). Probably not worth a knob. +* **Q6.** §4.5: delete the caldav-side 429/503 handling in 4.0 and let + `status_forcelist` do it? Blocked on `retry_after_max` reaching + urllib3-future, since without a cap on `Retry-After` the delegation is a + regression, not a simplification. Sub-question: is filing that upstream + something we want to spend the effort on? + +## 9. Rejected alternatives + +* **Hand-roll everything in `caldav`** (what #648 does). Rejected: reimplements + urllib3's `Retry` badly, and cannot see the failures that happen inside the + connection pool. +* **Take a dependency on `httpx-retries`.** Rejected: another HTTP-adjacent + dependency, in a project that is actively trying to shed its HTTP dependency + by v4.0. +* **`Retry(total=N)` and nothing else.** Rejected: urllib3's default + `allowed_methods` excludes `PROPFIND` and `REPORT`, so the headline bug would + stay unfixed. See §2. +* **Retry 429/503 at Layer A via `status_forcelist`, *in addition to* the + existing handling.** Rejected: two mechanisms racing over the same responses. + Replacing the existing handling with it is a different proposal, and a good + one - see §4.5 and Q6. +* **Reimplement status retries in caldav for cross-library parity.** Proposed in + an earlier draft of this document, rejected: it adds a caldav-side retry loop + for something urllib3 already does, to benefit a fallback library that most + users do not have installed. The §2 table plus a warning is the cheaper + answer to the parity problem. +* **Retry everything, POST included, on class (A).** Tempting, since nothing + was delivered. Rejected for now: urllib3 counts the #695-style + connection-closed case as a *read* error, so "class (A) only" is not a + distinction we can reliably make at the transport layer, and getting it wrong + means a duplicated scheduling POST. diff --git a/docs/design/TODO_SCHEDULE_TAG.md b/docs/design/TODO_SCHEDULE_TAG.md deleted file mode 100644 index 8e16d41f..00000000 --- a/docs/design/TODO_SCHEDULE_TAG.md +++ /dev/null @@ -1,130 +0,0 @@ -# Schedule-Tag TODO - -## What Schedule-Tag is (RFC 6638) - -Schedule-Tag is an opaque token attached to each scheduling object resource (an event/todo -that carries an ORGANIZER or ATTENDEE). It works like ETag for scheduling, but changes on -a different cadence: ETag changes on every PUT; Schedule-Tag changes only when the -**scheduling-significant** content changes. - -- **Organizer's copy**: tag changes on direct HTTP modifications (PUT/COPY/MOVE), but - **not** when the server auto-processes an attendee reply back onto the organizer's - resource. -- **Attendee's copy**: tag changes when the organizer sends an update, but **not** when - the attendee updates only their own participation status (PARTSTAT). - -The `If-Schedule-Tag-Match` request header lets a client say "only save this if my copy -has not been updated by the organizer since I fetched it". Without it, the classic race -is: - -1. Attendee fetches event (schedule-tag = "X"). -2. Organizer sends an update; server writes new data onto attendee's resource - (schedule-tag = "Y"). -3. Attendee PUTs their PARTSTAT change, unknowingly wiping out step 2. - -With `If-Schedule-Tag-Match: "X"`, step 3 returns 412 and the attendee client knows to -re-fetch and merge. - -References: -- https://datatracker.ietf.org/doc/html/rfc6638#section-3.2 -- https://datatracker.ietf.org/doc/html/rfc6638#section-3.3 -- https://datatracker.ietf.org/doc/html/rfc6638#section-8 - -## Current state in the codebase - -The infrastructure is half-built: - -- `cdav.ScheduleTag` element exists (`caldav/elements/cdav.py:202`). -- GET/load responses capture the `Schedule-Tag` response header into - `self.props[cdav.ScheduleTag.tag]` (`calendarobjectresource.py:873`, `918`). -- `save()` accepts `if_schedule_tag_match: bool = False` but the docstring says - *"is currently ignored"* — it is merely forwarded to `_async_save`, which also ignores - it (`calendarobjectresource.py:1136`). -- `_reply_to_invite_request` calls `get_property(ScheduleTag)` to populate the prop but - then never uses the value. -- `_put` / `_async_put` send a hardcoded header dict containing only `Content-Type` — no - conditional headers at all (not even `If-Match` / `If-None-Match` for ETag). - -## Suggested implementation - -### 1. Add extra-headers support to `_put` / `_async_put` - -`_put` needs to accept an optional extra-headers dict so callers can inject -`If-Schedule-Tag-Match` (and, in the future, `If-Match`): - -```python -def _put(self, retry_on_failure=True, extra_headers=None): - headers = {"Content-Type": 'text/calendar; charset="utf-8"'} - if extra_headers: - headers.update(extra_headers) - r = self.client.put(self.url, self.data, headers) - ... -``` - -### 2. Wire `if_schedule_tag_match` through `save()` → `_put()` - -In `save()` / `_async_save()`, when `if_schedule_tag_match=True`, look up the cached -schedule-tag property and inject the header: - -```python -if if_schedule_tag_match: - tag = self.props.get(cdav.ScheduleTag.tag) - if tag is None: - self.load() # fetch tag before sending conditional PUT - tag = self.props.get(cdav.ScheduleTag.tag) - if tag is not None: - extra_headers["If-Schedule-Tag-Match"] = tag -``` - -A missing cached tag is a notable edge case. The safest default is to do a `load()` -first so the tag is available; alternatively raise `ValueError` to surface the caller -error explicitly. - -### 3. Fix `_reply_to_invite_request` - -This method already calls `get_property(ScheduleTag)` but never uses the result. After -the fix to `save()`, the reply path should call `self.save(if_schedule_tag_match=True)` -so that the attendee's PARTSTAT update is protected against a racing organizer update. - -The current fallback logic in that method is also confused: it re-fetches the schedule-tag -and then recurses with `calendar=outbox`, which bypasses the conditional header entirely. -This needs a clean rewrite once the basic wiring is in place. - -### 4. Expose `schedule_tag` as a public property - -The tag is currently buried in `self.props[cdav.ScheduleTag.tag]`. A simple property -would be cleaner and avoid callers importing `cdav`: - -```python -@property -def schedule_tag(self) -> str | None: - return self.props.get(cdav.ScheduleTag.tag) -``` - -### 5. Raise a specific exception on 412 schedule-tag mismatch - -When the server returns 412 for a schedule-tag mismatch, `_put` raises a generic -`PutError`. A more specific exception lets callers handle the "re-fetch and merge" case: - -```python -class ScheduleTagMismatchError(PutError): - """Server returned 412 because If-Schedule-Tag-Match did not match.""" -``` - -Distinguishing a schedule-tag 412 from an ETag 412 may require inspecting the response -body or a `Schedule-Tag` precondition code. - -### 6. Add a `scheduling.schedule-tag` compatibility hint - -Not all RFC 6638 servers implement schedule-tag (it is a SHOULD, not a MUST). A feature -entry should be added and detected by checking for the `Schedule-Tag` header in a GET -response on a scheduling object resource. - -## What to test - -- `save(if_schedule_tag_match=True)` on a stale object (tag changed server-side) → 412 - → `ScheduleTagMismatchError`. -- `save(if_schedule_tag_match=True)` on a fresh object → 204 → tag updated in props. -- `_reply_to_invite_request` sends `If-Schedule-Tag-Match` and succeeds without wiping - an organizer's concurrent update. -- PARTSTAT-only update does **not** change the schedule-tag (server compliance check). diff --git a/docs/source/about.rst b/docs/source/about.rst index 056d5932..375d1692 100644 --- a/docs/source/about.rst +++ b/docs/source/about.rst @@ -265,7 +265,7 @@ private servers into tests/caldav_test_servers.yaml, see tests/caldav_test_serve Niquests vs Requests vs HTTPX ============================= -By default, CalDAV depends on the niquests library. Some people are not happy with that, so there exists fallbacks to utilize httpx and requests. See the :doc:`http-libraries` document. +By default, CalDAV depends on the niquests library. Some people are not happy with that, so there exists fallbacks to utilize the httpx family (httpx, httpxyz, httpx2) and requests. See the :doc:`http-libraries` document. Documentation ============= diff --git a/docs/source/http-libraries.rst b/docs/source/http-libraries.rst index dcc97b5f..efb06982 100644 --- a/docs/source/http-libraries.rst +++ b/docs/source/http-libraries.rst @@ -1,57 +1,77 @@ HTTP Library Configuration ========================== -As of v3.x, **niquests** is used for HTTP communication. niquests is a backwards-compatible fork of the requests library. It's a modern HTTP library with support for HTTP/2 and HTTP/3 and many other things. Due to popular demand, fallbacks to **requests** and **httpx** exists. +As of v3.x, **niquests** is the preferred, recommended and supported library for HTTP communication. niquests is a backwards-compatible fork of the requests library. It's a modern HTTP library with support for HTTP/2 and HTTP/3 and many other things. + +Due to popular demand, fallbacks to **requests** and to the **httpx** family (httpx, httpxyz, httpx2) exist. + +v4.x is planned to come without explicit dependencies on any HTTP library - the logic is that the consumer probably already imports some HTTP-library, and probably does not want to drag in another dependency. For backward compatibility, it will be necessary to depend on ``caldav[niquests]`` rather than just ``caldav``. Going forward from 3.3.0, every odd patch release will have ``niquests`` included in the dependencies, while every even patch release will have no http library dependencies, allowing consumers that don't want to drag in ``niquests`` (and the related ``urllib3-future``) to avoid them without having to patch the ``pyproject.toml`` file. Context ------- -There is also information in `GitHub issue #457 `_ +Somehow it seems extraordinarily difficult to agree on something as +simple as "how do we do HTTP requests" in the python environment ... Traditionally the CalDAV library only supported the traditional **requests** library, but this library seems to be at a dead end, version 2.0 went into "feature freeze" long ago, but version 3.0 never -materialized. It was suggested to replace it with **niquests**. Niquests (and urllib3-future) started as contributions to the upstream project, but the changes were rejected. Some research has been done before accepting niquests as a dependency in the CalDAV library. - -Niquests is a fork, a drop-in replacement, and just replacing the "re" -with "ni" in the code solved three long-standing issues. The change -was left in the master branch for quite a while, and pushed out in the -2.0-release. Almost immediately after pushing niquests in 2.0, -a complaint was raised from a distro package maintainer who found -niquests unacceptable. Due to that the CalDAV library now has a -fallback implemented; it can use requests for sync communication. - -For async communications (and also as a replacement for requests in -sync usage), **httpx** seems to have been the most popular library -candidate. However, niquests supports async communication. It was -decided to support both niquests and httpx for async communication. - -According to -https://github.com/python-caldav/caldav/issues/611#issuecomment-4278875543 -the httpx development seems stagnant, and httpx is even flagged as a -supply-chain risk in some Reddit-discussions. It seems like the http -user space is filled with drama and intrigues. +materialized (to be fair, the 2.x-series is under active maintenance, +and the 3.0-development hasn't been abandoned - but I'm not holding my +breath). + +**Niquests** was dropped to me in a PR. It is a fork, a drop-in +replacement, and just by replacing the "re" with "ni" in the code I +could close three long-standing issues from the caldav issue tracker. +Niquests (and urllib3-future) started as contributions to the upstream +project, but the changes were rejected. I've done some research - to +me the technical work done in niquests seems robust, to me niquests +3.x seems to be what requests 3.x should have been. + +The change was left in the master branch for quite a while, and pushed +out in the 2.0-release. Almost immediately after pushing niquests in +2.0, a complaint was raised from a distro package maintainer who found +the niquests-dependency unacceptable. Due to that the CalDAV library +now has a fallback implemented; it can use requests for sync +communication. + +Niquests can do both async and sync communication - the same is true +for **httpx**. My impression is that httpx used to be the most +popular library candidate for some time - but there was some `drama +about it +`_. +The package **httpx2** seems to be the continuation of the httpx +project. + +The fallback chain for async communication now is niquests, httpx2, +httpxyz and finally httpx, whichever imports first. + +The three httpx variants share one API, and the CalDAV library treats +them interchangeably. One difference is worth knowing if you write code +around this: httpxyz registers itself in ``sys.modules`` under the name +``httpx``, so ``import httpx`` gets you httpxyz; httpx2 does not, and +stays ``import httpx2``. + +Sync communication falls back to requests, not to httpx as of v3.3.0 - there is an issue on using httpx also for sync communication, see https://github.com/python-caldav/caldav/issues/696 + +More information in the issue tracker: + +* `GitHub issue #457 `_ +* `GitHub issue #611 `_ +* `GitHub issue #690 `_ Fallbacks --------- -To enable the fallbacks, just ensure the requests and/or httpx library is available and that niquests isn't available. In virtual environments, fix the dependencies in `pyproject.toml`. +To enable the fallbacks, just ensure the requests and/or httpxyz/httpx2/httpx library is available and that niquests isn't available. In virtual environments, pin things to the latest even release. Recommendations --------------- -* In general, stick to the package default - niquests. -* In a very sharp production environment, you may consider to use the - good old requests library, but set an appropriate timeout. Use the - sync code, do not use async as the async support is still a bit - experimental. -* If you're using the CalDAV library in a sync project that is already - heavily dependent on the requests library and don't want to drag in - extra dependencies, go for requests. -* If you're using the CalDAV library in an async project that is - already heavily dependent on httpx and don't want to drag in extra - dependencies, use httpx - but do your own due diligence. -* If you have strong personal opinions against niquests, then don't use it. Please share your thoughts at https://github.com/python-caldav/caldav/issues/611 +* If you're using some other http-library, are happy with it and don't want to overthink things, then there is no need to do anything at all - except, if your project depends on httpx you should probably do some due diligence and consider httpx2. +* If you have strong personal opinions against niquests, then you do have the option of actively avoiding it. Please share your thoughts at https://github.com/python-caldav/caldav/issues/611 +* In a very sharp production environment, you may consider to use the good old requests library, but set an appropriate timeout. In a very sharp production environment (as of 3.x), use the CalDAV library in a sync way, the async version of CalDAV still lacks some real-world testing. +* Otherwise, stick to the package default - niquests. Starting from 4.0, you will need to depend on ``caldav[niquests]`` rather than just ``caldav``. Multiplexing ------------ diff --git a/docs/source/jmap.rst b/docs/source/jmap.rst index bbeee9ba..b679ed72 100644 --- a/docs/source/jmap.rst +++ b/docs/source/jmap.rst @@ -26,14 +26,19 @@ Quick Start from caldav.jmap import get_jmap_client - client = get_jmap_client( + with get_jmap_client( url="https://jmap.example.com/.well-known/jmap", username="alice", password="secret", - ) - calendars = client.get_calendars() - for cal in calendars: - print(cal.name) + ) as client: + calendars = client.get_calendars() + for cal in calendars: + print(cal.name) + +The client keeps a persistent HTTP session, so connections are reused across +requests. The ``with`` block releases it at the end; if you would rather hold +on to the client, call ``client.close()`` when you are done +(``await client.aclose()`` on the async client). :func:`~caldav.jmap.get_jmap_client` reads configuration from the same sources as :func:`caldav.get_davclient`: explicit keyword arguments, then the @@ -82,9 +87,9 @@ parameter (niquests is API-compatible with requests). Unlike CalDAV, JMAP does not use a 401-challenge-retry dance — credentials are sent on every request, and a 401 or 403 is a hard :class:`~caldav.jmap.error.JMAPAuthError`. -Context manager usage is supported but not required — no persistent TCP connection is -held between calls (the JMAP Session object is cached after the first request, but -that is just a JSON document, not a socket): +The client holds a persistent HTTP session so connections are reused between calls, +so it is worth releasing it when you are done — either with a context manager or by +calling ``close()`` (``aclose()`` on the async client): .. code-block:: python @@ -197,7 +202,7 @@ scanning the full calendar: # ... time passes, events are created/modified/deleted ... # Fetch only the delta - added, modified, deleted = client.get_objects_by_sync_token(token) + added, modified, deleted, token = client.get_objects_by_sync_token(token) for ical_str in added: print("New:", ical_str) @@ -208,7 +213,9 @@ scanning the full calendar: ``added`` and ``modified`` are lists of VCALENDAR strings. ``deleted`` is a list of event IDs — the objects no longer exist on the server, so their data cannot be -fetched. +fetched. The fourth element is the server's new sync token; chaining straight +from it avoids the race window a separate +:meth:`~caldav.jmap.client.JMAPClient.get_sync_token` round-trip would open. :meth:`~caldav.jmap.client.JMAPClient.get_objects_by_sync_token` raises :class:`~caldav.jmap.error.JMAPMethodError` (``error_type="serverPartialFail"``) if @@ -238,9 +245,8 @@ A typical pattern is to persist the token between runs: token = client.get_sync_token() save_token(token) else: - added, modified, deleted = client.get_objects_by_sync_token(token) + added, modified, deleted, token = client.get_objects_by_sync_token(token) # process changes ... - token = client.get_sync_token() save_token(token) Tasks diff --git a/pyproject.toml b/pyproject.toml index 6fcfe317..7a16e6fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,19 +9,39 @@ source = "vcs" version-file = "caldav/_version.py" [tool.hatch.build.targets.sdist] +## The release is built from a clean checkout (see docs/design/RELEASE-HOWTO.md) +## and `tox -e package` fails if the sdist contains a single file git does not +## track, so this list does not have to enumerate every kind of junk - it only +## has to keep an *ordinary working tree* buildable. +## +## The reason it is needed at all: hatchling's VCS-ignore support honours only +## the ROOT .gitignore. A file hidden from `git status` by a nested .gitignore, +## or by the developer's global git ignore file, is invisible locally and lands +## in the tarball anyway. caldav-3.2.1.tar.gz shipped .claude/settings.json and +## 1755 files under venv/ that way - the venv because RELEASE-HOWTO used to +## create it inside the release clone. exclude = [ - ".#*", # Emacs lock files - "#*#", # Emacs auto-save files - "*~", # Backup files - "*.swp", # Vim swap files - "*.pyc", # Python bytecode + ".#*", # Emacs lock files + "\\#*\\#", # Emacs auto-save files - a bare "#" starts a comment + "*~", # Backup files + "*.swp", # Vim swap files + "*.pyc", # Python bytecode "__pycache__", + ".claude", # per-directory AI harness settings + ".prompts", # AI prompt archive + ".pytest_cache", + ".ruff_cache", + "venv", + "/docs/build", + "/tests/caldav_test_servers.yaml", # may hold credentials + "/tests/conf_private.py", ] [tool.hatch.build.targets.wheel] +packages = ["caldav"] exclude = [ ".#*", - "#*#", + "\\#*\\#", "*~", "*.swp", "*.pyc", @@ -53,7 +73,7 @@ classifiers = [ dependencies = [ "lxml", - "niquests", + "niquests", ## see docs/source/http-libraries.rst "recurring-ical-events>=2.0.0", "typing_extensions;python_version<'3.11'", "icalendar>6.0.0", @@ -71,6 +91,12 @@ Documentation = "https://caldav.readthedocs.io/" Changelog = "https://github.com/python-caldav/caldav/blob/master/CHANGELOG.md" [project.optional-dependencies] +## A no-op today, since niquests is an ordinary dependency above - it exists so +## that `caldav[niquests]` already resolves. v4.0 is planned to depend on no +## HTTP library by default, and this is how consumers will keep the current +## behaviour, so they can depend on it now and change nothing later. See +## docs/source/http-libraries.rst +niquests = ["niquests"] test = [ "vobject", "pytest", @@ -84,26 +110,35 @@ test = [ "pyfakefs", "httpx", "httpxyz", + ## Deliberately NOT enabled, and not an oversight. testCheckCompatibility + ## skips itself without caldav_server_tester, but enabling it here buys + ## nothing and costs something: + ## * caldav_server_tester depends on caldav, so this is a dependency cycle + ## in the published metadata that every downstream packager inherits. + ## * its probe set has to move in lockstep with compatibility_hints.py. A + ## *released* checker only probes the features that existed when it + ## shipped - 1.2.0 probes 63 of the 82 the current checkout does - so CI + ## would go green while covering none of the features a feature branch + ## just added. Green-but-vacuous is worse than skipped-and-honest. + ## testCheckCompatibility is therefore run manually, against a + ## caldav-server-tester checkout developed in step with this branch. See + ## tests/README.md. #"caldav_server_tester" "deptry>=0.24.0; python_version >= '3.10'", ] -[tool.setuptools_scm] -write_to = "caldav/_version.py" - -[tool.setuptools] -py-modules = ["caldav"] -include-package-data = true - -[tool.setuptools.packages.find] -exclude = ["tests"] -namespaces = false - [tool.deptry] ignore = ["DEP002"] # Test dependencies (pytest, coverage, etc.) are not imported in main code [tool.deptry.per_rule_ignores] -DEP001 = ["conf", "h2", "httpxyz"] # conf: Local test config, h2: Optional HTTP/2 support, httpxyz: Optional async HTTP client -DEP003 = ["aiohttp"] # aiohttp: optional dep used only in caldav/testing.py (XandikosServer) +DEP001 = ["conf", "h2"] # conf: Local test config, h2: Optional HTTP/2 support. +## httpxyz needed an entry here while it was imported by name; the httpx-family +## libraries are now imported dynamically (see _ASYNC_HTTPX_CANDIDATES), which +## deptry does not see at all. +DEP003 = ["aiohttp", "h2"] # aiohttp: optional dep used only in caldav/testing.py +## h2 needs an entry in both DEP001 and DEP003: which of the two the optional +## `import h2` in async_davclient.py trips depends on whether h2 happens to be +## installed - DEP001 "missing" when it is not (CI), DEP003 "transitive" when it +## is (a local checkout, where niquests or httpx has pulled it in). [tool.ruff] line-length = 100 @@ -125,14 +160,14 @@ ignore = [ "E722", # Bare except — 28 violations, each needs case-by-case review "F401", # Unused imports — many are re-exports or conditional; needs audit "F821", # Undefined names — often in TYPE_CHECKING blocks or docstrings - "F841", # Unused variables — often intentional (e.g. unpacking) "UP031", # %-format — 31 violations; ruff only offers unsafe auto-fix ] [tool.ruff.lint.per-file-ignores] -# Examples and docs may have undefined names for illustration -"examples/*.py" = ["F821"] -"docs/**/*.py" = ["F821"] +# Examples and docs may have undefined names for illustration, and an unused +# variable there is the point: it shows what a method hands back. +"examples/*.py" = ["F821", "F841"] +"docs/**/*.py" = ["F821", "F841"] # Tests may use assertions and have unusual patterns "tests/*.py" = ["B011", "B017"] diff --git a/tests/README.md b/tests/README.md index aaff6325..a3c307b5 100644 --- a/tests/README.md +++ b/tests/README.md @@ -96,6 +96,26 @@ my-server: You can also use defaults: `${VAR:-default_value}` +Three environment variables switch off a whole *category* of test servers. +Each defaults to enabled; set it to `0` (or `false`/`no`/`off`) to turn the +category off: + +| Variable | Turns off | +| --- | --- | +| `PYTHON_CALDAV_TEST_EMBEDDED` | Xandikos and Radicale, which run in-process | +| `PYTHON_CALDAV_TEST_DOCKER` | every server under `docker-test-servers/` | +| `PYTHON_CALDAV_TEST_EXTERNAL` | servers configured in `caldav_test_servers.yaml` | + +This is the quickest way to skip the Docker servers on a machine that has +Docker running but where you only want a fast offline test run: + +```bash +PYTHON_CALDAV_TEST_DOCKER=0 pytest tests/test_caldav.py +``` + +Individual servers are switched off with `enabled: false` on their entry in +`caldav_test_servers.yaml`. + ### Migration from conf_private.py If you have an existing `conf_private.py`, a migration script is provided: @@ -202,6 +222,35 @@ Tests run automatically on GitHub Actions for: See `.github/workflows/tests.yaml` for the full CI configuration. +### testCheckCompatibility does not run in CI + +`testCheckCompatibility` is the test that validates the declared feature +matrix in `caldav/compatibility_hints.py` against what the servers actually +do. It needs the [caldav-server-tester][cst] package, which is deliberately +*not* in the `test` extra, so the test skips itself unless you install it +yourself. Two reasons: + +* `caldav_server_tester` depends on `caldav`, so listing it would put a + dependency cycle into the published metadata. +* the checker's probe set has to move in lockstep with + `compatibility_hints.py`. A *released* checker only probes the features + that existed when it shipped — 1.2.0 probes 63 where the current checkout + probes 82 — so CI would report success while covering none of the features + a feature branch has just added. A green run that proves nothing is worse + than an honest skip. + +So run it by hand, against a checker checkout developed in step with the +branch you are working on: + +```bash +pip install --editable ../caldav-server-tester +pytest tests/test_caldav.py -k testCheckCompatibility +``` + +Expect it to take a few minutes per configured server. + +[cst]: https://github.com/python-caldav/caldav-server-tester + ## Coverage Generate a coverage report: diff --git a/tests/caldav_test_servers.yaml.example b/tests/caldav_test_servers.yaml.example index cb68b379..3beab28b 100644 --- a/tests/caldav_test_servers.yaml.example +++ b/tests/caldav_test_servers.yaml.example @@ -30,14 +30,17 @@ test-servers: # Docker servers (require docker-compose, see docker-test-servers/) # ========================================================================= # - # Set enabled to: - # - true: always enable - # - false: always disable - # - "auto": enable if docker is available (default for docker servers) + # Set enabled to true or false. Docker servers are skipped automatically + # when Docker is not available, and a running container is auto-detected + # even without a config entry, so listing them here is mainly to opt in or + # out explicitly and to override credentials/ports. + # + # To switch off the docker servers as a group, without editing this file, + # set PYTHON_CALDAV_TEST_DOCKER=0 (see tests/README.md). baikal: type: docker - enabled: ${TEST_BAIKAL:-auto} + enabled: true host: ${BAIKAL_HOST:-localhost} port: ${BAIKAL_PORT:-8800} username: ${BAIKAL_USERNAME:-testuser} @@ -47,7 +50,7 @@ test-servers: nextcloud: type: docker - enabled: ${TEST_NEXTCLOUD:-false} + enabled: false host: ${NEXTCLOUD_HOST:-localhost} port: ${NEXTCLOUD_PORT:-8801} username: ${NEXTCLOUD_USERNAME:-testuser} @@ -56,7 +59,7 @@ test-servers: cyrus: type: docker - enabled: ${TEST_CYRUS:-false} + enabled: false host: ${CYRUS_HOST:-localhost} port: ${CYRUS_PORT:-8802} username: ${CYRUS_USERNAME:-user1} @@ -76,7 +79,7 @@ test-servers: sogo: type: docker - enabled: ${TEST_SOGO:-false} + enabled: false host: ${SOGO_HOST:-localhost} port: ${SOGO_PORT:-8803} username: ${SOGO_USERNAME:-testuser} @@ -84,7 +87,7 @@ test-servers: bedework: type: docker - enabled: ${TEST_BEDEWORK:-false} + enabled: false host: ${BEDEWORK_HOST:-localhost} port: ${BEDEWORK_PORT:-8804} username: ${BEDEWORK_USERNAME:-admin} @@ -92,7 +95,7 @@ test-servers: davical: type: docker - enabled: ${TEST_DAVICAL:-false} + enabled: false host: ${DAVICAL_HOST:-localhost} port: ${DAVICAL_PORT:-8805} username: ${DAVICAL_USERNAME:-testuser} @@ -101,7 +104,7 @@ test-servers: davis: type: docker - enabled: ${TEST_DAVIS:-false} + enabled: false host: ${DAVIS_HOST:-localhost} port: ${DAVIS_PORT:-8806} username: ${DAVIS_USERNAME:-testuser} @@ -109,7 +112,7 @@ test-servers: ccs: type: docker - enabled: ${TEST_CCS:-false} + enabled: false host: ${CCS_HOST:-localhost} port: ${CCS_PORT:-8807} username: ${CCS_USERNAME:-user01} @@ -117,7 +120,7 @@ test-servers: zimbra: type: docker - enabled: ${TEST_ZIMBRA:-false} + enabled: false host: ${ZIMBRA_HOST:-zimbra-docker.zimbra.io} port: ${ZIMBRA_PORT:-8808} username: ${ZIMBRA_USERNAME:-testuser@zimbra.io} @@ -126,7 +129,7 @@ test-servers: stalwart: type: docker - enabled: ${TEST_STALWART:-false} + enabled: false host: ${STALWART_HOST:-localhost} port: ${STALWART_PORT:-8809} # v0.16+: username is a full email address; password must not be in zxcvbn common-word list. @@ -136,7 +139,7 @@ test-servers: # OX App Suite requires a locally built Docker image — run build.sh first. ox: type: docker - enabled: ${TEST_OX:-false} + enabled: false host: ${OX_HOST:-localhost} port: ${OX_PORT:-8810} username: ${OX_USERNAME:-oxadmin} diff --git a/tests/docker-test-servers/baikal/README.md b/tests/docker-test-servers/baikal/README.md index 104cbcd3..3ecb778d 100644 --- a/tests/docker-test-servers/baikal/README.md +++ b/tests/docker-test-servers/baikal/README.md @@ -62,8 +62,6 @@ baikal: enabled: false ``` -Or use the environment variable: `TEST_BAIKAL=false`. - Or simply don't install Docker - the tests will automatically skip Baikal if Docker is not available. ## GitHub Actions (CI/CD) @@ -99,7 +97,7 @@ You can add more secrets in GitHub Actions settings for credentials. The test suite will automatically detect and use Baikal if configured. Configuration is in `tests/caldav_test_servers.yaml` (copy from `tests/caldav_test_servers.yaml.example` and customize). -To enable Baikal testing, set `enabled: true` (or `enabled: auto` to auto-detect Docker availability) in the YAML config: +To enable Baikal testing, set `enabled: true` in the YAML config: ```yaml baikal: @@ -107,7 +105,9 @@ baikal: enabled: true ``` -Or use the environment variable: `TEST_BAIKAL=true`. +Docker servers are also auto-detected: a running container is picked up by the +test suite even without an explicit config entry, and is skipped automatically +when Docker is not available. ## Troubleshooting diff --git a/tests/docker-test-servers/baikal/configure_baikal.py b/tests/docker-test-servers/baikal/configure_baikal.py index 711a7e13..09a824fa 100755 --- a/tests/docker-test-servers/baikal/configure_baikal.py +++ b/tests/docker-test-servers/baikal/configure_baikal.py @@ -128,10 +128,12 @@ def create_baikal_database(db_path: Path, username: str, password: str) -> None: def main() -> int: """Main function.""" # Get configuration from environment + ## BAIKAL_ADMIN_PASSWORD and BAIKAL_PASSWORD are read by + ## create_baikal_config() / create_baikal_database(), which main() only + ## prints instructions for rather than calling; both are documented in the + ## module docstring. baikal_url = os.environ.get("BAIKAL_URL", "http://localhost:8800") - admin_password = os.environ.get("BAIKAL_ADMIN_PASSWORD", "admin") username = os.environ.get("BAIKAL_USERNAME", "testuser") - password = os.environ.get("BAIKAL_PASSWORD", "testpass") print(f"Configuring Baikal at {baikal_url}") print(f"Test user: {username}") diff --git a/tests/docker-test-servers/ccs/start.sh b/tests/docker-test-servers/ccs/start.sh index 02ea1447..658c9028 100755 --- a/tests/docker-test-servers/ccs/start.sh +++ b/tests/docker-test-servers/ccs/start.sh @@ -42,7 +42,7 @@ echo " Users: user01/user01, user02/user02, admin/admin" echo "" echo "Run tests from project root:" echo " cd ../../.." -echo " TEST_CCS=true pytest" +echo " pytest" echo "" echo "To stop CCS: ./stop.sh" echo "To view logs: docker-compose logs -f ccs" diff --git a/tests/docker-test-servers/cyrus/README.md b/tests/docker-test-servers/cyrus/README.md index 9c4dfdd2..02a15df1 100644 --- a/tests/docker-test-servers/cyrus/README.md +++ b/tests/docker-test-servers/cyrus/README.md @@ -67,8 +67,6 @@ cyrus: enabled: false ``` -Or use the environment variable: `TEST_CYRUS=false`. - Or simply don't install Docker - the tests will automatically skip Cyrus if Docker is not available. ## Troubleshooting diff --git a/tests/docker-test-servers/davical/start.sh b/tests/docker-test-servers/davical/start.sh index 5b9edb54..32add709 100755 --- a/tests/docker-test-servers/davical/start.sh +++ b/tests/docker-test-servers/davical/start.sh @@ -26,7 +26,7 @@ bash "$SCRIPT_DIR/setup_davical.sh" echo "" echo "Run tests from project root:" echo " cd ../../.." -echo " TEST_DAVICAL=true pytest" +echo " pytest" echo "" echo "To stop DAViCal: ./stop.sh" echo "To view logs: docker-compose logs -f" diff --git a/tests/docker-test-servers/davis/start.sh b/tests/docker-test-servers/davis/start.sh index 3e0235b0..34d52b22 100755 --- a/tests/docker-test-servers/davis/start.sh +++ b/tests/docker-test-servers/davis/start.sh @@ -23,7 +23,7 @@ bash "$SCRIPT_DIR/setup_davis.sh" echo "" echo "Run tests from project root:" echo " cd ../../.." -echo " TEST_DAVIS=true pytest" +echo " pytest" echo "" echo "To stop Davis: ./stop.sh" echo "To view logs: docker-compose logs -f davis" diff --git a/tests/docker-test-servers/nextcloud/README.md b/tests/docker-test-servers/nextcloud/README.md index 44e696b8..48b6bbe5 100644 --- a/tests/docker-test-servers/nextcloud/README.md +++ b/tests/docker-test-servers/nextcloud/README.md @@ -44,7 +44,8 @@ This will: This Nextcloud instance comes **pre-configured** with: - Admin user: `admin` / `admin` -- Test user: `testuser` / `TestPassword123!` +- Test user: `testuser` / `testpass` +- Scheduling test users: `user1` / `testpass1`, `user2` / `testpass2`, `user3` / `testpass3` - Calendar and Contacts apps enabled - CalDAV URL: `http://localhost:8801/remote.php/dav` @@ -56,7 +57,7 @@ This Nextcloud instance comes **pre-configured** with: - `NEXTCLOUD_URL`: URL of the Nextcloud server (default: `http://localhost:8801`) - `NEXTCLOUD_USERNAME`: Test user username (default: `testuser`) -- `NEXTCLOUD_PASSWORD`: Test user password (default: `TestPassword123!`) +- `NEXTCLOUD_PASSWORD`: Test user password (default: `testpass`) ## Disabling Nextcloud Tests @@ -68,8 +69,6 @@ nextcloud: enabled: false ``` -Or use the environment variable: `TEST_NEXTCLOUD=false`. - Or simply don't install Docker - the tests will automatically skip Nextcloud if Docker is not available. ## Troubleshooting @@ -95,6 +94,18 @@ docker-compose ps docker inspect nextcloud-test ``` +### Every request returns HTTP 500 +Usually a root-owned file inside `/var/www/html` that the server (running as +`www-data`) cannot write — `config.php` and the files under `data/` are the +usual suspects: +```bash +docker exec nextcloud-test find /var/www/html -user root -not -type l +``` +`occ` must therefore always be invoked as the web server user +(`docker exec -u www-data ...`, which is what `setup_nextcloud.sh` does); running +it as root is what creates such files in the first place. `./stop.sh && ./start.sh` +clears it, since the tree lives on tmpfs. + ### Reset Nextcloud ```bash # Stop and remove container with volumes (WARNING: deletes all data) diff --git a/tests/docker-test-servers/nextcloud/docker-compose.yml b/tests/docker-test-servers/nextcloud/docker-compose.yml index b02b4bf2..2b186850 100644 --- a/tests/docker-test-servers/nextcloud/docker-compose.yml +++ b/tests/docker-test-servers/nextcloud/docker-compose.yml @@ -4,6 +4,9 @@ services: container_name: nextcloud-test ports: - "8801:80" + # Read by the custom entrypoint below, not by the image's own entrypoint — + # keep the two in sync, an ignored variable here silently misconfigures the + # setup script (SQLITE_DATABASE decides the data/.db file name). environment: - SQLITE_DATABASE=nextcloud - NEXTCLOUD_ADMIN_USER=admin @@ -32,10 +35,11 @@ services: echo "Running Nextcloud installation..." su -s /bin/bash www-data -c "php /var/www/html/occ maintenance:install \ --database=sqlite \ - --admin-user=admin \ - --admin-pass=admin" + --database-name=${SQLITE_DATABASE:-nextcloud} \ + --admin-user=${NEXTCLOUD_ADMIN_USER:-admin} \ + --admin-pass=${NEXTCLOUD_ADMIN_PASSWORD:-admin}" # Set trusted domains - su -s /bin/bash www-data -c "php /var/www/html/occ config:system:set trusted_domains 0 --value=localhost" + su -s /bin/bash www-data -c "php /var/www/html/occ config:system:set trusted_domains 0 --value=${NEXTCLOUD_TRUSTED_DOMAINS:-localhost}" fi # Start apache diff --git a/tests/docker-test-servers/nextcloud/setup_nextcloud.sh b/tests/docker-test-servers/nextcloud/setup_nextcloud.sh index ef804171..46b863e6 100755 --- a/tests/docker-test-servers/nextcloud/setup_nextcloud.sh +++ b/tests/docker-test-servers/nextcloud/setup_nextcloud.sh @@ -4,14 +4,29 @@ set -e -CONTAINER_NAME="nextcloud-test" +CONTAINER_NAME="${NEXTCLOUD_CONTAINER:-nextcloud-test}" +NEXTCLOUD_PORT="${NEXTCLOUD_PORT:-8801}" TEST_USER="testuser" TEST_PASSWORD="testpass" +# Nextcloud requires occ to be run as the web server user. `docker exec` runs as +# root by default, and any file occ creates then ends up root-owned inside +# /var/www/html — which the server itself (running as www-data) cannot write +# afterwards, leaving an install that answers HTTP 500 on every request. +# Usage: occ [-e VAR=value]... +occ() { + local envs=() + while [ "$1" = "-e" ]; do + envs+=(-e "$2") + shift 2 + done + docker exec "${envs[@]}" -u www-data "$CONTAINER_NAME" php occ "$@" +} + echo "Waiting for Nextcloud to be ready..." max_attempts=60 for i in $(seq 1 $max_attempts); do - if docker exec $CONTAINER_NAME php occ status 2>/dev/null | grep -q "installed: true"; then + if occ status 2>/dev/null | grep -q "installed: true"; then echo "✓ Nextcloud is ready" break fi @@ -25,41 +40,41 @@ done echo "" echo "Disabling password policy for testing..." -docker exec $CONTAINER_NAME php occ app:disable password_policy || true +occ app:disable password_policy || true echo "Creating test users..." # Create test user (ignore error if already exists) -docker exec -e OC_PASS="$TEST_PASSWORD" $CONTAINER_NAME php occ user:add --password-from-env --display-name="Test User" $TEST_USER 2>/dev/null || echo "User may already exist" +occ -e OC_PASS="$TEST_PASSWORD" user:add --password-from-env --display-name="Test User" $TEST_USER 2>/dev/null || echo "User may already exist" # Create scheduling test users for i in 1 2 3; do - docker exec -e OC_PASS="testpass${i}" $CONTAINER_NAME php occ user:add --password-from-env --display-name="User ${i}" "user${i}" 2>/dev/null || echo "user${i} may already exist" + occ -e OC_PASS="testpass${i}" user:add --password-from-env --display-name="User ${i}" "user${i}" 2>/dev/null || echo "user${i} may already exist" # Set email address — required for CalDAV scheduling (calendar-user-address-set) - docker exec $CONTAINER_NAME php occ user:setting "user${i}" settings email "user${i}@localhost" || true + occ user:setting "user${i}" settings email "user${i}@localhost" || true done echo "Enabling calendar app..." -docker exec $CONTAINER_NAME php occ app:enable calendar || true +occ app:enable calendar || true echo "Enabling contacts app..." -docker exec $CONTAINER_NAME php occ app:enable contacts || true +occ app:enable contacts || true echo "Configuring bruteforce protection..." # Temporarily enable bruteforce protection so we can reset accumulated failed # auth attempts (which pile up while the server is starting before users exist). -docker exec $CONTAINER_NAME php occ config:system:set auth.bruteforce.protection.enabled --value=true --type=boolean || true +occ config:system:set auth.bruteforce.protection.enabled --value=true --type=boolean || true for ip in 127.0.0.1 ::1; do - docker exec $CONTAINER_NAME php occ security:bruteforce:reset "$ip" 2>/dev/null || true + occ security:bruteforce:reset "$ip" 2>/dev/null || true done # Detect the Docker gateway IP and reset it too -GATEWAY_IP=$(docker exec $CONTAINER_NAME sh -c "ip route | awk '/default/{print \$3}'" 2>/dev/null || true) +GATEWAY_IP=$(docker exec "$CONTAINER_NAME" sh -c "ip route | awk '/default/{print \$3}'" 2>/dev/null || true) if [ -n "$GATEWAY_IP" ]; then - docker exec $CONTAINER_NAME php occ security:bruteforce:reset "$GATEWAY_IP" 2>/dev/null || true + occ security:bruteforce:reset "$GATEWAY_IP" 2>/dev/null || true fi # Now disable bruteforce protection — the caldav library handles 429 via # rate_limit_handle, but Nextcloud's bruteforce gives no Retry-After header # and would make tests slow. -docker exec $CONTAINER_NAME php occ app:disable bruteforcesettings || true -docker exec $CONTAINER_NAME php occ config:system:set auth.bruteforce.protection.enabled --value=false --type=boolean || true +occ app:disable bruteforcesettings || true +occ config:system:set auth.bruteforce.protection.enabled --value=false --type=boolean || true echo "Disabling CalDAV trashbin (calendar retention)..." # Setting calendarRetentionObligation to '0' (the string) disables the trashbin in @@ -68,26 +83,62 @@ echo "Disabling CalDAV trashbin (calendar retention)..." # causing UNIQUE constraint violations when tests recreate a calendar with the same slug # (Nextcloud 33+ reuses the calendarid, keeping old soft-deleted objects, so adding # an event with the same UID fails). -docker exec $CONTAINER_NAME php occ config:app:set dav calendarRetentionObligation --value=0 || true +occ config:app:set dav calendarRetentionObligation --value=0 || true # Purge any leftover soft-deleted calendars/objects from previous runs -docker exec $CONTAINER_NAME php occ dav:retention:clean-up || true +occ dav:retention:clean-up || true echo "Configuring CalDAV rate limits..." -docker exec $CONTAINER_NAME php occ config:app:set dav rateLimitCalendarCreation --value=99999 || true -docker exec $CONTAINER_NAME php occ config:app:set dav maximumCalendarsSubscriptions --value=-1 || true +occ config:app:set dav rateLimitCalendarCreation --value=99999 || true +occ config:app:set dav maximumCalendarsSubscriptions --value=-1 || true echo "Adding IP whitelist for rate limiting..." # Service is test-only and never exposed externally, so whitelist everything -docker exec $CONTAINER_NAME php occ config:system:set ratelimit.whitelist.0 --value='0.0.0.0/0' || true -docker exec $CONTAINER_NAME php occ config:system:set ratelimit.whitelist.1 --value='::/0' || true +occ config:system:set ratelimit.whitelist.0 --value='0.0.0.0/0' || true +occ config:system:set ratelimit.whitelist.1 --value='::/0' || true echo "Clearing rate limit cache..." -docker exec $CONTAINER_NAME php -r " -\$db = new PDO('sqlite:/var/www/html/data/nextcloud.db'); -\$db->exec('DELETE FROM oc_ratelimit_entries'); -\$db->exec('DELETE FROM oc_bruteforce_attempts'); -echo 'Cleared rate limit and bruteforce caches\n'; -" || true +# The SQLite file is named after the `dbname` config value, and Nextcloud's +# default is `owncloud`, not `nextcloud` — look it up instead of guessing. Guard +# on the file existing as well: PDO happily *creates* a missing SQLite file, so a +# wrong path leaves a bogus empty database behind and every DELETE below fails +# with "no such table". +DB_NAME=$(occ config:system:get dbname 2>/dev/null | tr -d '\r\n') +DB_PATH="/var/www/html/data/${DB_NAME:-owncloud}.db" +if docker exec -u www-data "$CONTAINER_NAME" test -f "$DB_PATH"; then + docker exec -u www-data "$CONTAINER_NAME" php -r " + \$db = new PDO('sqlite:$DB_PATH'); + foreach (['oc_ratelimit_entries', 'oc_bruteforce_attempts'] as \$table) { + try { + \$db->exec(\"DELETE FROM \$table\"); + } catch (PDOException \$e) { + fwrite(STDERR, \"skipping \$table: \" . \$e->getMessage() . \"\n\"); + } + } + echo \"Cleared rate limit and bruteforce caches\n\"; + " || true +else + echo "✗ No database found at $DB_PATH — skipping cache cleanup" +fi + +echo "Verifying the CalDAV endpoint..." +# The configuration above is worthless if the server itself is broken (a +# root-owned config.php or data file will make every request 500). Fail loudly +# here rather than leaving it for the test suite to discover. +for i in $(seq 1 30); do + HTTP_CODE=$(curl -s -o /dev/null -w '%{http_code}' -u "$TEST_USER:$TEST_PASSWORD" \ + "http://localhost:${NEXTCLOUD_PORT}/remote.php/dav/" || echo 000) + case "$HTTP_CODE" in + 2*|3*) + echo "✓ CalDAV endpoint answers HTTP $HTTP_CODE" + break + ;; + esac + if [ "$i" -eq 30 ]; then + echo "✗ CalDAV endpoint answers HTTP $HTTP_CODE — this Nextcloud is broken" + exit 1 + fi + sleep 1 +done echo "" echo "✓ Nextcloud setup complete!" @@ -96,5 +147,5 @@ echo "Credentials:" echo " Admin: admin / admin" echo " Test user: $TEST_USER / $TEST_PASSWORD" echo " Scheduling users: user1/testpass1, user2/testpass2, user3/testpass3" -echo " CalDAV URL: http://localhost:8801/remote.php/dav" +echo " CalDAV URL: http://localhost:${NEXTCLOUD_PORT}/remote.php/dav" echo "" diff --git a/tests/docker-test-servers/ox/README.md b/tests/docker-test-servers/ox/README.md index eedf2695..ff5a6347 100644 --- a/tests/docker-test-servers/ox/README.md +++ b/tests/docker-test-servers/ox/README.md @@ -1,6 +1,6 @@ # OX App Suite CalDAV Test Server -[OX App Suite](https://www.open-xchange.com/) is a commercial groupware platform with CalDAV/CardDAV support. +[OX App Suite](https://ox.io/) is a commercial groupware platform with CalDAV/CardDAV support. ## Prerequisites @@ -42,7 +42,7 @@ are used so the container always starts clean). ```bash cd ../../.. -TEST_OX=true pytest tests/test_caldav.py -k OX -v +pytest tests/test_caldav.py -k OX -v ``` ## Notes diff --git a/tests/docker-test-servers/ox/start.sh b/tests/docker-test-servers/ox/start.sh index 38de5c98..6c923f84 100755 --- a/tests/docker-test-servers/ox/start.sh +++ b/tests/docker-test-servers/ox/start.sh @@ -60,7 +60,7 @@ echo " User: oxadmin / oxadmin" echo "" echo "Run tests from project root:" echo " cd ../../.." -echo " TEST_OX=true pytest tests/test_caldav.py -k OX -v" +echo " pytest tests/test_caldav.py -k OX -v" echo "" echo "To stop: ./stop.sh" echo "To view logs: docker-compose logs -f ox" diff --git a/tests/docker-test-servers/sogo/README.md b/tests/docker-test-servers/sogo/README.md index a9fce51e..230157fc 100644 --- a/tests/docker-test-servers/sogo/README.md +++ b/tests/docker-test-servers/sogo/README.md @@ -66,8 +66,6 @@ sogo: enabled: false ``` -Or use the environment variable: `TEST_SOGO=false`. - Or simply don't install Docker - the tests will automatically skip SOGo if Docker is not available. ## Troubleshooting diff --git a/tests/docker-test-servers/stalwart/start.sh b/tests/docker-test-servers/stalwart/start.sh index b9e7d0fc..39bc91a8 100755 --- a/tests/docker-test-servers/stalwart/start.sh +++ b/tests/docker-test-servers/stalwart/start.sh @@ -17,7 +17,7 @@ bash "$SCRIPT_DIR/setup_stalwart.sh" echo "" echo "Run tests from project root:" echo " cd ../../.." -echo " TEST_STALWART=true pytest" +echo " pytest" echo "" echo "To stop Stalwart: ./stop.sh" echo "To view logs: docker-compose logs -f stalwart" diff --git a/tests/docker-test-servers/zimbra/README.md b/tests/docker-test-servers/zimbra/README.md index 05e34cfc..213b3d3a 100644 --- a/tests/docker-test-servers/zimbra/README.md +++ b/tests/docker-test-servers/zimbra/README.md @@ -45,7 +45,7 @@ The start script will: ```bash cd ../../.. -TEST_ZIMBRA=true pytest tests/test_caldav.py -k Zimbra -v +pytest tests/test_caldav.py -k Zimbra -v ``` ## Notes diff --git a/tests/docker-test-servers/zimbra/start.sh b/tests/docker-test-servers/zimbra/start.sh index 2e0f267c..81142b8e 100755 --- a/tests/docker-test-servers/zimbra/start.sh +++ b/tests/docker-test-servers/zimbra/start.sh @@ -74,7 +74,7 @@ echo " testuser3@$ZIMBRA_DOMAIN / testpass" echo "" echo "Run tests from project root:" echo " cd ../../.." -echo " TEST_ZIMBRA=true pytest tests/test_caldav.py -k Zimbra -v" +echo " pytest tests/test_caldav.py -k Zimbra -v" echo "" echo "To stop Zimbra: ./stop.sh" echo "To view logs: docker-compose logs -f zimbra" diff --git a/tests/fixture_helpers.py b/tests/fixture_helpers.py index b20eb45a..b7bb7b99 100644 --- a/tests/fixture_helpers.py +++ b/tests/fixture_helpers.py @@ -237,6 +237,39 @@ async def cleanup_calendar_objects(calendar: Any) -> None: try: await _maybe_await(obj.delete()) except Exception: + # Best-effort, and deliberately broad: one object that refuses + # to go must not stop the rest of the calendar from being + # emptied, and every caller treats a failed cleanup as + # acceptable. The catch is wide enough to hide a client-side + # bug too - see adelete_calendar_if_present() below, where + # exactly that happened - so narrow it if you get the chance. pass except Exception: + # Ditto for listing the calendar: if search() fails there is nothing + # this helper can clean up, and it is not the test's business to fail + # over it. + pass + + +async def adelete_calendar_if_present(principal: Any, cal_id: str) -> None: + """Best-effort removal of a leftover test calendar from a previous run. + + A test that recreates a calendar with a fixed ``cal_id`` must first clear + any leftover, or the recreate MKCALENDAR 405s ("resource already exists"). + + Only ``NotFoundError`` (the calendar isn't there) is swallowed - everything + else propagates. A previous incarnation wrapped this in a bare + ``except Exception: pass``, which silently hid a real bug (async + ``principal.calendar()`` raising ``TypeError``), so the cleanup never ran + and calendars leaked. Keep the catch narrow so that can't recur. + """ + from caldav.lib import error + + calendar = await _maybe_await(principal.calendar(cal_id=cal_id)) + await cleanup_calendar_objects(calendar) + try: + await _maybe_await(calendar.delete()) + except error.NotFoundError: + # Already gone, which is exactly what this function wants. Nothing + # wider is caught on purpose - see the docstring. pass diff --git a/tests/test_async_davclient.py b/tests/test_async_davclient.py index 7d327563..13d2973a 100644 --- a/tests/test_async_davclient.py +++ b/tests/test_async_davclient.py @@ -6,6 +6,7 @@ communication. We use Mock/MagicMock to emulate server communication. """ +import inspect import os from unittest.mock import AsyncMock, MagicMock, patch @@ -186,7 +187,7 @@ def test_session_creation_without_proxy_does_not_pass_proxy_kwarg(self) -> None: if not _USE_HTTPX: pytest.skip("test only relevant for httpx backend") - with patch("httpx.AsyncClient") as mock_client: + with patch("caldav.async_davclient.httpx.AsyncClient") as mock_client: AsyncDAVClient(url="https://caldav.example.com/dav/") _, call_kwargs = mock_client.call_args assert "proxy" not in call_kwargs, ( @@ -200,7 +201,7 @@ def test_session_creation_with_proxy_passes_proxy_kwarg(self) -> None: if not _USE_HTTPX: pytest.skip("test only relevant for httpx backend") - with patch("httpx.AsyncClient") as mock_client: + with patch("caldav.async_davclient.httpx.AsyncClient") as mock_client: AsyncDAVClient( url="https://caldav.example.com/dav/", proxy="proxy.example.com:8080", @@ -1058,3 +1059,177 @@ async def test_rate_limit_max_sleep_stops_adaptive_retries(self): with patch("caldav.async_davclient.asyncio.sleep", new_callable=AsyncMock): with pytest.raises(error.RateLimitError): await client.request("/") + + +class TestAsyncPrincipalCalendar: + """``principal.calendar()`` must work with async clients. + + Regression test: ``principal.calendar(cal_id=)`` used to raise + ``TypeError: argument of type 'coroutine' is not a container or iterable`` + for async clients, because the synchronous ``calendar_home_set`` property + evaluated ``"@" in `` without awaiting the async ``get_property``. + The cleanup blocks in the integration tests wrapped the call in a bare + ``except``, so calendars leaked silently and a later MKCALENDAR 405'd. + """ + + @pytest.mark.asyncio + async def test_calendar_by_cal_id_returns_awaitable(self) -> None: + """A plain cal_id needs the home set, so async returns a coroutine.""" + from caldav.collection import Calendar, Principal + + client = AsyncDAVClient(url="https://caldav.example.com/dav/") + principal = Principal(client=client, url="https://caldav.example.com/dav/principals/user/") + + ## The calendar-home-set discovery is the only would-be round-trip; mock + ## the async get_property so the test stays offline. + with patch.object( + Principal, + "get_property", + new=AsyncMock(return_value="https://caldav.example.com/dav/calendars/user/"), + ): + result = principal.calendar(cal_id="testcal") + assert inspect.iscoroutine(result), "async calendar() must return a coroutine" + calendar = await result + + assert isinstance(calendar, Calendar) + assert str(calendar.url).endswith("/calendars/user/testcal/") + + @pytest.mark.asyncio + async def test_calendar_by_full_url_stays_synchronous(self) -> None: + """A full-URL cal_id needs no home set, so it must NOT become a coroutine. + + ``test_calendar_by_full_url`` calls this without ``await`` and reads + ``.url`` directly, so the sync short-circuit must be preserved. + """ + from caldav.collection import Calendar, Principal + + client = AsyncDAVClient(url="https://caldav.example.com/dav/") + principal = Principal(client=client, url="https://caldav.example.com/dav/principals/user/") + + calendar = principal.calendar( + cal_id="https://caldav.example.com/dav/calendars/user/testcal/" + ) + assert isinstance(calendar, Calendar) + assert str(calendar.url).endswith("/calendars/user/testcal/") + + @pytest.mark.asyncio + async def test_calendar_by_name_returns_awaitable(self) -> None: + """Gate finding F5: only the bare-``cal_id`` half was fixed. A + ``name`` lookup went on to iterate the *coroutine* returned by the + async ``get_calendars()``, raising ``TypeError: 'coroutine' object is + not iterable``.""" + from caldav.collection import Calendar, CalendarSet, Principal + + client = AsyncDAVClient(url="https://caldav.example.com/dav/") + principal = Principal(client=client, url="https://caldav.example.com/dav/principals/user/") + + wanted = Calendar(client, url="https://caldav.example.com/dav/calendars/user/wanted/") + other = Calendar(client, url="https://caldav.example.com/dav/calendars/user/other/") + + async def fake_display_name(self: Calendar) -> str: + return "Wanted" if str(self.url).endswith("/wanted/") else "Other" + + with ( + patch.object( + Principal, + "get_property", + new=AsyncMock(return_value="https://caldav.example.com/dav/calendars/user/"), + ), + patch.object(CalendarSet, "get_calendars", new=AsyncMock(return_value=[other, wanted])), + patch.object(Calendar, "get_display_name", new=fake_display_name), + ): + result = principal.calendar(name="Wanted") + assert inspect.iscoroutine(result), "async calendar() must return a coroutine" + calendar = await result + + assert isinstance(calendar, Calendar) + assert str(calendar.url).endswith("/calendars/user/wanted/") + + @pytest.mark.asyncio + async def test_calendar_by_unknown_name_raises_notfound(self) -> None: + from caldav.collection import Calendar, CalendarSet, Principal + from caldav.lib import error + + client = AsyncDAVClient(url="https://caldav.example.com/dav/") + principal = Principal(client=client, url="https://caldav.example.com/dav/principals/user/") + other = Calendar(client, url="https://caldav.example.com/dav/calendars/user/other/") + + async def fake_display_name(self: Calendar) -> str: + return "Other" + + with ( + patch.object( + Principal, + "get_property", + new=AsyncMock(return_value="https://caldav.example.com/dav/calendars/user/"), + ), + patch.object(CalendarSet, "get_calendars", new=AsyncMock(return_value=[other])), + patch.object(Calendar, "get_display_name", new=fake_display_name), + ): + with pytest.raises(error.NotFoundError): + await principal.calendar(name="Wanted") + + +class TestAsyncHttpLibrarySelection: + """Which async HTTP library the module picks, and what happens when none is there. + + niquests is preferred; failing that, the httpx family is tried in order. + These tests exercise the selection itself rather than whichever library the + test run happens to have installed - one process can only ever have made + one choice, so the choosing has to be testable on its own. + """ + + def test_httpx2_is_an_accepted_library(self) -> None: + """https://github.com/python-caldav/caldav/issues/611 - httpx2 is Pydantic's + continuation of httpx and has to be usable as a fallback.""" + from caldav.async_davclient import _ASYNC_HTTPX_CANDIDATES + + assert "httpx2" in _ASYNC_HTTPX_CANDIDATES + + def test_httpx2_is_preferred_over_httpxyz_and_httpx(self) -> None: + """A deliberate ordering decision, not an accident of the list: httpx2 is + the maintained continuation of httpx, httpxyz is a fork of it.""" + from caldav.async_davclient import _ASYNC_HTTPX_CANDIDATES + + order = {name: i for i, name in enumerate(_ASYNC_HTTPX_CANDIDATES)} + assert order["httpx2"] < order["httpxyz"] < order["httpx"] + + def test_candidates_are_tried_in_order_and_stop_at_the_first_hit(self) -> None: + from caldav.async_davclient import _import_first_available + + wanted = MagicMock(name="httpx2") + tried = [] + + def importer(name: str) -> MagicMock: + tried.append(name) + if name == "httpx2": + return wanted + raise ImportError(f"No module named {name!r}") + + assert _import_first_available(("httpxyz", "httpx2", "httpx"), importer) == ( + "httpx2", + wanted, + ) + assert tried == ["httpxyz", "httpx2"], "should not keep importing after a hit" + + def test_nothing_importable_yields_no_library(self) -> None: + from caldav.async_davclient import _import_first_available + + def importer(name: str) -> None: + raise ImportError(f"No module named {name!r}") + + assert _import_first_available(("httpxyz", "httpx2", "httpx"), importer) == (None, None) + + def test_the_missing_library_error_names_every_option(self) -> None: + """The error is the only guidance a user gets, so it must list all of them.""" + from caldav.async_davclient import _ASYNC_HTTPX_CANDIDATES, _NO_ASYNC_LIBRARY_ERROR + + for name in ("niquests", *_ASYNC_HTTPX_CANDIDATES): + assert name in _NO_ASYNC_LIBRARY_ERROR + + def test_httpxyz_is_reported_as_httpxyz(self) -> None: + """_USE_HTTPXYZ is asserted on by the CI fallback jobs, so it has to keep + meaning "the httpxyz fork specifically", not "some httpx-alike".""" + from caldav.async_davclient import _HTTPX_FLAVOUR, _USE_HTTPXYZ + + assert _USE_HTTPXYZ == (_HTTPX_FLAVOUR == "httpxyz") diff --git a/tests/test_async_integration.py b/tests/test_async_integration.py index ce1c10aa..36d8f502 100644 --- a/tests/test_async_integration.py +++ b/tests/test_async_integration.py @@ -19,6 +19,7 @@ from caldav import Event, FreeBusy, Todo from caldav.compatibility_hints import FeatureSet +from caldav.lib import error from .test_caldav import ( ev1 as ev1_static, # old-date (2006); distinct from ev1() near-future generator @@ -28,6 +29,10 @@ from .test_caldav import evr as evr_static # recurring annual event (1997) from .test_caldav import evr2 as evr2_static # bi-weekly with exception (2024) from .test_caldav import journal as journal_static +from .test_caldav import ( + near_now_ics, # shift an ical event's DTSTART/DTEND to ~now (sliding-window servers) + next_anniversary_windows, # near-future search windows for a FREQ=YEARLY event +) from .test_caldav import todo as todo_static # avoids clash with local var in add_todo() from .test_caldav import todo2 as todo2_static # avoids clash with todo2() generator from .test_caldav import todo3 as todo3_static @@ -54,6 +59,27 @@ async def wrapper(*args, **kwargs): return wrapper +## HTTP methods that change server state; a "write-delay" server settles each of +## these asynchronously, so we sleep AFTER every such request (the write-side +## counterpart of the search-cache delay, which only delays searches). +_WRITE_HTTP_METHODS = frozenset( + {"PUT", "DELETE", "MKCALENDAR", "MKCOL", "PROPPATCH", "MOVE", "COPY", "POST"} +) + + +def _async_write_delay_decorator(f, t=10): + """Sleep after every write request, to let an asynchronous server settle.""" + + @wraps(f) + async def wrapper(url, method="GET", *args, **kwargs): + response = await f(url, method, *args, **kwargs) + if str(method).upper() in _WRITE_HTTP_METHODS: + await asyncio.sleep(t) + return response + + return wrapper + + # Dynamic test data generators - use near-future dates to avoid # min-date-time restrictions on servers like CCS. _base_date = None @@ -219,6 +245,17 @@ async def async_client(self, test_server: TestServer, monkeypatch: Any) -> Any: _async_delay_decorator(AsyncCalendar.search, t=delay), ) + ## Apply write-delay (sleep after every write) for asynchronous servers. + ## Wrapped on the client instance, so monkeypatch reverts it after the test. + write_delay_config = client.features.is_supported("write-delay", dict) + if write_delay_config.get("behaviour") == "delay": + delay = write_delay_config.get("delay", 10) + monkeypatch.setattr( + client, + "request", + _async_write_delay_decorator(client.request, t=delay), + ) + yield client await client.close() @@ -472,21 +509,25 @@ async def test_principal_make_calendar(self, async_client: Any) -> None: from caldav.aio import AsyncCalendarSet, AsyncPrincipal from caldav.lib.error import AuthorizationError, MkcalendarError, NotFoundError - from .fixture_helpers import cleanup_calendar_objects + from .fixture_helpers import adelete_calendar_if_present, cleanup_calendar_objects cal_id = "pythoncaldav-async-test" calendar = None principal = None - # Try principal-based calendar creation (most servers) + # Try principal-based calendar creation (most servers). Clear any + # leftover with this cal_id first, so we exercise real creation and + # don't accumulate calendars (some servers enforce a quota). try: principal = await AsyncPrincipal.create(async_client) + await adelete_calendar_if_present(principal, cal_id) calendar = await principal.make_calendar(name="Async Test", cal_id=cal_id) except (MkcalendarError, AuthorizationError): - # Calendar already exists from a previous run - reuse it - # (mirrors sync _fixCalendar_ pattern) + # Calendar exists and can't be (re)created (e.g. no delete support + # to clear it first) - reuse it. Note: principal.calendar() returns + # a coroutine for async clients, so it must be awaited. if principal is not None: - calendar = principal.calendar(cal_id=cal_id) + calendar = await principal.calendar(cal_id=cal_id) except NotFoundError: # Principal discovery failed pass @@ -497,17 +538,24 @@ async def test_principal_make_calendar(self, async_client: Any) -> None: try: calendar = await calendar_home.make_calendar(name="Async Test", cal_id=cal_id) except (MkcalendarError, AuthorizationError): + # client.calendar() builds a Calendar by URL with no I/O, so it + # is not a coroutine and must not be awaited. calendar = async_client.calendar(cal_id=cal_id) assert calendar is not None - assert calendar.url is not None - - # Clean up based on server capabilities - if self.is_supported("delete-calendar"): - await calendar.delete() - else: - # Can't delete the calendar, just wipe its objects - await cleanup_calendar_objects(calendar) + try: + assert calendar.url is not None + finally: + # Always clean up so calendars don't accumulate (quota safety). + try: + if self.is_supported("delete-calendar"): + await calendar.delete() + else: + # Can't delete the calendar, just wipe its objects + await cleanup_calendar_objects(calendar) + except NotFoundError: + # Already gone - the cleanup got the outcome it wanted. + pass @pytest.mark.asyncio async def test_search_events(self, async_calendar: Any) -> None: @@ -541,6 +589,112 @@ async def test_search_events_by_date_range(self, async_calendar: Any) -> None: assert len(events) >= 1 assert "Async Test Event" in events[0].data + @pytest.mark.asyncio + async def test_search_without_comptype_with_date_range(self, async_calendar: Any) -> None: + """Async mirror of testSearchWithoutCompTypeWithDateRange. + + Test for https://github.com/python-caldav/caldav/issues/681 + + A time-range search that does NOT specify a component type must work + even on SabreDAV-based servers (Baikal, Nextcloud, ...) which - correctly + per RFC4791 section 9.7 - reject a CALDAV:time-range placed directly under + VCALENDAR with HTTP 400. The library works around this by splitting the + search into one query per component type. + + The search is run twice: once with the server's real feature + configuration, and once with search.time-range.comp-type-optional forced + to "supported", exercising the reactive HTTP-400 fallback. + """ + self.skip_unless_support("search.time-range.event") + base = _get_base_date() + uid = f"issue681-async-{uuid.uuid4()}@example.com" + await add_event( + async_calendar, + make_event( + uid, + "issue 681 async comp-type-less time-range", + base, + base + timedelta(hours=1), + ), + ) + + start = base - timedelta(hours=1) + end = base + timedelta(days=1) + + async def _assert_event_found(): + ## must not raise (the crux of issue #681) and must find the event + objects = await async_calendar.search(start=start, end=end) + assert [o for o in objects if uid in o.data], ( + "comp-type-less time-range search did not return the event" + ) + + ## Run 1: the server's real feature configuration (proactive comp-type split) + await _assert_event_found() + + ## Determine how this server reacts to the raw comp-type-less time-range + ## query. Only SabreDAV-style servers reject it with a ReportError (HTTP + ## 400) - the case the reactive fallback (issue #681 item 4) recovers from. + ## Others return nothing or a different error (e.g. Cyrus may answer 403), + ## where forcing the feature on is an unrecoverable misconfiguration. + from caldav.lib import error + + try: + await async_calendar.search(start=start, end=end, compatibility_workarounds=False) + raw_report_error = False + except error.ReportError: + raw_report_error = True + except error.DAVError: + raw_report_error = False + + ## Run 2 (only meaningful where the raw query raises a ReportError): force + ## the feature ON and verify the reactive fallback recovers and finds the event. + if raw_report_error: + features = async_calendar.client.features + key = "search.time-range.comp-type-optional" + had_key = key in features._server_features + saved = features._server_features.get(key) + features.set_feature(key, {"support": "full"}) + try: + objects = await async_calendar.search(start=start, end=end) + assert [o for o in objects if uid in o.data], ( + "reactive fallback did not recover the comp-type-less time-range search" + ) + finally: + if had_key: + features._server_features[key] = saved + else: + features._server_features.pop(key, None) + + @pytest.mark.asyncio + async def test_search_without_comptype_with_category(self, async_calendar: Any) -> None: + """Async mirror of testSearchWithoutCompTypeWithCategory. + + Test for https://github.com/python-caldav/caldav/issues/681 + + A property filter (CATEGORIES) without a component type must work. Under + the VCALENDAR comp-filter it targets VCALENDAR's own properties (no + CATEGORIES), so servers match nothing; the library splits the search into + one query per component type (search.text.comp-type-optional unsupported). + """ + self.skip_unless_support("search.text.category") + base = _get_base_date() + category = "issue681cat" + uuid.uuid4().hex[:8] + uid = f"issue681cat-async-{uuid.uuid4()}@example.com" + data = make_event( + uid, + "issue 681 async comp-type-less category search", + base, + base + timedelta(hours=1), + ).replace("END:VEVENT", f"CATEGORIES:{category}\nEND:VEVENT") + await add_event(async_calendar, data) + + ## Only the proactive split is testable here: servers silently return + ## nothing for a prop-filter under VCALENDAR (no error to recover from). + objects = await async_calendar.search(category=category) + assert [o for o in objects if uid in o.data], ( + "comp-type-less category search did not return the event" + ) + @pytest.mark.asyncio async def test_search_todos_pending(self, async_task_list: Any) -> None: """Test searching for pending todos.""" @@ -649,8 +803,9 @@ async def test_lookup_event(self, async_calendar: Any) -> None: self.skip_unless_support("save-load.event") c = async_calendar - # create the event - e1 = await c.add_event(ev1_static) + # create the event (near-now date so it stays visible to REPORT-based + # lookups on sliding-window servers; see near_now_ics) + e1 = await c.add_event(near_now_ics(ev1_static)) assert e1.url is not None # Verify that we can look it up from calendar by url @@ -661,7 +816,10 @@ async def test_lookup_event(self, async_calendar: Any) -> None: # look up by UID e3 = await c.get_event_by_uid("20010712T182145Z-123401@example.com") assert str(e3.icalendar_component["uid"]) == "20010712T182145Z-123401@example.com" - assert e3.url == e1.url + ## get_event_by_uid may return a different (canonical) URL than the PUT + ## URL on servers that don't preserve it (e.g. OX); see save-load.stable-url + if self.is_supported("save-load.stable-url"): + assert e3.url == e1.url # load directly from URL without going through the calendar object e4 = Event(client=c.client, url=e1.url) @@ -679,23 +837,31 @@ async def test_create_overwrite_delete_event(self, async_calendar: Any) -> None: self.skip_unless_support("save-load.event") c = async_calendar + ## near-now date so the event stays visible to REPORT-based lookups on + ## sliding-window servers (e.g. OX); see near_now_ics + ev1_now = near_now_ics(ev1_static) + # attempting to update a non-existing event must raise ConsistencyError with pytest.raises(error.ConsistencyError): - await c.add_event(ev1_static, no_create=True) + await c.add_event(ev1_now, no_create=True) # no_create + no_overwrite is always an error with pytest.raises(error.ConsistencyError): - await c.add_event(ev1_static, no_create=True, no_overwrite=True) + await c.add_event(ev1_now, no_create=True, no_overwrite=True) - e1 = await c.add_event(ev1_static) + e1 = await c.add_event(ev1_now) assert e1.url is not None - # same UID again → overwrite (unless server forbids it) - if not self.is_supported("save-load.mutable"): - e2 = await c.add_event(ev1_static) + # same UID again → overwrite (unless server forbids it). Overwriting via + # a fresh PUT without an If-Match etag is gated on save-load.mutable.if-match-optional: + # OX enforces optimistic concurrency and rejects such a PUT with 409. + if self.is_supported("save-load.mutable") and self.is_supported( + "save-load.mutable.if-match-optional" + ): + await c.add_event(ev1_now) # no_create on an existing event must succeed - e2 = await c.add_event(ev1_static, no_create=True) + e2 = await c.add_event(ev1_now, no_create=True) # modify and save with no_create e2.icalendar_component["summary"] = "Bastille Day Party!" @@ -706,7 +872,7 @@ async def test_create_overwrite_delete_event(self, async_calendar: Any) -> None: # no_overwrite on an existing event must raise ConsistencyError with pytest.raises(error.ConsistencyError): - await c.add_event(ev1_static, no_overwrite=True) + await c.add_event(ev1_now, no_overwrite=True) await e1.delete() @@ -766,14 +932,18 @@ async def test_load_event(self, async_calendar: Any, async_calendar2: Any) -> No c1 = async_calendar - e1_ = await c1.add_event(ev1_static) + e1_ = await c1.add_event(near_now_ics(ev1_static)) await e1_.load() # load the object returned by add_event events = await c1.get_events() assert len(events) >= 1 e1 = events[0] await e1.load() # load a freshly fetched handle - assert e1.url == e1_.url + ## e1 came from a search and may carry a different (canonical) URL than + ## the PUT URL on servers that don't preserve it (e.g. OX); see + ## save-load.stable-url + if self.is_supported("save-load.stable-url"): + assert e1.url == e1_.url @pytest.mark.asyncio async def test_copy_event(self, async_calendar: Any, async_calendar2: Any) -> None: @@ -784,14 +954,15 @@ async def test_copy_event(self, async_calendar: Any, async_calendar2: Any) -> No c1 = async_calendar c2 = async_calendar2 - e1_ = await c1.add_event(ev1_static) + await c1.add_event(near_now_ics(ev1_static)) events = await c1.get_events() e1 = events[0] # duplicate in same calendar with a new UID - e1_dup = e1.copy() - await e1_dup.save() - assert len(await c1.get_events()) == 2 + if self.is_supported("save.duplicate-event"): + e1_dup = e1.copy() + await e1_dup.save() + assert len(await c1.get_events()) == 2 # copy cross-calendar keeping the same UID if self.is_supported("save.duplicate-uid.cross-calendar"): @@ -808,8 +979,12 @@ async def test_copy_event(self, async_calendar: Any, async_calendar2: Any) -> No # copy in same calendar keeping UID — same-UID PUT is a no-op / overwrite e1_dup2 = e1.copy(keep_uid=True) await e1_dup2.save() - # count should still be 2 (not 3) because same UID overwrites - assert len(await c1.get_events()) == 2 + # same UID overwrites, so the count is unchanged: 2 where a new-UID + # duplicate was created above, 1 where duplicates are not allowed + if self.is_supported("save.duplicate-event"): + assert len(await c1.get_events()) == 2 + else: + assert len(await c1.get_events()) == 1 @pytest.mark.asyncio async def test_multi_get(self, async_calendar: Any) -> None: @@ -861,7 +1036,7 @@ async def test_object_by_sync_token(self, async_calendar: Any) -> None: objcnt += len(await c.get_todos()) objcnt += len(await c.get_events()) - obj = await c.add_event(ev1_static) + obj = await c.add_event(near_now_ics(ev1_static)) objcnt += 1 if self.is_supported("save-load.event.recurrences"): await c.add_event(evr_static) @@ -921,7 +1096,7 @@ async def test_object_by_sync_token(self, async_calendar: Any) -> None: if is_time_based: await asyncio.sleep(1) - obj3 = await c.add_event(ev3_static) + await c.add_event(near_now_ics(ev3_static)) if is_time_based: await asyncio.sleep(1) my_changed_objects = await c.get_objects_by_sync_token( @@ -983,7 +1158,7 @@ async def test_sync(self, async_calendar: Any) -> None: objcnt += len(await c.get_todos()) objcnt += len(await c.get_events()) - obj = await c.add_event(ev1_static) + obj = await c.add_event(near_now_ics(ev1_static)) objcnt += 1 if self.is_supported("save-load.event.recurrences"): await c.add_event(evr_static) @@ -1001,6 +1176,25 @@ async def test_sync(self, async_calendar: Any) -> None: assert my_objects.sync_token != "" assert len(list(my_objects)) == objcnt + stable_url = self.is_supported("save-load.stable-url") + + def synced_match(o): + """Return the synced object corresponding to o, or None. + + objects_by_url() is keyed by the server-reported URL, which on + servers that don't preserve the PUT URL (e.g. OX; see + save-load.stable-url) differs from o.url - so fall back to matching + by UID there. + """ + synced = my_objects.objects_by_url() + if stable_url: + return synced.get(o.url) + uid = o.icalendar_component["uid"] + return next( + (cand for cand in synced.values() if cand.icalendar_component["uid"] == uid), + None, + ) + if is_time_based: await asyncio.sleep(1) @@ -1024,12 +1218,12 @@ async def test_sync(self, async_calendar: Any) -> None: if not is_fragile: assert len(list(updated)) == 1 assert len(list(deleted)) == 0 - assert "foobar" in my_objects.objects_by_url()[obj.url].data + assert "foobar" in synced_match(obj).data if is_time_based: await asyncio.sleep(1) - obj3 = await c.add_event(ev3_static) + obj3 = await c.add_event(near_now_ics(ev3_static)) if is_time_based: await asyncio.sleep(1) @@ -1038,7 +1232,7 @@ async def test_sync(self, async_calendar: Any) -> None: if not is_fragile: assert len(list(updated)) == 1 assert len(list(deleted)) == 0 - assert obj3.url in my_objects.objects_by_url() + assert synced_match(obj3) is not None self.skip_unless_support("sync-token.delete") @@ -1052,7 +1246,7 @@ async def test_sync(self, async_calendar: Any) -> None: if not is_fragile: assert len(list(updated)) == 0 assert len(list(deleted)) == 1 - assert obj.url not in my_objects.objects_by_url() + assert synced_match(obj) is None if is_time_based: await asyncio.sleep(1) @@ -1203,33 +1397,39 @@ def cleanse(tasks: list) -> list: dtstart=date(2022, 10, 11), uid="async-sort-test1", ) + assert t1 is not None t2 = await c.add_todo( summary="2 task future", due=datetime.now() + timedelta(hours=15), dtstart=datetime.now() + timedelta(minutes=15), uid="async-sort-test2", ) + assert t2 is not None t3 = await c.add_todo( summary="3 task future due", due=datetime.now() + timedelta(hours=15), dtstart=datetime(2022, 12, 11, 10, 9, 8), uid="async-sort-test3", ) + assert t3 is not None t4 = await c.add_todo( summary="4 task priority is set to nine which is the lowest", priority=9, uid="async-sort-test4", ) + assert t4 is not None t5 = await c.add_todo( summary="5 task status is set to COMPLETED and this will disappear from the ordinary todo search", status="COMPLETED", uid="async-sort-test5", ) + assert t5 is not None t6 = await c.add_todo( summary="6 task has categories", categories="home,garden,sunshine", uid="async-sort-test6", ) + assert t6 is not None def check_order(tasks: list, order: tuple) -> None: assert [str(x.icalendar_component["uid"]) for x in tasks] == [ @@ -1304,30 +1504,35 @@ async def test_recurring_date_search(self, async_calendar: Any) -> None: self.skip_unless_support("search.recurrences.includes-implicit.event") c = async_calendar + # evr is a yearly event starting at 1997-11-02. Search the next future + # Nov-2 anniversary rather than a fixed historic year, so sliding-window + # servers (e.g. OX) can serve the time range. + year, narrow_start, narrow_end, wide_end = next_anniversary_windows() + await c.add_event(evr_static) r = await c.search( event=True, - start=datetime(2008, 11, 1, 17, 0, 0), - end=datetime(2008, 11, 3, 17, 0, 0), + start=narrow_start, + end=narrow_end, expand=False, ) assert len(r) == 1 r = await c.search( event=True, - start=datetime(2008, 11, 1, 17, 0, 0), - end=datetime(2008, 11, 3, 17, 0, 0), + start=narrow_start, + end=narrow_end, expand=True, ) assert len(r) == 1 assert r[0].data.count("END:VEVENT") == 1 - assert r[0].data.count("DTSTART;VALUE=DATE:2008") == 1 + assert r[0].data.count(f"DTSTART;VALUE=DATE:{year}") == 1 r2 = await c.search( event=True, - start=datetime(2008, 11, 1, 17, 0, 0), - end=datetime(2009, 11, 3, 17, 0, 0), + start=narrow_start, + end=wide_end, expand=True, ) assert len(r2) == 2 @@ -1469,6 +1674,7 @@ async def test_todo_completion(self, async_task_list: Any) -> None: c = async_task_list t1 = await c.add_todo(todo_static) + assert t1 is not None t2 = await c.add_todo(todo2_static) t3 = await c.add_todo(todo3_static, status="NEEDS-ACTION") @@ -1591,8 +1797,8 @@ async def test_todo_datesearch(self, async_task_list: Any) -> None: foo = 5 if not self.is_supported("search.recurrences.includes-implicit.todo"): foo -= 1 - if self.check_compatibility_flag( - "vtodo_datesearch_nodtstart_task_is_skipped" + if not self.is_supported( + "search.time-range.todo.no-dtstart" ) or self.check_compatibility_flag( "vtodo_datesearch_nodtstart_task_is_skipped_in_closed_date_range" ): @@ -1649,7 +1855,9 @@ async def test_scheduling_info(self, async_client: Any) -> None: self.skip_unless_support("scheduling.calendar-user-address-set") principal = await async_client.principal() calendar_user_address_set = await principal.calendar_user_address_set() + assert calendar_user_address_set is not None me_a_participant = await principal.get_vcal_address() + assert me_a_participant is not None @pytest.mark.asyncio async def test_scheduling_mailboxes(self, async_client: Any) -> None: @@ -1657,14 +1865,18 @@ async def test_scheduling_mailboxes(self, async_client: Any) -> None: self.skip_unless_support("scheduling.mailbox") principal = await async_client.principal() inbox = await principal.schedule_inbox() + assert inbox is not None outbox = await principal.schedule_outbox() + assert outbox is not None @pytest.mark.asyncio async def test_propfind(self, async_client: Any) -> None: """Raw XML propfind returns a multistatus response.""" from caldav.lib.python_utilities import to_local - self._skip_on_compatibility_flag("propfind_allprop_failure") + ## This only asserts a multistatus is returned, so (unlike the sync + ## testPropfind, which checks for DAV:resourcetype) it needs no + ## propfind.allprop.resourcetype gate. principal = await async_client.principal() foo = await async_client.propfind( principal.url, @@ -1740,7 +1952,7 @@ async def test_create_delete_calendar(self, async_client: Any) -> None: from caldav.aio import AsyncPrincipal from caldav.lib.error import AuthorizationError, NotFoundError - from .fixture_helpers import cleanup_calendar_objects + from .fixture_helpers import adelete_calendar_if_present principal = None try: @@ -1749,18 +1961,15 @@ async def test_create_delete_calendar(self, async_client: Any) -> None: pytest.skip("Cannot discover principal") cal_id = "pythoncaldav-async-createdelete-test" - try: - existing = principal.calendar(cal_id=cal_id) - await cleanup_calendar_objects(existing) - await existing.delete() - except Exception: - pass + await adelete_calendar_if_present(principal, cal_id) c = await principal.make_calendar(name="Yep", cal_id=cal_id) - assert c.url is not None - events = await c.get_events() - assert len(events) == 0 - await c.delete() + try: + assert c.url is not None + events = await c.get_events() + assert len(events) == 0 + finally: + await c.delete() @pytest.mark.asyncio async def test_calendar_by_full_url(self, async_calendar: Any, async_client: Any) -> None: @@ -1793,9 +2002,12 @@ async def test_set_calendar_properties(self, async_client: Any) -> None: from caldav.elements import dav from caldav.lib.error import AuthorizationError, NotFoundError - from .fixture_helpers import cleanup_calendar_objects + from .fixture_helpers import adelete_calendar_if_present self.skip_unless_support("create-calendar.set-displayname") + ## This test expects the display name to round-trip at a stable URL; + ## servers that relocate the calendar when a name is set (Zimbra) can't. + self.skip_unless_support("create-calendar.stable-url") self.skip_unless_support("delete-calendar") self.skip_unless_support("create-calendar") @@ -1806,26 +2018,23 @@ async def test_set_calendar_properties(self, async_client: Any) -> None: pytest.skip("Cannot discover principal") cal_id = "pythoncaldav-async-props-test" - try: - existing = principal.calendar(cal_id=cal_id) - await cleanup_calendar_objects(existing) - await existing.delete() - except Exception: - pass - - c = await principal.make_calendar(name="Yep", cal_id=cal_id) + await adelete_calendar_if_present(principal, cal_id) + + ## Use a distinct display name (not the sync fixture's "Yep") so that an + ## interrupted run of this test can never leave behind a second calendar + ## named "Yep" that would make the sync suite's principal.calendar(name="Yep") + ## lookup ambiguous. This test only checks that the display name round-trips, + ## so the actual name is irrelevant. + c = await principal.make_calendar(name="AsyncYep", cal_id=cal_id) try: props = await c.get_properties([dav.DisplayName()]) - assert "Yep" == props[dav.DisplayName.tag] + assert "AsyncYep" == props[dav.DisplayName.tag] - await c.set_properties([dav.DisplayName("hooray")]) + await c.set_properties([dav.DisplayName("hooray-async")]) props = await c.get_properties([dav.DisplayName()]) - assert props[dav.DisplayName.tag] == "hooray" + assert props[dav.DisplayName.tag] == "hooray-async" finally: - try: - await c.delete() - except Exception: - pass + await c.delete() # ==================== Group F – Regressions ==================== @@ -1916,6 +2125,9 @@ async def test_change_attendee_status_with_email_given( ) -> None: """change_attendee_status(attendee=email) updates PARTSTAT correctly.""" self.skip_unless_support("save-load.event") + ## Some servers (e.g. OX) forbid changing an attendee's PARTSTAT via a + ## direct PUT (403 Forbidden) and require iTIP scheduling instead. + self.skip_unless_support("save-load.mutable.attendee-partstat") c = async_calendar event = await c.add_event( uid="test1", @@ -1926,6 +2138,7 @@ async def test_change_attendee_status_with_email_given( event.change_attendee_status(attendee="testuser@example.com", PARTSTAT="ACCEPTED") await event.save() event2 = await c.get_event_by_uid("test1") + assert event2 is not None @pytest.mark.asyncio async def test_add_orphaned_recurrence(self, async_calendar: Any) -> None: @@ -1963,55 +2176,69 @@ async def test_edit_single_recurrence(self, async_calendar: Any) -> None: self.skip_unless_support("search.text") cal = async_calendar + ## Anchor the daily recurring event a few days in the future so servers + ## with a sliding REPORT window / no old-date support (e.g. CCS, ref + ## search.time-range.event.old-dates) can still serve the time ranges. + ## The integer passed to search()/summary_on() is a day offset from this + ## anchor day; the values just need to be distinct future days. + base = (datetime.now() + timedelta(days=2)).replace( + hour=8, minute=7, second=6, microsecond=0 + ) + await cal.add_event( uid="test1", summary="daily test", - dtstart=datetime(2015, 1, 1, 8, 7, 6), - dtend=datetime(2015, 1, 1, 9, 7, 6), + dtstart=base, + dtend=base + timedelta(hours=1), rrule={"FREQ": "DAILY"}, ) - async def search(month): + def day_start(offset): + return (base + timedelta(days=offset)).replace( + hour=0, minute=0, second=0, microsecond=0 + ) + + async def search(offset): recurrence = await cal.search( event=True, - start=datetime(2015, month, 1), - end=datetime(2015, month, 2), + start=day_start(offset), + end=day_start(offset) + timedelta(days=1), expand=True, ) assert len(recurrence) == 1 return recurrence[0] - async def summary_by_month(month): - return (await search(month)).icalendar_component["summary"] + async def summary_on(offset): + return (await search(offset)).icalendar_component["summary"] recurrence = await search(7) recurrence.icalendar_component["summary"] = "half a year of daily testing" await recurrence.save() - assert await summary_by_month(6) == "daily test" - assert await summary_by_month(7) == "half a year of daily testing" - assert await summary_by_month(8) == "daily test" + assert await summary_on(6) == "daily test" + assert await summary_on(7) == "half a year of daily testing" + assert await summary_on(8) == "daily test" recurrence = await search(2) recurrence.icalendar_component["summary"] = "one month of daily testing" await recurrence.save() - assert await summary_by_month(1) == "daily test" - assert await summary_by_month(2) == "one month of daily testing" - assert await summary_by_month(7) == "half a year of daily testing" + assert await summary_on(1) == "daily test" + assert await summary_on(2) == "one month of daily testing" + assert await summary_on(7) == "half a year of daily testing" recurrence = await search(7) recurrence.icalendar_component["summary"] = "six months of daily testing" await recurrence.save() - assert await summary_by_month(7) == "six months of daily testing" + assert await summary_on(7) == "six months of daily testing" recurrence = await search(9) recurrence.icalendar_component["summary"] = "daily testing" await recurrence.save(all_recurrences=True) - assert await summary_by_month(1) == "daily testing" - assert await summary_by_month(2) == "one month of daily testing" - assert await summary_by_month(3) == "daily testing" - assert await summary_by_month(7) == "six months of daily testing" + assert await summary_on(1) == "daily testing" + assert await summary_on(2) == "one month of daily testing" + assert await summary_on(3) == "daily testing" + assert await summary_on(7) == "six months of daily testing" # ==================== Group G – Auth errors & misc ==================== @@ -2113,6 +2340,7 @@ async def test_offset_url(self, async_client: Any, async_calendar: Any) -> None: conn = await self._make_async_client_with_params(url=url) p = await conn.principal() calendars = await p.get_calendars() + assert calendars is not None @pytest.mark.asyncio async def test_utf8_event(self, async_client: Any) -> None: @@ -2123,7 +2351,7 @@ async def test_utf8_event(self, async_client: Any) -> None: from caldav.aio import AsyncPrincipal from caldav.lib.error import AuthorizationError, NotFoundError - from .fixture_helpers import cleanup_calendar_objects + from .fixture_helpers import adelete_calendar_if_present principal = None try: @@ -2132,24 +2360,18 @@ async def test_utf8_event(self, async_client: Any) -> None: pytest.skip("Cannot discover principal") cal_id = "pythoncaldav-async-utf8-test" - try: - existing = await principal.calendar(cal_id=cal_id) - await cleanup_calendar_objects(existing) - await existing.delete() - except Exception: - pass + await adelete_calendar_if_present(principal, cal_id) c = await principal.make_calendar(name="Yølp", cal_id=cal_id) try: - await c.add_event(ev1_static.replace("Bastille Day Party", "Bringebærsyltetøyfestival")) + await c.add_event( + near_now_ics(ev1_static).replace("Bastille Day Party", "Bringebærsyltetøyfestival") + ) events = await c.get_events() if "zimbra" not in str(c.url): assert len(events) == 1 finally: - try: - await c.delete() - except Exception: - pass + await c.delete() @pytest.mark.asyncio async def test_create_calendar_and_event_from_vobject(self, async_calendar: Any) -> None: @@ -2158,7 +2380,7 @@ async def test_create_calendar_and_event_from_vobject(self, async_calendar: Any) self.skip_unless_support("save-load.event") c = async_calendar cnt = len(await c.get_events()) - ve1 = vobject.readOne(ev1_static) + ve1 = vobject.readOne(near_now_ics(ev1_static)) await c.add_event(ve1) cnt += 1 events = await c.get_events() @@ -2224,6 +2446,7 @@ async def test_create_task_list_and_todo(self, async_task_list: Any) -> None: self.skip_unless_support("save-load.todo") c = async_task_list t = await c.add_todo(uid="well_known_t1", summary="Well-known async task") + assert t is not None todos = await c.get_todos() assert any(str(x.icalendar_component.get("uid", "")) == "well_known_t1" for x in todos) obj = await c.get_object_by_uid("well_known_t1") @@ -2365,26 +2588,49 @@ async def test_invite_and_respond(self, scheduling_setup: Any) -> None: ## scheduling asynchronously, so poll with backoff before giving up. new_attendee_inbox_items: list[Any] = [] auto_scheduled = False + last_scan_error: Exception | None = None for _ in range(30): - new_attendee_inbox_items = [ - item for item in await inbox1.get_items() if item.url not in inbox_urls_before - ] - ## Check whether the server auto-scheduled the event directly into - ## the attendee's calendar. The event may land in any calendar, - ## so search all attendee calendars for the event UID. - ## Always check even when inbox items were found: some servers (e.g. - ## Davis/sabre/dav) deliver iTIP to the inbox AND auto-schedule. - if not auto_scheduled: - for cal in await principals[1].calendars(): - for event in await cal.get_events(): - if event.id == event_uid: - auto_scheduled = True + try: + ## Correlate by UID: a late METHOD:CANCEL from another scheduling + ## test's teardown can otherwise land here as a stray "new" item + ## (see testAcceptInviteUsernameEmailFallback). + new_attendee_inbox_items = [ + item + for item in await inbox1.get_items() + if item.url not in inbox_urls_before and item.id == event_uid + ] + ## Check whether the server auto-scheduled the event directly into + ## the attendee's calendar. The event may land in any calendar, + ## so search all attendee calendars for the event UID. + ## Always check even when inbox items were found: some servers (e.g. + ## Davis/sabre/dav) deliver iTIP to the inbox AND auto-schedule. + if not auto_scheduled: + for cal in await principals[1].calendars(): + for event in await cal.get_events(): + if event.id == event_uid: + auto_scheduled = True + break + if auto_scheduled: break - if auto_scheduled: - break + except (error.ResponseError, ValueError) as scan_error: + ## A scheduling object that is present but not yet readable - an + ## empty or non-iCalendar body - is exactly what this poll exists + ## to wait out, so it must not abort it. Seen against Zimbra as + ## icalendar's "Found no components where exactly one is + ## required". Kept for the failure message if we run out of + ## rounds, so a permanently unreadable object is still reported + ## rather than silently timing out. + last_scan_error = scan_error + new_attendee_inbox_items = [] if new_attendee_inbox_items or auto_scheduled: break await asyncio.sleep(1) + else: + if last_scan_error is not None: + pytest.fail( + "scheduling data never became readable within 30s; last error: " + f"{last_scan_error!r}" + ) if len(new_attendee_inbox_items) == 0 or auto_scheduled: ## Server implements automatic scheduling. Some servers (e.g. @@ -2441,6 +2687,16 @@ async def test_freebusy(self, scheduling_setup: Any) -> None: ## Just verify it completes without raising; response format varies per server. await coro + ## §1.5 regression: a Principal object (not a pre-resolved vCalAddress) + ## must also work as an attendee. Both the sync and async freebusy + ## tests above only ever passed a resolved address, so the + ## _async_freebusy_request branch that awaits Principal.get_vcal_address() + ## went uncovered — before the fix add_attendee() received an un-awaited + ## coroutine and crashed on attendee_obj.params[...]. + coro = principals[0].freebusy_request(dtstart, dtend, [principals[0]]) + assert asyncio.iscoroutine(coro) + await coro + # ------------------------------------------------------------------ # # Schedule-Tag tests (RFC 6638 section 3.2–3.3) # # These are async counterparts of the sync tests in # @@ -2454,8 +2710,6 @@ async def test_schedule_tag_returned_on_save(self, scheduling_setup: Any) -> Non """Saving a scheduling object must return a Schedule-Tag header. Async counterpart of testScheduleTagReturnedOnSave. - Expected to fail: _async_put() does not yet capture the Schedule-Tag - response header into event.props. """ import uuid @@ -2575,8 +2829,6 @@ async def test_schedule_tag_changes_on_organizer_update(self, scheduling_setup: """Organizer update must advance the Schedule-Tag on the attendee's copy. Async counterpart of testScheduleTagChangesOnOrganizerUpdate. - Expected to fail: _async_load() does not yet capture the Schedule-Tag - response header. """ import uuid @@ -2655,8 +2907,6 @@ async def test_schedule_tag_mismatch_raises_error(self, scheduling_setup: Any) - """save() with a stale Schedule-Tag must raise ScheduleTagMismatchError. Async counterpart of testScheduleTagMismatchRaisesError. - Expected to fail: _async_put() does not yet send If-Schedule-Tag-Match - or raise ScheduleTagMismatchError on a 412 response. """ import uuid @@ -2709,8 +2959,6 @@ async def test_schedule_tag_match_succeeds(self, scheduling_setup: Any) -> None: """save() with the correct Schedule-Tag must succeed. Async counterpart of testScheduleTagMatchSucceeds. - Expected to fail: _async_put() does not yet send If-Schedule-Tag-Match, - so the conditional PUT is not exercised. """ import uuid diff --git a/tests/test_caldav.py b/tests/test_caldav.py index 2df72fbf..2e329735 100644 --- a/tests/test_caldav.py +++ b/tests/test_caldav.py @@ -12,6 +12,7 @@ import logging import os import random +import re import sys import tempfile import time @@ -138,6 +139,44 @@ def _make_client( END:VEVENT """ + +def near_now_ics(ics, days=30, hours=11): + """Return a copy of an ical event string with DTSTART/DTEND shifted to ~now. + + Some servers (e.g. OX App Suite) only return objects within a sliding ~±1 + year window from REPORT-based lookups (ref search.unlimited-time-range), so + an event with a static historic date (the year-2006 dates in ev1/broken_ev1) + is invisible to get_events()/get_event_by_uid() even though it was stored + correctly. Using a near-now date keeps these save-then-search tests + meaningful on such servers. Only plain "DTSTART:"/"DTEND:" properties are + shifted; recurring all-day templates (DTSTART;VALUE=DATE:) are left alone. + """ + start = datetime.now() + timedelta(days=days) + end = start + timedelta(hours=hours) + ics = re.sub(r"DTSTART:[0-9T]+Z?", start.strftime("DTSTART:%Y%m%dT%H%M%SZ"), ics) + ics = re.sub(r"DTEND:[0-9T]+Z?", end.strftime("DTEND:%Y%m%dT%H%M%SZ"), ics) + return ics + + +def next_anniversary_windows(month=11, day=2, hour=17): + """Search windows around the next future anniversary of (month, day). + + A FREQ=YEARLY event (e.g. evr, anchored at 1997-11-02) recurs forever, so + historic search windows like 2008-11 are arbitrary. Servers with a sliding + REPORT window (e.g. OX App Suite, ref search.time-range.event.old-dates) can + only serve time ranges near now, so we search the next future occurrence + instead. Returns (year, narrow_start, narrow_end, wide_end): a ±1-day window + catching one occurrence in `year`, plus a wide_end one year later so that + [narrow_start, wide_end] catches two consecutive occurrences. + """ + now = datetime.now() + year = now.year if (now.month, now.day) <= (month, day) else now.year + 1 + narrow_start = datetime(year, month, day - 1, hour, 0, 0) + narrow_end = datetime(year, month, day + 1, hour, 0, 0) + wide_end = datetime(year + 1, month, day + 1, hour, 0, 0) + return year, narrow_start, narrow_end, wide_end + + ev2 = """BEGIN:VCALENDAR VERSION:2.0 PRODID:-//Example Corp.//CalDAV Client//EN @@ -795,24 +834,40 @@ def testInviteAndRespond(self): ## process scheduling asynchronously, so poll with backoff before giving up. new_attendee_inbox_items = [] auto_scheduled = False + last_scan_error = None for _ in range(30): - new_attendee_inbox_items = [ - item - for item in self.principals[1].schedule_inbox().get_items() - if item.url not in inbox_items - ] - ## Check whether the server auto-scheduled the event directly into - ## the attendee's calendar (server-side automatic scheduling). - ## The event may land in any calendar (e.g. Cyrus uses Default, not the - ## test calendar), so search all attendee calendars for the event UID. - auto_scheduled = any( - event.id == event_uid - for cal in self.principals[1].calendars() - for event in cal.get_events() - ) + try: + ## Correlate by UID: a late METHOD:CANCEL from another scheduling + ## test's teardown can otherwise land here as a stray "new" item + ## (see testAcceptInviteUsernameEmailFallback). + new_attendee_inbox_items = [ + item + for item in self.principals[1].schedule_inbox().get_items() + if item.url not in inbox_items and item.id == event_uid + ] + ## Check whether the server auto-scheduled the event directly into + ## the attendee's calendar (server-side automatic scheduling). + ## The event may land in any calendar (e.g. Cyrus uses Default, not the + ## test calendar), so search all attendee calendars for the event UID. + auto_scheduled = any( + event.id == event_uid + for cal in self.principals[1].calendars() + for event in cal.get_events() + ) + except (error.ResponseError, ValueError) as scan_error: + ## Same as the async twin: a scheduling object that is present but + ## not yet readable is what the poll is here to wait out. + last_scan_error = scan_error + new_attendee_inbox_items = [] if new_attendee_inbox_items or auto_scheduled: break time.sleep(1) + else: + if last_scan_error is not None: + pytest.fail( + "scheduling data never became readable within 30s; last error: " + f"{last_scan_error!r}" + ) if len(new_attendee_inbox_items) == 0 or auto_scheduled: ## Server implements automatic scheduling. Some servers (e.g. @@ -904,12 +959,19 @@ def testAcceptInviteUsernameEmailFallback(self): ) self._auto_scheduled_event_uids.append(saved_event.id) + ## Correlate the inbox item to THIS invite by UID. Picking the first + ## arbitrary "new" item is flaky: deleting an organizer event (e.g. a + ## previous scheduling test's teardown) makes Zimbra deliver a late + ## METHOD:CANCEL for the old UID, which can land in this test's poll + ## window before our own REQUEST arrives. We match on UID (not method) + ## so that a wrongly-delivered non-REQUEST for our own UID still fails + ## the is_invite_request() assertion below rather than being hidden. new_attendee_inbox_items = [] for _ in range(30): new_attendee_inbox_items = [ item for item in self.principals[1].schedule_inbox().get_items() - if item.url not in inbox_items + if item.url not in inbox_items and item.id == saved_event.id ] if new_attendee_inbox_items: break @@ -934,10 +996,9 @@ def testAcceptInviteUsernameEmailFallback(self): ## inbox/outbox? # ------------------------------------------------------------------ # - # Schedule-Tag tests (RFC 6638 section 3.2–3.3) # - # All tests below are expected to FAIL until the implementation is # - # complete. See docs/design/TODO_SCHEDULE_TAG.md and # - # https://github.com/python-caldav/caldav/issues/660 # + # Schedule-Tag tests (RFC 6638 section 3.2–3.3) # + # Implemented in v3.2.0, see # + # https://github.com/python-caldav/caldav/issues/660 # # ------------------------------------------------------------------ # def testScheduleTagReturnedOnSave(self): @@ -1099,6 +1160,7 @@ def _make_ical(summary): _make_ical("Original summary"), [self.principals[0], attendee_addr], ) + assert saved is not None self._auto_scheduled_event_uids.append(uid) ## Find the attendee's copy and record the tag @@ -1258,6 +1320,25 @@ def foo(*a, **kwa): return foo +## HTTP methods that change server state. A "write-delay" server settles each of +## these asynchronously, so we sleep AFTER every such request to let the change +## become visible before the test reads it back (the general, write-side +## counterpart of the search-cache delay, which only delays searches). +_WRITE_HTTP_METHODS = frozenset( + {"PUT", "DELETE", "MKCALENDAR", "MKCOL", "PROPPATCH", "MOVE", "COPY", "POST"} +) + + +def _write_delay_decorator(f, t=10): + def foo(url, method="GET", *a, **kwa): + response = f(url, method, *a, **kwa) + if str(method).upper() in _WRITE_HTTP_METHODS: + time.sleep(t) + return response + + return foo + + class RepeatedFunctionalTestsBaseClass: """This is a class with functional tests (tests that goes through basic functionality and actively communicates with third parties) @@ -1333,6 +1414,12 @@ def setup_method(self): if foo.get("behaviour") == "delay": Calendar._search = Calendar.search Calendar.search = _delay_decorator(Calendar.search, t=foo["delay"]) + foo = self.is_supported("write-delay", dict) + if foo.get("behaviour") == "delay": + ## Every write goes through the client request(); sleep after the + ## write verbs so the asynchronous change has settled before read-back. + ## Instance-level wrap (like rate-limit), torn down with the client. + self.caldav.request = _write_delay_decorator(self.caldav.request, t=foo["delay"]) if False and self.check_compatibility_flag("no-current-user-principal"): self.principal = Principal(client=self.caldav, url=self.server_params["principal_url"]) @@ -1458,10 +1545,30 @@ def _fixCalendar_(self, **kwargs): return self._default_calendar # Pre-processing: set up defaults for name and cal_id + comp_set = kwargs.get("supported_calendar_component_set", []) + # A component-restricted fixture (VTODO-only / VJOURNAL-only) is always + # looked up by cal_id, never by display name. + restricted = bool(comp_set) and "VEVENT" not in comp_set if "name" not in kwargs: if self.cleanup_regime in ("light", "pre"): self._teardownCalendar(cal_id=self.testcal_id) - if not self.is_supported("create-calendar.set-displayname"): + # Only give a display name when the server accepts one and keeps the + # calendar at the requested cal_id URL. On servers that assign a + # different canonical URL when a name is set (create-calendar.stable-url + # unsupported: Zimbra relocates, OX uses an opaque id) the library + # re-points to the canonical URL, so a named fixture would still work - + # but we keep the fixture nameless there so the bulk of the suite keeps + # addressing the fixture by its cal_id (simpler, and avoids the + # name-ambiguity issues below). Component-restricted fixtures also stay + # nameless: they are only ever found by cal_id, and giving them the same + # "Yep" name as the primary fixture would make principal.calendar( + # name="Yep") ambiguous and, on servers enforcing per-principal unique + # calendar names (SOGo), block the primary calendar from being (re)named. + if ( + restricted + or not self.is_supported("create-calendar.set-displayname") + or not self.is_supported("create-calendar.stable-url") + ): kwargs["name"] = None else: kwargs["name"] = "Yep" @@ -1470,10 +1577,9 @@ def _fixCalendar_(self, **kwargs): # that a VTODO-only calendar and a VJOURNAL-only calendar don't share the # same slot and cause MKCALENDAR failures (and wrong-type PUT errors) when # the calendar persists across tests under wipe-calendar cleanup regime. - comp_set = kwargs.get("supported_calendar_component_set", []) if comp_set and "VJOURNAL" in comp_set and "VEVENT" not in comp_set: kwargs["cal_id"] = self.testcal_id + "-journals" - elif comp_set and "VEVENT" not in comp_set: + elif restricted: kwargs["cal_id"] = self.testcal_id + "-tasks" else: kwargs["cal_id"] = self.testcal_id @@ -1493,7 +1599,14 @@ def testCheckCompatibility(self, request) -> None: try: from caldav_server_tester import ServerQuirkChecker except ImportError: - pytest.skip("caldav_server_tester is not installed") + ## Not in the test extra on purpose - see tests/README.md, + ## "testCheckCompatibility does not run in CI". Install a + ## caldav-server-tester checkout that matches this branch and this + ## test starts working. + pytest.skip( + "caldav_server_tester is not installed - install it to run this test, " + "see tests/README.md" + ) # Use pdb debug mode if pytest was run with --pdb, otherwise use logging debug_mode = "pdb" if request.config.option.usepdb else "logging" @@ -1543,37 +1656,11 @@ def testCheckCompatibility(self, request) -> None: fo = checker.features_checked fe = self.caldav.features - ## dotted list expected and observed - ## Snapshot checked features before compact=True calls collapse(), which - ## mutates _server_features by removing subfeatures that collapse into - ## their parent — making tested features look like untested ones. - checked_features = set(fo._server_features.keys()) - observed = fo.dotted_feature_set_list(compact=True) - expected = fe.dotted_feature_set_list(compact=True) - - for feature in set(observed.keys()).union(set(expected.keys())): - observation = fo.is_supported(feature, str) - expectation = fe.is_supported(feature, str) - if "fragile" in (observation, expectation): - continue - if "unknown" in (observation, expectation): - continue - ## Skip features the checker never explicitly tested - - ## the observation would just be a default, not a real result - if feature not in observed and feature not in checked_features: - continue - type_ = fo.find_feature(feature).get("type", "server-feature") - if type_ in ( - "client-feature", - "server-observation", - "tests-behaviour", - "client-hints", - "server-peculiarity", - ): - continue - assert expectation == observation, ( - f"expectation is {expectation}, observation is {observation} for {feature}" - ) + mismatches = fe.compare(fo) + assert not mismatches, "compatibility mismatches:\n" + "\n".join( + f" {m['feature']}: declared {m['expected']!r}, observed {m['observed']!r}" + for m in mismatches + ) def testSupport(self): """ @@ -1588,7 +1675,9 @@ def testSupport(self): def testSchedulingInfo(self): self.skip_unless_support("scheduling.calendar-user-address-set") calendar_user_address_set = self.principal.calendar_user_address_set() + assert calendar_user_address_set is not None me_a_participant = self.principal.get_vcal_address() + assert me_a_participant is not None def testAddOrganizer(self): """add_organizer() sets ORGANIZER from the current principal (issue #524). @@ -1680,7 +1769,9 @@ def testIssue399ChangeAttendeeStatusUsernameEmailFallback(self): def testSchedulingMailboxes(self): self.skip_unless_support("scheduling.mailbox") inbox = self.principal.schedule_inbox() + assert inbox is not None outbox = self.principal.schedule_outbox() + assert outbox is not None def testFindCalendarOwner(self): cal = self._fixCalendar() @@ -1777,9 +1868,9 @@ def testPropfind(self): this is implicitly run by the setup) """ # ResourceType MUST be defined, and SHOULD be returned on a propfind - # for "allprop" if I have the permission to see it. - # So, no ResourceType returned seems like a bug in bedework - self.skip_on_compatibility_flag("propfind_allprop_failure") + # for "allprop" if I have the permission to see it (RFC4918 section 9.1). + # A few servers (bedework, CCS) omit it. + self.skip_unless_support("propfind.allprop.resourcetype") # first a raw xml propfind to the root URL foo = self.caldav.propfind( @@ -1792,12 +1883,15 @@ def testPropfind(self): assert "resourcetype" in to_local(foo.raw) # next, the internal _query_properties, returning an xml tree ... + # (DAV:status is a response-only element, not a queryable property - + # asking for it makes some servers, e.g. CCS, answer 400 - so we query a + # real live property instead and assert on this response, not the first.) foo2 = self.principal._query_properties( [ - dav.Status(), + dav.ResourceType(), ] ) - assert "resourcetype" in to_local(foo.raw) + assert "resourcetype" in to_local(foo2.raw) # TODO: more advanced asserts def testGetCalendarHomeSet(self): @@ -1836,6 +1930,7 @@ def testGetCalendar(self): str_ = str(c) repr_ = repr(c) + assert str_ and repr_ ## Not sure if those asserts make much sense, the main point here is to exercise ## the __str__ and __repr__ methods on the Calendar object. @@ -1848,10 +1943,12 @@ def testGetCalendar(self): assert str(c.url) in repr(c) def _notFound(self): - if self.check_compatibility_flag("non_existing_raises_other"): - return error.DAVError - else: + if self.is_supported("non-existing-raises-not-found"): return error.NotFoundError + else: + ## Some servers answer 403 instead of 404 (e.g. Robur); accept any + ## DAVError in that case. + return error.DAVError def testPrincipal(self): collections = self.principal.get_calendars() @@ -1890,13 +1987,22 @@ def testCreateDeleteCalendar(self): assert len(events) == 0 c.delete() - if self.is_supported("create-calendar.auto"): + if not self.is_supported("create-calendar.auto"): + # Two separate pytest.raises blocks: with both probes in a single + # block, the first one to raise would short-circuit the second, + # leaving it untested (and on auto-create servers get_events() + # doesn't raise at all, so the block passed only because + # get_display_name() happened to 404). with pytest.raises(self._notFound()): self.principal.calendar(cal_id="shouldnotexist").get_events() + with pytest.raises(self._notFound()): self.principal.calendar(cal_id="shouldnotexist").get_display_name() def testChangeAttendeeStatusWithEmailGiven(self): self.skip_unless_support("save-load.event") + ## Some servers (e.g. OX) forbid changing an attendee's PARTSTAT via a + ## direct PUT (403 Forbidden) and require iTIP scheduling instead. + self.skip_unless_support("save-load.mutable.attendee-partstat") c = self._fixCalendar() event = c.add_event( @@ -1950,8 +2056,9 @@ def cleanse(events): ## we're supposed to be working towards a brand new calendar assert len(existing_events) == 0 - # add event - c.add_event(broken_ev1) + # add event (near-now date so it stays visible to REPORT-based lookups + # on sliding-window servers; see near_now_ics) + c.add_event(near_now_ics(broken_ev1)) # c.get_events() should give a full list of events events = cleanse(c.get_events()) @@ -1963,8 +2070,14 @@ def cleanse(events): assert len(events2) == 1 assert events2[0].url == events[0].url - if self.is_supported("create-calendar") and self.is_supported( - "create-calendar.set-displayname" + if ( + self.is_supported("create-calendar") + and self.is_supported("create-calendar.set-displayname") + ## _fixCalendar only gives the calendar a display name ("Yep") when + ## the server also keeps the URL stable; on servers that assign a + ## different canonical URL when a name is set (Zimbra, OX) the fixture + ## is created nameless. + and self.is_supported("create-calendar.stable-url") ): ## We should be able to access the calender through the name c2 = self.principal.calendar(name="Yep") @@ -1973,22 +2086,81 @@ def cleanse(events): self.is_supported("delete-calendar") or self.is_supported("delete-calendar", str) == "fragile" ): - assert c2.url == c.url + ## A name lookup may return a different (canonical) calendar URL + ## than the one we created it at on servers that don't preserve + ## the URL (e.g. OX exposes the calendar under an internal + ## cal://0/NNN id); see save-load.stable-url. + if self.is_supported("save-load.stable-url"): + assert c2.url == c.url events2 = cleanse(c2.get_events()) assert len(events2) == 1 assert events2[0].url == events[0].url # add another event, it should be doable without having premade ICS + _dt = datetime.now() + timedelta(days=31) ev2 = c.add_event( - dtstart=datetime(2015, 10, 10, 8, 7, 6), + dtstart=_dt, summary="This is a test event", - dtend=datetime(2016, 10, 10, 9, 8, 7), + dtend=_dt + timedelta(hours=1), uid="ctuid1", ) events = c.get_events() assert len(events) == len(existing_events) + 2 ev2.delete() + def testNamedCalendarUrlIsUsable(self): + """A calendar created WITH a display name must be fully usable by URL. + + Exercises create-calendar.stable-url: on servers that assign a different + canonical URL when a name is set (unsupported - Zimbra relocates the + collection to a display-name-derived path, OX uses an opaque cal://0/NNN), + Calendar._create() must discover and adopt the canonical URL so that + object operations addressed via the returned calendar's .url resolve. + Regression guard for the Zimbra "event 404s on the cal_id URL even though + the collection is reachable there" quirk. On stable servers the calendar + simply stays at the requested cal_id and the test still passes. + """ + self.skip_unless_support("create-calendar") + self.skip_unless_support("create-calendar.set-displayname") + self.skip_unless_support("save-load.event") + + cal_id = self.testcal_id + "-named-url" + name = "csc-repoint-" + str(uuid.uuid4()) + self._teardownCalendar(cal_id=cal_id) + cal = self.principal.make_calendar(cal_id=cal_id, name=name) + try: + ## the display name stuck (set-displayname is supported) + assert cal.get_display_name() == name + + ## store an event and look it up BY URL through the returned calendar; + ## cal.url must point at the address that actually resolves. + uid = "csc-repoint-" + str(uuid.uuid4()) + _dt = datetime.now() + timedelta(days=20) + stored = cal.add_event( + dtstart=_dt, + dtend=_dt + timedelta(hours=1), + summary="re-point url test", + uid=uid, + ) + + ## REPORT-based lookup may lag on indexing servers (e.g. OX); retry. + fetched = None + for _ in range(10): + try: + fetched = cal.event_by_url(stored.url) + break + except error.NotFoundError: + time.sleep(1) + assert fetched is not None, "event not retrievable by URL on the created calendar" + assert fetched.url == stored.url + assert fetched.icalendar_component["uid"] == uid + finally: + try: + cal.delete() + except Exception: + logging.warning(f"unable to delete calendar {cal.url} with name {name}") + self._teardownCalendar(cal_id=cal_id) + @pytest.mark.parametrize("klass", ["Calendar", "Event"]) def testCreateEventFromiCal(self, klass): c = self._fixCalendar() @@ -2012,6 +2184,7 @@ def testCreateEventFromiCal(self, klass): ## Parametrized test - we should test both with the Calendar object and the Event object obj = {"Calendar": icalcal, "Event": icalevent}[klass] event = c.add_event(obj) + assert event is not None events = c.get_events() assert len([x for x in events if x.icalendar_component["uid"] == "ctuid1"]) == 1 @@ -2026,6 +2199,7 @@ def testAlarm(self): alarm_trigger=timedelta(minutes=-15), alarm_action="AUDIO", ) + assert ev is not None self.skip_unless_support("search.time-range.alarm") @@ -2096,7 +2270,7 @@ def testObjectBySyncToken(self): if self.is_supported("save-load.todo.mixed-calendar"): objcnt += len(c.get_todos()) objcnt += len(c.get_events()) - obj = c.add_event(ev1) + obj = c.add_event(near_now_ics(ev1)) objcnt += 1 if self.is_supported("save-load.event.recurrences"): c.add_event(evr) @@ -2178,7 +2352,7 @@ def testObjectBySyncToken(self): ## ADDING yet another object ... and it should also be reported if is_time_based: time.sleep(1) - obj3 = c.add_event(ev3) + c.add_event(near_now_ics(ev3)) if is_time_based: time.sleep(1) my_changed_objects = c.get_objects_by_sync_token(sync_token=my_changed_objects.sync_token) @@ -2242,7 +2416,7 @@ def testSync(self): if self.is_supported("save-load.todo.mixed-calendar"): objcnt += len(c.get_todos()) objcnt += len(c.get_events()) - obj = c.add_event(ev1) + obj = c.add_event(near_now_ics(ev1)) objcnt += 1 if self.is_supported("save-load.event.recurrences"): c.add_event(evr) @@ -2261,6 +2435,25 @@ def testSync(self): assert my_objects.sync_token != "" assert len(list(my_objects)) == objcnt + stable_url = self.is_supported("save-load.stable-url") + + def synced_match(o): + """Return the synced object corresponding to o, or None. + + objects_by_url() is keyed by the server-reported URL, which on + servers that don't preserve the PUT URL (e.g. OX; see + save-load.stable-url) differs from o.url - so fall back to matching + by UID there. + """ + synced = my_objects.objects_by_url() + if stable_url: + return synced.get(o.url) + uid = o.icalendar_component["uid"] + return next( + (cand for cand in synced.values() if cand.icalendar_component["uid"] == uid), + None, + ) + if is_time_based: time.sleep(1) @@ -2287,13 +2480,13 @@ def testSync(self): if not is_fragile: assert len(list(updated)) == 1 assert len(list(deleted)) == 0 - assert "foobar" in my_objects.objects_by_url()[obj.url].data + assert "foobar" in synced_match(obj).data if is_time_based: time.sleep(1) ## ADDING yet another object ... and it should also be reported - obj3 = c.add_event(ev3) + obj3 = c.add_event(near_now_ics(ev3)) if is_time_based: time.sleep(1) @@ -2302,7 +2495,7 @@ def testSync(self): if not is_fragile: assert len(list(updated)) == 1 assert len(list(deleted)) == 0 - assert obj3.url in my_objects.objects_by_url() + assert synced_match(obj3) is not None self.skip_unless_support("sync-token.delete") @@ -2317,7 +2510,7 @@ def testSync(self): if not is_fragile: assert len(list(updated)) == 0 assert len(list(deleted)) == 1 - assert obj.url not in my_objects.objects_by_url() + assert synced_match(obj) is None if is_time_based: time.sleep(1) @@ -2336,13 +2529,18 @@ def testLoadEvent(self): self._teardownCalendar(cal_id=self.testcal_id2) c1 = self._fixCalendar(name="Yep", cal_id=self.testcal_id) c2 = self._fixCalendar(name="Yapp", cal_id=self.testcal_id2) + assert c2 is not None - e1_ = c1.add_event(ev1) + e1_ = c1.add_event(near_now_ics(ev1)) if not self.check_compatibility_flag("event_by_url_is_broken"): e1_.load() e1 = c1.get_events()[0] if not self.check_compatibility_flag("event_by_url_is_broken"): - assert e1.url == e1_.url + ## e1 came from a search and may carry a different (canonical) URL + ## than the PUT URL on servers that don't preserve it (e.g. OX); see + ## save-load.stable-url. + if self.is_supported("save-load.stable-url"): + assert e1.url == e1_.url e1.load() if self.cleanup_regime == "post": self._teardownCalendar(cal_id=self.testcal_id) @@ -2361,10 +2559,11 @@ def testCopyEvent(self): assert not len(c1.get_events()) assert not len(c2.get_events()) - e1_ = c1.add_event(ev1) + e1_ = c1.add_event(near_now_ics(ev1)) + assert e1_ is not None e1 = c1.get_events()[0] - if not self.check_compatibility_flag("duplicates_not_allowed"): + if self.is_supported("save.duplicate-event"): ## Duplicate the event in the same calendar, with new uid e1_dup = e1.copy() e1_dup.save() @@ -2392,10 +2591,10 @@ def testCopyEvent(self): ## this makes no sense, there won't be any duplication e1_dup2 = e1.copy(keep_uid=True) e1_dup2.save() - if self.check_compatibility_flag("duplicates_not_allowed"): - assert len(c1.get_events()) == 1 - else: + if self.is_supported("save.duplicate-event"): assert len(c1.get_events()) == 2 + else: + assert len(c1.get_events()) == 1 if self.cleanup_regime == "post": self._teardownCalendar(cal_id=self.testcal_id) @@ -2408,8 +2607,9 @@ def testCreateCalendarAndEventFromVobject(self): ## in case the calendar is reused cnt = len(c.get_events()) - # add event from vobject data - ve1 = vobject.readOne(ev1) + # add event from vobject data (near-now date so it stays visible to + # REPORT-based lookups on sliding-window servers; see near_now_ics) + ve1 = vobject.readOne(near_now_ics(ev1)) c.add_event(ve1) cnt += 1 @@ -2630,17 +2830,20 @@ def cleanse(tasks): dtstart=datetime.now() + timedelta(minutes=15), uid="test2", ) + assert t2 is not None t3 = c.add_todo( summary="3 task future due", due=datetime.now() + timedelta(hours=15), dtstart=datetime(2022, 12, 11, 10, 9, 8), uid="test3", ) + assert t3 is not None t4 = c.add_todo( summary="4 task priority is set to nine which is the lowest", priority=9, uid="test4", ) + assert t4 is not None t5 = c.add_todo( summary="5 task status is set to COMPLETED and this will disappear from the ordinary todo search", status="COMPLETED", @@ -2652,6 +2855,7 @@ def cleanse(tasks): categories="home,garden,sunshine", uid="test6", ) + assert t6 is not None def check_order(tasks, order): assert [str(x.icalendar_component["uid"]) for x in tasks] == [ @@ -2693,9 +2897,12 @@ def testSearchTodos(self): pre_cnt = len(c.get_todos()) t1 = c.add_todo(todo) + assert t1 is not None t2 = c.add_todo(todo2) + assert t2 is not None t3 = c.add_todo(todo3) t4 = c.add_todo(todo4) + assert t4 is not None t5 = c.add_todo(todo5) t6 = c.add_todo(todo6) @@ -3049,6 +3256,7 @@ def testSetDue(self): uid="ctuid4", parent=[some_todo.id], ) + assert child is not None ## This should still work out (set the children due to some time before the parents due) ## (The fact that we now have a child does not affect it anyhow) @@ -3083,6 +3291,7 @@ def testCreateJournalListAndJournalEntry(self): description="A quick birth, in the middle of the night", uid="ctuid1", ) + assert j2 is not None assert len(c.get_journals()) == 2 assert len(c.search(journal=True)) == 2 todos = c.get_todos() @@ -3124,6 +3333,7 @@ def testCreateTaskListAndTodo(self): assert len(todos2) == 1 t3 = c.add_todo(summary="mop the floor", categories=["housework"], priority=4, uid="ctuid1") + assert t3 is not None assert len(c.get_todos()) == 2 # adding a todo without a UID, it should also work (library will add the missing UID) @@ -3265,9 +3475,12 @@ def testTodoDatesearch(self): # add todo-item t1 = c.add_todo(todo) t2 = c.add_todo(todo2) + assert t2 is not None t3 = c.add_todo(todo3) + assert t3 is not None t4 = c.add_todo(todo4) t5 = c.add_todo(todo5) + assert t5 is not None t6 = c.add_todo(todo6) todos = c.get_todos() assert len(todos) == 6 @@ -3335,8 +3548,8 @@ def testTodoDatesearch(self): ) if not self.is_supported("search.recurrences.includes-implicit.todo"): foo -= 1 ## t6 will not be returned - if self.check_compatibility_flag( - "vtodo_datesearch_nodtstart_task_is_skipped" + if not self.is_supported( + "search.time-range.todo.no-dtstart" ) or self.check_compatibility_flag( "vtodo_datesearch_nodtstart_task_is_skipped_in_closed_date_range" ): @@ -3369,10 +3582,22 @@ def testTodoDatesearch(self): todos2 = c.search(start=datetime(2025, 4, 14), todo=True, include_completed=True) todos3 = c.search(start=datetime(2025, 4, 14), todo=True) + ## On a compliant server t1/t4/t6 are returned by an open-ended future + ## search, so we get Todo objects back. Some servers legitimately return + ## nothing here: they skip no-dtstart todos (t1/t4) and don't carry the + ## recurring todo (t6) into the future - e.g. Stalwart, which skips + ## no-dtstart todos and only marks implicit-recurrence todos "fragile". + ## The presence/absence of each todo is verified by the urls_found logic + ## below; here we type-check whatever did come back -- every object, + ## not just the first, and unconditionally: guarding on non-emptiness + ## would make the check disappear silently on exactly the servers + ## where it is most likely to catch something. if self.is_supported("search.time-range.open.end"): - assert isinstance(todos1[0], Todo) - assert isinstance(todos2[0], Todo) - assert isinstance(todos3[0], Todo) + for todos in (todos1, todos2, todos3): + assert all(isinstance(x, Todo) for x in todos), ( + f"search returned non-Todo objects: " + f"{[type(x).__name__ for x in todos if not isinstance(x, Todo)]}" + ) ## * t6 should be returned, as it's a yearly task spanning over 2025 ## * t1 should probably be returned, as it has no due date set and hence @@ -3385,8 +3610,8 @@ def testTodoDatesearch(self): urls_found = set(urls_found) if self.is_supported("search.recurrences.includes-implicit.todo", accept_fragile=True): urls_found.discard(t6.url) - if not self.check_compatibility_flag( - "vtodo_datesearch_nodtstart_task_is_skipped" + if self.is_supported( + "search.time-range.todo.no-dtstart" ) and not self.check_compatibility_flag("vtodo_datesearch_notime_task_is_skipped"): urls_found.discard(t4.url) if self.check_compatibility_flag("vtodo_no_due_infinite_duration"): @@ -3412,6 +3637,135 @@ def testSearchWithoutCompType(self): assert len(objects) == 2 assert set([type(x).__name__ for x in objects]) == {"Todo", "Event"} + def testSearchWithoutCompTypeWithDateRange(self): + """Test for https://github.com/python-caldav/caldav/issues/681 + + A time-range search that does NOT specify a component type must work + even on SabreDAV-based servers (Baikal, Nextcloud, ...) which - correctly + per RFC4791 section 9.7 - reject a CALDAV:time-range placed directly under + the VCALENDAR comp-filter with HTTP 400. The library works around this by + splitting the search into one query per component type + (search.time-range.comp-type-optional being unsupported). + + The search is run twice: once with the server's real feature + configuration, and once with search.time-range.comp-type-optional forced + to "supported". The forced run makes the library optimistically send the + comp-type-less time-range query that SabreDAV rejects, exercising the + reactive 400-fallback. Without that fallback the forced run fails on + Baikal. + """ + self.skip_unless_support("search.time-range.event") + cal = self._fixCalendar() + + ## Near-future dates, to steer clear of servers that restrict old-date + ## time-range searches. + now = datetime.now(timezone.utc) + dtstart = now + timedelta(days=1) + dtend = dtstart + timedelta(hours=1) + uid = "issue681-" + uuid.uuid4().hex + ical = ( + "BEGIN:VCALENDAR\r\n" + "VERSION:2.0\r\n" + "PRODID:-//python-caldav//issue681 test//EN\r\n" + "BEGIN:VEVENT\r\n" + f"UID:{uid}\r\n" + f"DTSTAMP:{now.strftime('%Y%m%dT%H%M%SZ')}\r\n" + f"DTSTART:{dtstart.strftime('%Y%m%dT%H%M%SZ')}\r\n" + f"DTEND:{dtend.strftime('%Y%m%dT%H%M%SZ')}\r\n" + "SUMMARY:issue 681 comp-type-less time-range search\r\n" + "END:VEVENT\r\n" + "END:VCALENDAR\r\n" + ) + cal.save_event(ical) + + start = now + end = now + timedelta(days=2) + + def _assert_event_found(): + ## must not raise (this is the crux of issue #681) and must find the event + objects = cal.search(start=start, end=end) + assert [o for o in objects if uid in o.data], ( + "comp-type-less time-range search did not return the event" + ) + + ## Run 1: the server's real feature configuration (proactive comp-type split) + _assert_event_found() + + ## Determine how this server reacts to the raw comp-type-less time-range + ## query. Only SabreDAV-style servers (Baikal, Nextcloud) reject it with a + ## ReportError (HTTP 400) - that is the case the reactive fallback (issue + ## #681 item 4) is designed to recover from. Others return nothing, or a + ## different error (e.g. Cyrus may answer 403), where forcing the feature on + ## is an unrecoverable misconfiguration not worth asserting on. + try: + cal.search(start=start, end=end, compatibility_workarounds=False) + raw_report_error = False + except error.ReportError: + raw_report_error = True + except error.DAVError: + raw_report_error = False + + ## Run 2 (only meaningful where the raw query raises a ReportError): force + ## search.time-range.comp-type-optional ON and verify the reactive fallback + ## recovers and still finds the event. + if raw_report_error: + features = self.caldav.features + key = "search.time-range.comp-type-optional" + had_key = key in features._server_features + saved = features._server_features.get(key) + features.set_feature(key, {"support": "full"}) + try: + objects = cal.search(start=start, end=end) + assert [o for o in objects if uid in o.data], ( + "reactive fallback did not recover the comp-type-less time-range search" + ) + finally: + if had_key: + features._server_features[key] = saved + else: + features._server_features.pop(key, None) + + def testSearchWithoutCompTypeWithCategory(self): + """Test for https://github.com/python-caldav/caldav/issues/681 + + A property filter (here CATEGORIES) without a component type must work. + Placed directly under the VCALENDAR comp-filter the prop-filter targets + VCALENDAR's own properties, which lack component properties like + CATEGORIES, so servers (Xandikos, SabreDAV, ...) match nothing. The + library works around this by splitting the search into one query per + component type (search.text.comp-type-optional being unsupported). + """ + self.skip_unless_support("search.text.category") + cal = self._fixCalendar() + + category = "issue681cat" + uuid.uuid4().hex[:8] + uid = "issue681cat-" + uuid.uuid4().hex + ical = ( + "BEGIN:VCALENDAR\r\n" + "VERSION:2.0\r\n" + "PRODID:-//python-caldav//issue681 test//EN\r\n" + "BEGIN:VEVENT\r\n" + f"UID:{uid}\r\n" + f"DTSTAMP:{datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')}\r\n" + f"DTSTART:{(datetime.now(timezone.utc) + timedelta(days=1)).strftime('%Y%m%dT%H%M%SZ')}\r\n" + f"DTEND:{(datetime.now(timezone.utc) + timedelta(days=1, hours=1)).strftime('%Y%m%dT%H%M%SZ')}\r\n" + "SUMMARY:issue 681 comp-type-less category search\r\n" + f"CATEGORIES:{category}\r\n" + "END:VEVENT\r\n" + "END:VCALENDAR\r\n" + ) + cal.save_event(ical) + + ## The proactive per-component-type split is the only safe fix here: unlike + ## the time-range case (where SabreDAV returns HTTP 400, which the reactive + ## fallback can catch), servers silently return nothing for a prop-filter + ## under VCALENDAR, so there is no error to recover from. Hence we only + ## verify the default (proactive) behaviour. + objects = cal.search(category=category) + assert [o for o in objects if uid in o.data], ( + "comp-type-less category search did not return the event" + ) + def testTodoCompletion(self): """ Will check that todo-items can be completed and deleted @@ -3424,6 +3778,7 @@ def testTodoCompletion(self): # add todo-items t1 = c.add_todo(todo) + assert t1 is not None t2 = c.add_todo(todo2) t3 = c.add_todo(todo3, status="NEEDS-ACTION") @@ -3530,7 +3885,10 @@ def testUtf8Event(self): c = self._fixCalendar(name="Yølp", cal_id=self.testcal_id) # add event - e1 = c.add_event(ev1.replace("Bastille Day Party", "Bringebærsyltetøyfestival")) + e1 = c.add_event( + near_now_ics(ev1).replace("Bastille Day Party", "Bringebærsyltetøyfestival") + ) + assert e1 is not None # fetch it back events = c.get_events() @@ -3555,7 +3913,10 @@ def testUnicodeEvent(self): c = self._fixCalendar(name="Yølp", cal_id=self.testcal_id) # add event - e1 = c.add_event(to_str(ev1.replace("Bastille Day Party", "Bringebærsyltetøyfestival"))) + e1 = c.add_event( + to_str(near_now_ics(ev1).replace("Bastille Day Party", "Bringebærsyltetøyfestival")) + ) + assert e1 is not None # c.get_events() should give a full list of events events = c.get_events() @@ -3566,6 +3927,9 @@ def testUnicodeEvent(self): def testSetCalendarProperties(self): self.skip_unless_support("create-calendar.set-displayname") + ## This test expects the fixture's display name ("Yep") and renames the + ## calendar in place; both require the URL to stay put when a name is set. + self.skip_unless_support("create-calendar.stable-url") self.skip_unless_support("delete-calendar") c = self._fixCalendar() @@ -3595,58 +3959,72 @@ def testSetCalendarProperties(self): if not self.is_supported("delete-calendar"): raise - c.set_properties( - [ - dav.DisplayName("hooray"), - ] - ) - props = c.get_properties( - [ - dav.DisplayName(), - ] - ) - assert props[dav.DisplayName.tag] == "hooray" - - ## calendar color and calendar order are extra properties not - ## described by RFC5545, but anyway supported by quite some - ## server implementations - if self.check_compatibility_flag("calendar_color"): - props = c.get_properties( - [ - ical.CalendarColor(), - ] - ) - assert props[ical.CalendarColor.tag] != "sort of blueish" - c.set_properties( - [ - ical.CalendarColor("blue"), - ] - ) - props = c.get_properties( - [ - ical.CalendarColor(), - ] - ) - assert props[ical.CalendarColor.tag] == "blue" - if self.check_compatibility_flag("calendar_order"): - props = c.get_properties( - [ - ical.CalendarOrder(), - ] - ) - assert props[ical.CalendarOrder.tag] != "-434" + try: c.set_properties( [ - ical.CalendarOrder("12"), + dav.DisplayName("hooray"), ] ) props = c.get_properties( [ - ical.CalendarOrder(), + dav.DisplayName(), ] ) + assert props[dav.DisplayName.tag] == "hooray" + finally: + ## Restore the fixture's canonical display name. Under the + ## wipe-calendar cleanup regime the calendar is reused (never + ## deleted) between tests, so a lingering "hooray" would make this + ## test non-idempotent (the next run reads "hooray", not "Yep") and, + ## on servers enforcing per-principal unique calendar names (SOGo), + ## block other calendars from taking the "hooray" name. + try: + c.set_properties([dav.DisplayName("Yep")]) + except error.PropsetError: + ## Best-effort cleanup only: the assertion of interest has + ## already run above. Some servers reject setting the display + ## name (PropsetError); if so there's nothing to restore and + ## nothing actionable to do here, so swallow it silently. + pass + + ## calendar color and calendar order are extra properties not + ## described by RFC5545, but anyway supported by quite some server + ## implementations. How they behave is probed in detail by the + ## server-tester (calendar-color / calendar-color.hex / + ## calendar-order); here we mirror the core of what it asserts. + ## + ## The colour cannot be compared against the value that was set: a + ## server marked `full` is allowed to normalise a colour name to its + ## hex form. But asserting only that *something* came back would + ## pass even if set_properties() were a complete no-op, so instead we + ## set two different values and require two different values back. + if self.is_supported("calendar-color"): + self._assert_property_tracks_input(c, ical.CalendarColor, "blue", "green") + if self.is_supported("calendar-color.hex"): + self._assert_property_tracks_input(c, ical.CalendarColor, "#FF0000FF", "#00FF00FF") + if self.is_supported("calendar-order"): + c.set_properties([ical.CalendarOrder("12")]) + props = c.get_properties([ical.CalendarOrder()]) assert props[ical.CalendarOrder.tag] == "12" + def _assert_property_tracks_input(self, cal, element, value1, value2): + """Assert that a collection property actually stores what it is given. + + Set two different values in turn and require two different values + back. This distinguishes a property that works (even one the server + normalises) from one that is read-only or silently dropped, without + assuming the stored form equals the form that was set. + """ + cal.set_properties([element(value1)]) + got1 = cal.get_properties([element()])[element.tag] + cal.set_properties([element(value2)]) + got2 = cal.get_properties([element()])[element.tag] + assert got1, f"{element.tag}: nothing was stored for {value1!r}" + assert got1 != got2, ( + f"{element.tag}: {got1!r} comes back for both {value1!r} and {value2!r} - " + f"the property is read-only, or set_properties() did nothing" + ) + def testLookupEvent(self): """ Makes sure we can add events and look them up by URL and ID @@ -3656,8 +4034,9 @@ def testLookupEvent(self): c = self._fixCalendar() assert c.url is not None - # add event - e1 = c.add_event(ev1) + # add event, with a near-now date so it stays visible to REPORT-based + # lookups on sliding-window servers (see near_now_ics()). + e1 = c.add_event(near_now_ics(ev1)) assert e1.url is not None # Verify that we can look it up, both by URL and by ID @@ -3668,7 +4047,15 @@ def testLookupEvent(self): # look up by UID e3 = c.get_event_by_uid("20010712T182145Z-123401@example.com") assert e3.vobject_instance.vevent.uid == e1.vobject_instance.vevent.uid - assert e3.url == e1.url + if self.is_supported("save-load.stable-url"): + assert e3.url == e1.url + else: + ## The server reports the object under a different (canonical) URL + ## than the one we stored it at (e.g. OX App Suite). We can't compare + ## URLs, but we can confirm the looked-up URL is a real, fetchable + ## resource holding the same event. + e3.load() + assert e3.icalendar_component["uid"] == e1.icalendar_component["uid"] e4 = Event(client=self.caldav, url=e1.url) e4.load() @@ -3686,17 +4073,21 @@ def testCreateOverwriteDeleteEvent(self): c = self._fixCalendar() assert c.url is not None + ## near-now date so the event stays visible to REPORT-based lookups on + ## sliding-window servers (e.g. OX); see near_now_ics + ev1_now = near_now_ics(ev1) + # attempts on updating/overwriting a non-existing event should fail: with pytest.raises(error.ConsistencyError): - c.add_event(ev1, no_create=True) + c.add_event(ev1_now, no_create=True) # no_create and no_overwrite is mutually exclusive, this will always # raise an error (unless the ical given is blank) with pytest.raises(error.ConsistencyError): - c.add_event(ev1, no_create=True, no_overwrite=True) + c.add_event(ev1_now, no_create=True, no_overwrite=True) # add event - e1 = c.add_event(ev1) + e1 = c.add_event(ev1_now) todo_ok = self.is_supported("save-load.todo.mixed-calendar") if todo_ok: @@ -3706,19 +4097,30 @@ def testCreateOverwriteDeleteEvent(self): assert t1.url is not None if not self.check_compatibility_flag("event_by_url_is_broken"): assert c.event_by_url(e1.url).url == e1.url - assert c.get_event_by_uid(e1.id).url == e1.url + ## get_event_by_uid may return a different (canonical) URL than the PUT + ## URL on servers that don't preserve it (e.g. OX); see save-load.stable-url + e_by_uid = c.get_event_by_uid(e1.id) + if self.is_supported("save-load.stable-url"): + assert e_by_uid.url == e1.url + else: + assert e_by_uid.icalendar_component["uid"] == e1.icalendar_component["uid"] no_create = True ## add same event again. As it has same uid, it should be overwritten - ## (but some calendars may throw a "409 Conflict") - if self.is_supported("save-load.mutable"): - e2 = c.add_event(ev1) + ## (but some calendars may throw a "409 Conflict"). Overwriting via a + ## fresh PUT without an If-Match etag is gated on save-load.mutable.if-match-optional: + ## OX enforces optimistic concurrency and rejects such a PUT with 409 + ## (etag-conditional save() still works, so save-load.mutable stays full). + if self.is_supported("save-load.mutable") and self.is_supported( + "save-load.mutable.if-match-optional" + ): + e2 = c.add_event(ev1_now) if todo_ok: t2 = c.add_todo(todo) ## add same event with "no_create". Should work like a charm. - e2 = c.add_event(ev1, no_create=no_create) + e2 = c.add_event(ev1_now, no_create=no_create) if todo_ok: t2 = c.add_todo(todo, no_create=no_create) @@ -3740,7 +4142,7 @@ def testCreateOverwriteDeleteEvent(self): ## "no_overwrite" should throw a ConsistencyError. with pytest.raises(error.ConsistencyError): - c.add_event(ev1, no_overwrite=True) + c.add_event(ev1_now, no_overwrite=True) if todo_ok: with pytest.raises(error.ConsistencyError): c.add_todo(todo, no_overwrite=True) @@ -3750,15 +4152,13 @@ def testCreateOverwriteDeleteEvent(self): if todo_ok: t1.delete() - if self.check_compatibility_flag("non_existing_raises_other"): - expected_error = error.DAVError - else: - expected_error = error.NotFoundError - # Verify that we can't look it up, both by URL and by ID with pytest.raises(self._notFound()): c.event_by_url(e1.url) - if self.is_supported("save-load.mutable"): + ## e2 only exists if the put-overwrite block above ran + if self.is_supported("save-load.mutable") and self.is_supported( + "save-load.mutable.if-match-optional" + ): with pytest.raises(self._notFound()): c.event_by_url(e2.url) if not self.check_compatibility_flag("event_by_url_is_broken"): @@ -3876,20 +4276,24 @@ def testRecurringDateSearch(self): self.skip_unless_support("search.recurrences.includes-implicit.event") c = self._fixCalendar() - # evr is a yearly event starting at 1997-11-02 + # evr is a yearly event starting at 1997-11-02. We search the next + # future Nov-2 anniversary rather than a fixed historic year, so that + # sliding-window servers (e.g. OX) can serve the time range. + year, narrow_start, narrow_end, wide_end = next_anniversary_windows() e = c.add_event(evr) + assert e is not None - ## Without "expand", we should still find it when searching over 2008 ... + ## Without "expand", we should still find it when searching the anniversary with pytest.deprecated_call(): r = c.date_search( - datetime(2008, 11, 1, 17, 00, 00), - datetime(2008, 11, 3, 17, 00, 00), + narrow_start, + narrow_end, expand=False, ) r2 = c.search( event=True, - start=datetime(2008, 11, 1, 17, 00, 00), - end=datetime(2008, 11, 3, 17, 00, 00), + start=narrow_start, + end=narrow_end, expand=False, ) assert len(r) == 1 @@ -3899,46 +4303,46 @@ def testRecurringDateSearch(self): ## legacy method name with pytest.deprecated_call(): r1 = c.date_search( - datetime(2008, 11, 1, 17, 00, 00), - datetime(2008, 11, 3, 17, 00, 00), + narrow_start, + narrow_end, expand=True, ) ## server expansion, with client side fallback r2 = c.search( event=True, - start=datetime(2008, 11, 1, 17, 00, 00), - end=datetime(2008, 11, 3, 17, 00, 00), + start=narrow_start, + end=narrow_end, expand=True, ) ## r3 was client-side expansion, but this is the default now ## server side expansion r4 = c.search( event=True, - start=datetime(2008, 11, 1, 17, 00, 00), - end=datetime(2008, 11, 3, 17, 00, 00), + start=narrow_start, + end=narrow_end, server_expand=True, ) assert len(r1) == 1 assert len(r2) == 1 assert r1[0].data.count("END:VEVENT") == 1 assert r2[0].data.count("END:VEVENT") == 1 - ## due to expandation, the DTSTART should be in 2008 - assert r1[0].data.count("DTSTART;VALUE=DATE:2008") == 1 - assert r2[0].data.count("DTSTART;VALUE=DATE:2008") == 1 + ## due to expandation, the DTSTART should be in the anniversary year + assert r1[0].data.count(f"DTSTART;VALUE=DATE:{year}") == 1 + assert r2[0].data.count(f"DTSTART;VALUE=DATE:{year}") == 1 if self.is_supported("search.recurrences.expanded.event"): - assert r4[0].data.count("DTSTART;VALUE=DATE:2008") == 1 + assert r4[0].data.count(f"DTSTART;VALUE=DATE:{year}") == 1 ## With expand=True and searching over two recurrences ... with pytest.deprecated_call(): r1 = c.date_search( - datetime(2008, 11, 1, 17, 00, 00), - datetime(2009, 11, 3, 17, 00, 00), + narrow_start, + wide_end, expand=True, ) r2 = c.search( event=True, - start=datetime(2008, 11, 1, 17, 00, 00), - end=datetime(2009, 11, 3, 17, 00, 00), + start=narrow_start, + end=wide_end, expand=True, ) @@ -3976,6 +4380,7 @@ def testRecurringDateWithExceptionSearch(self): # evr2 is a bi-weekly event starting 2024-04-11 ## It has an exception, edited summary for recurrence id 20240425T123000Z e = c.add_event(evr2) + assert e is not None rc = c.search( start=datetime(2024, 3, 31, 0, 0), @@ -4052,30 +4457,45 @@ def testEditSingleRecurrence(self): cal = self._fixCalendar() + ## Anchor the daily recurring event a few days in the future so servers + ## with a sliding REPORT window / no old-date support (e.g. CCS, ref + ## search.time-range.event.old-dates) can still serve the time ranges. + ## The integer passed to search()/summary_on() is a day offset from this + ## anchor day; the values just need to be distinct future days. + base = (datetime.now() + timedelta(days=2)).replace( + hour=8, minute=7, second=6, microsecond=0 + ) + ## Create a daily recurring event cal.add_event( uid="test1", summary="daily test", - dtstart=datetime(2015, 1, 1, 8, 7, 6), - dtend=datetime(2015, 1, 1, 9, 7, 6), + dtstart=base, + dtend=base + timedelta(hours=1), rrule={"FREQ": "DAILY"}, ) - def search(month): + def day_start(offset): + return (base + timedelta(days=offset)).replace( + hour=0, minute=0, second=0, microsecond=0 + ) + + def search(offset): """ - Internal function to find one recurrence object + Internal function to find one recurrence object - the occurrence on + the day `offset` days after the event's anchor day. """ recurrence = cal.search( event=True, - start=datetime(2015, month, 1), - end=datetime(2015, month, 2), + start=day_start(offset), + end=day_start(offset) + timedelta(days=1), expand=True, ) assert len(recurrence) == 1 return recurrence[0] - def summary_by_month(month): - return search(month).icalendar_component["summary"] + def summary_on(offset): + return search(offset).icalendar_component["summary"] ## Search for a recurrence recurrence = search(7) @@ -4085,52 +4505,56 @@ def summary_by_month(month): recurrence.save() ## Only one day should be affected - assert summary_by_month(6) == "daily test" - assert summary_by_month(7) == "half a year of daily testing" - assert summary_by_month(8) == "daily test" + assert summary_on(6) == "daily test" + assert summary_on(7) == "half a year of daily testing" + assert summary_on(8) == "daily test" ## let's try to set several recurrence exceptions recurrence = search(2) recurrence.icalendar_component["summary"] = "one month of daily testing" recurrence.save() - assert summary_by_month(1) == "daily test" - assert summary_by_month(2) == "one month of daily testing" - assert summary_by_month(7) == "half a year of daily testing" + assert summary_on(1) == "daily test" + assert summary_on(2) == "one month of daily testing" + assert summary_on(7) == "half a year of daily testing" ## Changing any of the exceptions should also work recurrence = search(7) recurrence.icalendar_component["summary"] = "six months of daily testing" recurrence.save() - assert summary_by_month(7) == "six months of daily testing" + assert summary_on(7) == "six months of daily testing" ## parameter all_recurrences should change all recurrences - - ## except February and July + ## except the two edited exceptions (offsets 2 and 7) recurrence = search(9) recurrence.icalendar_component["summary"] = "daily testing" recurrence.save(all_recurrences=True) - assert summary_by_month(1) == "daily testing" - assert summary_by_month(2) == "one month of daily testing" - assert summary_by_month(3) == "daily testing" - assert summary_by_month(7) == "six months of daily testing" - - ## Last ... let's change the dtend and dtstart of the recurrence - recurrence = search(9) - recurrence.icalendar_component.pop("dtstart") - recurrence.icalendar_component.add("dtstart", datetime(2015, 9, 1, 8, 0, 0)) - recurrence.icalendar_component.pop("dtend") - recurrence.icalendar_component.add("dtend", datetime(2015, 9, 1, 10, 0, 0)) - recurrence.save(all_recurrences=True) - - recurrence = search(8) - assert ( - recurrence.icalendar_component.start.astimezone() - == datetime(2015, 8, 1, 8, 0, 0).astimezone() - ) - assert ( - recurrence.icalendar_component.end.astimezone() - == datetime(2015, 8, 1, 10, 0, 0).astimezone() - ) + assert summary_on(1) == "daily testing" + assert summary_on(2) == "one month of daily testing" + assert summary_on(3) == "daily testing" + assert summary_on(7) == "six months of daily testing" + + ## Last ... let's change the dtend and dtstart of the recurrence. + ## This reschedules the whole series (moves the master DTSTART) while the + ## two exceptions above are still attached - some servers (e.g. OX) reject + ## that re-anchoring with a 409 Conflict, so it is gated on its own flag. + if self.is_supported("save-load.event.recurrences.exception.reschedule"): + recurrence = search(9) + recurrence.icalendar_component.pop("dtstart") + recurrence.icalendar_component.add("dtstart", day_start(9).replace(hour=8)) + recurrence.icalendar_component.pop("dtend") + recurrence.icalendar_component.add("dtend", day_start(9).replace(hour=10)) + recurrence.save(all_recurrences=True) + + recurrence = search(8) + assert ( + recurrence.icalendar_component.start.astimezone() + == day_start(8).replace(hour=8).astimezone() + ) + assert ( + recurrence.icalendar_component.end.astimezone() + == day_start(8).replace(hour=10).astimezone() + ) def testOffsetURL(self): """ @@ -4146,6 +4570,8 @@ def testOffsetURL(self): conn = client(**connect_params, url=url) principal = conn.principal() calendars = principal.get_calendars() + assert calendars is not None + assert calendars is not None def testObjects(self): # TODO: description ... what are we trying to test for here? @@ -4203,15 +4629,15 @@ def setup_method(self, *largs, **kwargs): def testNoProxyRaisesError(self): with client(**self.server_params) as conn: with pytest.raises(AssertionError): - principal = conn.principal() + conn.principal() def testWithProxyParams(self): with client(proxy=self.proxy, **self.server_params) as conn: - principal = conn.principal() + assert conn.principal() is not None def testWithProxyParamsWithoutScheme(self): with client(proxy=f"localhost:{self.PROXY.flags.port}", **self.server_params) as conn: - principal = conn.principal() + assert conn.principal() is not None ## TODO: figure out how to test this properly. @pytest.mark.skipif(True, reason="work in progress ... this doesn't seem to work") @@ -4219,7 +4645,7 @@ def testWithEnvironment(self): os.environ["HTTP_PROXY"] = self.proxy os.environ["HTTPS_PROXY"] = self.proxy with client(**self.server_params) as conn: - principal = conn.principal() + assert conn.principal() is not None ## TODO: test socks proxy as well. ## TODO: test https proxying as well diff --git a/tests/test_caldav_unit.py b/tests/test_caldav_unit.py index 8ae1c68b..2bd9b44f 100755 --- a/tests/test_caldav_unit.py +++ b/tests/test_caldav_unit.py @@ -8,6 +8,7 @@ import pickle from datetime import date, datetime, timedelta, timezone +from typing import Any from unittest import mock from urllib.parse import urlparse @@ -445,6 +446,57 @@ def testLoadByMultiGet404(self): with pytest.raises(error.NotFoundError): object.load_by_multiget() + def testPropfindResponseLevelNotFound(self): + """A PROPFIND answered with a response-level 404 must raise NotFoundError. + + RFC 4918 section 14.24 lets a carry either propstat + elements or a bare , so a server may report "this resource + does not exist" inside a 207 Multi-Status rather than as a transport + level 404. Xandikos does exactly that for PROPFIND on a missing + collection (while answering REPORT on the very same URL with a plain + 404). We used to look only at the propstats, find none, and hand the + caller a None value for every requested property. + """ + xml = """ + + + /calendars/shouldnotexist/ + HTTP/1.1 404 Not Found + +""" + client = MockedDAVClient(xml) + calendar = Calendar(client, url="/calendars/shouldnotexist/") + with pytest.raises(error.NotFoundError): + calendar.get_display_name() + with pytest.raises(error.NotFoundError): + calendar.get_properties([dav.DisplayName()]) + + def testPropfindResponseLevelNotFoundOnlyWhenAllAreMissing(self): + """A 404 for one href among several must NOT raise. + + On a Depth: 1 PROPFIND a single missing child is a normal, expected + part of the reply - only a reply where nothing at all was found means + the requested resource is gone. + """ + xml = """ + + + /calendars/ + + calendars + HTTP/1.1 200 OK + + + + /calendars/vanished/ + HTTP/1.1 404 Not Found + +""" + client = MockedDAVClient(xml) + calendar = Calendar(client, url="/calendars/") + props = calendar.get_properties([dav.DisplayName()], depth=1) + assert props[dav.DisplayName.tag] == "calendars" + @mock.patch("caldav.davclient.requests.Session.request") def testRequestCustomHeaders(self, mocked): """ @@ -485,7 +537,8 @@ def testEmptyXMLNoContentLength(self, mocked): mocked().status_code = 200 mocked().headers = {"Content-Type": "text/xml"} mocked().content = "" - client = DAVClient(url="AsdfasDF").request("/") + response = DAVClient(url="AsdfasDF").request("/") + assert response.status == 200 @mock.patch("caldav.davclient.requests.Session.request") def testNonValidXMLNoContentLength(self, mocked): @@ -656,6 +709,14 @@ def testAbsoluteURL(self): def _load(self, only_if_unloaded=True): self.data = todo6 + def _batch_load(self, objects): + ## Search results are batch-loaded via a single calendar-multiget REPORT + ## (Calendar._batch_load_objects); the mocked server returns no + ## calendar-data, so inject todo6 here the same way _load does per object. + for obj in objects: + obj.data = todo6 + + @mock.patch("caldav.collection.Calendar._batch_load_objects", new=_batch_load) @mock.patch("caldav.calendarobjectresource.CalendarObjectResource.load", new=_load) def testDateSearch(self): """ @@ -708,6 +769,8 @@ def testDateSearch(self): """ client = MockedDAVClient(xml) calendar = Calendar(client, url="/principals/calendar/home@petroski.example.com/963/") + ## expand=False does no client-side time-range filtering, so all three + ## server-returned hrefs are returned regardless of the search window. with pytest.deprecated_call(): results = calendar.date_search(datetime(2021, 2, 1), datetime(2021, 2, 7), expand=False) assert len(results) == 3 @@ -1150,6 +1213,14 @@ def test_xml_parsing(self): "{urn:ietf:params:xml:ns:caldav}calendar-data": None, }, } + ## This assert was missing: the XML and the expected dict above were + ## built and then never compared. Note the third href is + ## percent-encoded in the XML and plain in the expected keys, so this + ## also covers the unquoting. + assert ( + MockedDAVResponse(xml).expand_simple_props(props=[dav.GetEtag(), cdav.CalendarData()]) + == expected_results + ) def testHugeTreeParam(self): """ @@ -1461,12 +1532,12 @@ def testDataAPIStateTransitions(self): assert isinstance(event._state, RawDataState) # edit_icalendar_instance() SHOULD change state to IcalendarState - with event.edit_icalendar_instance() as cal: + with event.edit_icalendar_instance(): pass assert isinstance(event._state, IcalendarState) # edit_vobject_instance() SHOULD change state to VobjectState - with event.edit_vobject_instance() as vobj: + with event.edit_vobject_instance(): pass assert isinstance(event._state, VobjectState) @@ -1503,6 +1574,85 @@ def testDataAPINoDataState(self): assert event._get_component_type_cheap() is None assert event._has_data() is False + def test_set_data_updates_state_cache(self) -> None: + """§2.9: _set_data (raw string branch) must reset _state so that + get_data()/get_icalendar_instance()/id return the new content. + + Bug: _set_data cleared _data/_vobject_instance/_icalendar_instance + but never updated self._state. Once _state was cached by an earlier + call to _ensure_state() (e.g. via event.id or is_loaded()), all + subsequent reads through the new API served stale content. + """ + from caldav.datastate import RawDataState + + client = DAVClient(url="http://cal.example.com/") + ev2 = """BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Example Corp.//CalDAV Client//EN +BEGIN:VEVENT +UID:updated-uid@example.com +DTSTAMP:20260101T000000Z +DTSTART:20260601T100000Z +DTEND:20260601T110000Z +SUMMARY:Updated Event +END:VEVENT +END:VCALENDAR +""" + event = Event(client, data=ev1) + + # Prime the state cache — simulates the common scenario where the + # object is accessed before a reload (e.g. event.id or is_loaded()) + assert event.id == "20010712T182145Z-123401@example.com" + assert isinstance(event._state, RawDataState) + + # Simulate what load() does: assign new raw data + event.data = ev2 + + # _state must now reflect the new data + assert isinstance(event._state, RawDataState) + assert event.get_data() == ev2, "get_data() returned stale pre-reload content" + assert event.id == "updated-uid@example.com", "id returned stale UID after reload" + assert "Updated Event" in event.get_data() + + def test_vfreebusy_component_type_detection(self) -> None: + """§2.10: RawDataState.get_component_type() tested for 'BEGIN:FREEBUSY' + but real iCalendar data uses 'BEGIN:VFREEBUSY', so FreeBusy objects + got component_type=None → is_loaded()/has_component() False → save() + silent no-op and load(only_if_unloaded=True) spuriously reloads. + Also fixes get_uid()/get_component_type() in DataState base class + which listed 'FREEBUSY' instead of 'VFREEBUSY' as comp.name. + """ + from caldav.datastate import RawDataState + + freebusy_data = """BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Test//Test//EN +BEGIN:VFREEBUSY +UID:freebusy@example.com +DTSTAMP:20240101T120000Z +DTSTART:20240601T090000Z +DTEND:20240601T110000Z +FREEBUSY:20240601T090000Z/20240601T100000Z +END:VFREEBUSY +END:VCALENDAR +""" + state = RawDataState(freebusy_data) + assert state.get_component_type() == "VFREEBUSY", ( + "RawDataState.get_component_type() returned None for VFREEBUSY data " + "(was checking for 'BEGIN:FREEBUSY' instead of 'BEGIN:VFREEBUSY')" + ) + assert state.get_uid() == "freebusy@example.com" + + # Also verify via the base class parsers (IcalendarState path) + import icalendar + + from caldav.datastate import IcalendarState + + ical = icalendar.Calendar.from_ical(freebusy_data) + istate = IcalendarState(ical) + assert istate.get_component_type() == "VFREEBUSY" + assert istate.get_uid() == "freebusy@example.com" + def testDataAPIEdgeCases(self): """Test edge cases in the data API (issue #613).""" cal_url = "http://me:hunter2@calendar.example:80/" @@ -1532,9 +1682,9 @@ def testDataAPIEdgeCases(self): # Test that nested borrowing (even same type) raises error # This prevents confusing ownership semantics event2 = Event(client, data=ev1) - with event2.edit_icalendar_instance() as cal1: + with event2.edit_icalendar_instance(): with pytest.raises(RuntimeError): - with event2.edit_icalendar_instance() as cal2: + with event2.edit_icalendar_instance(): pass # Test sequential edits work fine @@ -1566,7 +1716,13 @@ def testTodoDuration(self): assert my_todo2.get_duration() == timedelta(days=6) assert my_todo2.get_due() == orig_start assert my_todo3.get_duration() == timedelta(days=5) - foo6 = my_todo3.get_due().strftime("%s") == "1177945200" + ## This used to read `foo6 = ...strftime("%s") == "1177945200"`: an + ## assertion assigned to a variable and never checked. It could not have + ## been asserted as written either - "%s" is a glibc extension that + ## ignores tzinfo, so the comparison only holds in the author's own + ## timezone (1177945200 is 2007-04-30T16:00+02:00; in UTC the same + ## datetime gives 1177948800). Asserted on the value instead. + assert my_todo3.get_due() == datetime(2007, 4, 30, 16, 0, tzinfo=timezone.utc) some_date = date(2011, 1, 1) my_todo1.set_due(some_date) @@ -1625,6 +1781,31 @@ def testTodoDuration(self): assert "DUE" not in my_todo4.component assert my_todo4.component["duration"].dt == timedelta(2) + def testTodoDurationTimedDtstart(self): + """§2.11: _get_duration must return timedelta(0) for a VTODO with a timed DTSTART + and no DUE/DURATION — not timedelta(days=1). + + isinstance(i["DTSTART"], datetime) tested the vDDDTypes wrapper (always False), + so the date-vs-datetime branch always took the 'is a date' path, returning 1 day. + Fix: test isinstance(i["DTSTART"].dt, datetime) instead. + """ + cal_url = "http://me:hunter2@calendar.example:80/" + client = DAVClient(url=cal_url) + todo_timed_dtstart = """BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Test//EN +BEGIN:VTODO +UID:timed-dtstart@example.com +DTSTAMP:20240101T000000Z +DTSTART:20240601T100000Z +SUMMARY:Todo with timed DTSTART only +END:VTODO +END:VCALENDAR""" + todo_item = Todo(client, data=todo_timed_dtstart) + assert todo_item.get_duration() == timedelta(0), ( + f"Expected timedelta(0) for timed DTSTART with no DUE, got {todo_item.get_duration()}" + ) + def testURL(self): """Exercising the URL class""" long_url = "http://foo:bar@www.example.com:8080/caldav.php/?foo=bar" @@ -1722,6 +1903,28 @@ def testURL(self): == URL("//www.example.com/bar/").canonical() ) + # 9b) canonical() must strip credentials (§2.5a) + cred_url = URL("https://user:pass@example.com/cal/") + canon = cred_url.canonical() + assert "user" not in str(canon), "canonical() leaked username" + assert "pass" not in str(canon), "canonical() leaked password" + + # 9c) canonical() must not mutate self (§2.5b): + # a URL with no auth — unauth() returns self, canonical() must + # still return a fresh object and leave self unchanged. + plain_url = URL("http://example.com/cal/path/") + before = str(plain_url) + _ = plain_url.canonical() + assert str(plain_url) == before, "canonical() mutated self" + + # 9d) __eq__ calls canonical() — must not mutate self (§2.5b) + # A literal '+' in the path would become '%2B' after quote(unquote()), + # silently changing what resource subsequent requests target. + plus_url = URL("http://example.com/cal/foo+bar/") + before_plus = str(plus_url) + _ = plus_url == URL("http://example.com/cal/foo+bar/") + assert str(plus_url) == before_plus, "__eq__ mutated URL containing '+'" + # 10) pickle assert pickle.loads(pickle.dumps(url1)) == url1 @@ -1733,7 +1936,11 @@ def testFilters(self): ) ) ) - # print(filter) + filter_xml = str(filter) + assert 'name="VCALENDAR"' in filter_xml + assert 'name="VEVENT"' in filter_xml + assert 'name="UID"' in filter_xml + assert "pouet" in filter_xml crash = cdav.CompFilter() value = None @@ -1781,6 +1988,9 @@ def testExtractAuth(self): "basic", "digest", } + # §1.8: trailing comma (seen in the wild) must not raise IndexError + assert client.extract_auth_types("Basic,") == {"basic"} + assert client.extract_auth_types('Basic realm="x",') == {"basic"} def testAutoUrlEcloudWithEmailUsername(self) -> None: """ @@ -2095,8 +2305,8 @@ def test_add_object_orphan_does_not_raise_notfound(self): created = [] original_create = CalendarObjectResource._create - CalendarObjectResource._create = ( - lambda self_, id=None, path=None, retry_on_failure=True: created.append(True) + CalendarObjectResource._create = lambda self_, id=None, path=None, retry_on_failure=True: ( + created.append(True) ) try: calendar.add_object(Event, self._orphan_ical) @@ -2756,6 +2966,77 @@ def test_rate_limit_max_sleep_stops_adaptive_retries(self, mocked): client.request("/") +class TestAsyncProbeResponseNotReturnedAsReal: + """§2.15: async _async_request: when the probe GET for issue-#158 workaround does + not receive a 401+WWW-Authenticate response, the original exception must be re-raised. + + Before the fix, a probe response with status != 401 (e.g. 200 HTML login page) fell + through to response = DAVResponse(r, self), returning the probe GET response as if + it were the real request's response — status 200 for a PUT that never happened. + """ + + @pytest.mark.asyncio + async def test_probe_200_reraises_original_exception(self): + """If the probe GET returns 200 (not a 401 challenge), the original error must propagate.""" + from unittest.mock import AsyncMock, patch + + from caldav.async_davclient import AsyncDAVClient + + client = AsyncDAVClient(url="http://cal.example.com/", password="secret") + + probe_resp = mock.MagicMock() + probe_resp.status_code = 200 + probe_resp.reason = "OK" + probe_resp.headers = {"Content-Type": "text/html"} + probe_resp.reason_phrase = "OK" + + original_error = ConnectionError("server aborted connection") + + async def mock_request(*args, **kwargs): + if kwargs.get("method") == "GET" and not kwargs.get("auth"): + return probe_resp + raise original_error + + with patch.object(client.session, "request", side_effect=mock_request): + with pytest.raises((ConnectionError, Exception)): + await client._async_request("/some/resource", "PUT", "data", {}) + + +class TestRateLimitNoPlusNone: + """§1.3: rate-limit retry must not raise TypeError when second 429 has no usable Retry-After. + + sleep_seconds += rate_limit_time_slept / 2 executed before the is-None check, + so None += 2.5 raised TypeError instead of the documented RateLimitError. + """ + + def _make_response(self, status_code, headers=None): + r = mock.MagicMock() + r.status_code = status_code + r.headers = headers or {} + r.reason = "Too Many Requests" + return r + + @mock.patch("caldav.davclient.requests.Session.request") + def test_second_429_without_retry_after_raises_rate_limit_error(self, mocked): + """Second 429 with Retry-After: 0 (compute_sleep_seconds → None) must raise + RateLimitError, not TypeError.""" + ok = mock.MagicMock() + ok.status_code = 200 + ok.headers = {} + mocked.side_effect = [ + self._make_response(429, {"Retry-After": "5"}), + self._make_response(429, {"Retry-After": "0"}), # compute_sleep_seconds → None + ] + client = DAVClient( + url="http://cal.example.com/", + rate_limit_handle=True, + rate_limit_default_sleep=None, + ) + with mock.patch("caldav.davclient.time.sleep"): + with pytest.raises(error.RateLimitError): + client.request("/") + + class TestDateToUtcConversion: """ RFC 4791 §9.9: time-range start/end MUST be UTC datetime values. @@ -2897,6 +3178,26 @@ def test_recursive_meta_section(self): } assert set(expand_config_section(config, "all")) == {"a", "b", "c"} + def test_missing_section_returns_empty(self): + """§1.9: expand_config_section(config, "default") when "default" is absent must + return [] rather than raising KeyError.""" + from caldav.config import expand_config_section + + config = {"work": {"caldav_url": "https://work.example.com/"}} + # Requesting a section that doesn't exist should return [] (no match), not crash + assert expand_config_section(config, "default") == [] + + def test_disable_respected_for_named_sections(self): + """§2.17: disable:true must suppress named sections, not just glob '*' results. + + The old code used the literal string 'section' instead of the variable, + so disable was only effective under the '*' glob path. + """ + from caldav.config import expand_config_section + + config = {"work": {"caldav_url": "https://work.example.com/", "disable": True}} + assert expand_config_section(config, "work") == [] + class TestConfigSectionInheritance: """Unit tests for caldav.config.config_section (inherits key).""" @@ -3047,6 +3348,50 @@ def test_calendar_url_extracted_from_section(self, tmp_path): assert len(results) == 1 assert results[0]["calendar_url"] == "/dav/user/mycalendar/" + def test_section_with_features_but_no_url(self, tmp_path): + """A section without caldav_url is usable when it has features — + the client constructor resolves the URL from auto-connect.url hints.""" + import json + + from caldav.config import get_all_file_connection_params + + config = { + "ecloud": { + "caldav_username": "user@e.email", + "caldav_password": "pass", + "features": "ecloud", + } + } + config_file = tmp_path / "calendar.conf" + config_file.write_text(json.dumps(config)) + results = get_all_file_connection_params(str(config_file), "ecloud") + assert len(results) == 1 + assert results[0]["username"] == "user@e.email" + assert results[0]["features"] + + def test_get_connection_params_features_but_no_url(self, tmp_path): + """Same as above, but through get_connection_params — the code path + used by get_davclient(config_section=...).""" + import json + + from caldav.config import get_connection_params + + config = { + "ecloud": { + "caldav_username": "user@e.email", + "caldav_password": "pass", + "features": "ecloud", + } + } + config_file = tmp_path / "calendar.conf" + config_file.write_text(json.dumps(config)) + params = get_connection_params( + config_file=str(config_file), config_section="ecloud", environment=False + ) + assert params is not None + assert params["username"] == "user@e.email" + assert params["features"] + def test_meta_section_returns_multiple_dicts(self, tmp_path): import json @@ -3074,6 +3419,97 @@ def test_meta_section_returns_multiple_dicts(self, tmp_path): } +class TestExplicitParamsMerge: + """§2.18: get_connection_params explicit kwargs must be merged with env/file config. + + The old code only returned explicit_params when 'url' or 'features' was present; + params like password-only were silently discarded when an env/file source was found. + """ + + def test_explicit_password_merged_with_env_url(self, monkeypatch): + """get_connection_params(password='secret') with CALDAV_URL in env must include the password.""" + from caldav.config import get_connection_params + + monkeypatch.setenv("CALDAV_URL", "https://env.example.com/") + monkeypatch.setenv("CALDAV_USERNAME", "envuser") + # Unset file config to avoid config-file interference + monkeypatch.delenv("CALDAV_CONFIG_FILE", raising=False) + result = get_connection_params(password="secret", check_config_file=False) + assert result is not None + assert result.get("password") == "secret" + assert result.get("url") == "https://env.example.com/" + + def test_explicit_none_does_not_clobber_env_url(self, monkeypatch): + """Gate finding F8: a kwarg that is explicitly None is "not supplied", + not "unset it". + + The common CLI wrapper -- get_davclient(url=args.url, + username=args.user, password=args.password) where the user only gave + --password -- overlaid url=None on top of the winning source and + wiped CALDAV_URL.""" + from caldav.config import get_connection_params + + monkeypatch.setenv("CALDAV_URL", "https://env.example.com/") + monkeypatch.setenv("CALDAV_USERNAME", "envuser") + monkeypatch.delenv("CALDAV_CONFIG_FILE", raising=False) + result = get_connection_params( + url=None, username=None, password="secret", check_config_file=False + ) + assert result is not None + assert result.get("url") == "https://env.example.com/" + assert result.get("username") == "envuser" + assert result.get("password") == "secret" + + def test_empty_string_username_is_still_explicit(self, monkeypatch): + """An empty username is meaningful (servers with no auth) and must + not be discarded along with the Nones.""" + from caldav.config import get_connection_params + + monkeypatch.setenv("CALDAV_URL", "https://env.example.com/") + monkeypatch.setenv("CALDAV_USERNAME", "envuser") + monkeypatch.delenv("CALDAV_CONFIG_FILE", raising=False) + result = get_connection_params(username="", check_config_file=False) + assert result is not None + assert result.get("username") == "" + + def test_explicit_params_merged_with_test_server_config(self, monkeypatch): + """Gate finding F8: the testconfig branch returned the test-server + config verbatim, dropping the explicit kwargs entirely.""" + from caldav import config as config_module + from caldav.config import get_connection_params + + monkeypatch.setattr( + config_module, + "_get_test_server_config", + lambda *a, **kw: {"url": "https://testserver.example.com/", "username": "testuser"}, + ) + result = get_connection_params(testconfig=True, password="secret") + assert result is not None + assert result.get("url") == "https://testserver.example.com/" + assert result.get("password") == "secret" + + +class TestResolveFeaturesMutation: + """§2.19: resolve_features and testing.py server classes must deepcopy hint dicts. + + Returning or shallow-copying a module-level dict then mutating a nested key + permanently corrupts the module-level dict for all subsequent users. + """ + + def test_resolve_features_string_returns_independent_copy(self): + """resolve_features('xandikos') must return a deep copy, not the module object.""" + from caldav import compatibility_hints as hints + from caldav.config import resolve_features + + original_domain = hints.xandikos.get("auto-connect.url", {}).get("domain", "") + result = resolve_features("xandikos") + # Mutate the returned copy + if "auto-connect.url" in result and isinstance(result["auto-connect.url"], dict): + result["auto-connect.url"]["domain"] = "MUTATED:9999" + # Original must be unchanged + assert hints.xandikos.get("auto-connect.url", {}).get("domain") == original_domain + + class TestResolveProperties: """Tests for _resolve_properties unbound variable bug (issue #647 / calendar-cli #114).""" @@ -3245,3 +3681,499 @@ def test_change_attendee_status_raises_when_username_not_email(self): ev = self._make_event_with_mock_client("just_a_username") with pytest.raises(caldav_error.NotFoundError): ev.change_attendee_status(partstat="ACCEPTED") + + +class TestAddAttendee: + """§1.6: add_attendee() crashes with UnboundLocalError on uppercase MAILTO: scheme. + + RFC 3986 §3.1 specifies URI schemes are case-insensitive, so "MAILTO:user@example.com" + is valid and common in real-world iCalendar data. The old code only matched lowercase + "mailto:" — uppercase fell through all string branches, leaving attendee_obj unassigned. + """ + + _base_event = """\ +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Test//Test//EN +BEGIN:VEVENT +UID:test-add-attendee@example.com +DTSTAMP:20240101T000000Z +DTSTART:20240601T100000Z +DTEND:20240601T110000Z +SUMMARY:Test event +END:VEVENT +END:VCALENDAR +""" + + def test_add_attendee_uppercase_mailto(self): + """add_attendee('MAILTO:user@example.com') must not raise UnboundLocalError.""" + ev = Event(data=self._base_event) + ev.add_attendee("MAILTO:user@example.com") + attendee = ev.icalendar_component["attendee"] + assert "user@example.com" in str(attendee).lower() + + def test_add_attendee_mixed_case_mailto(self): + """Mixed-case scheme variants like 'Mailto:' must also work.""" + ev = Event(data=self._base_event) + ev.add_attendee("Mailto:user@example.com") + attendee = ev.icalendar_component["attendee"] + assert "user@example.com" in str(attendee).lower() + + +class TestChangeAttendeeStatusNoAttendees: + """§1.7: change_attendee_status() raises bare KeyError when event has no ATTENDEE property. + + ical_obj["attendee"] raises KeyError when the key is absent; the NotFoundError-catching + loop in the Principal branch never sees it, so the "Principal is not invited" message + is unreachable and callers get an unexpected KeyError instead. + + Also: the not-found message contained a literal '%s' placeholder that was never + substituted. + """ + + _event_no_attendees = """\ +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Test//Test//EN +BEGIN:VEVENT +UID:test-no-attendees@example.com +DTSTAMP:20240101T000000Z +DTSTART:20240601T100000Z +DTEND:20240601T110000Z +SUMMARY:Event with no attendees +END:VEVENT +END:VCALENDAR +""" + + def test_change_attendee_status_no_attendees_raises_not_found(self): + """Calling change_attendee_status on an event with no ATTENDEE must raise + NotFoundError, not KeyError.""" + ev = Event(data=self._event_no_attendees) + with pytest.raises(error.NotFoundError): + ev.change_attendee_status("mailto:nobody@example.com", partstat="ACCEPTED") + + def test_change_attendee_status_error_message_contains_attendee(self): + """The not-found error message must contain the attendee address, not a literal '%s'.""" + ev = Event(data=self._event_no_attendees) + with pytest.raises(error.NotFoundError) as exc_info: + ev.change_attendee_status("mailto:nobody@example.com", partstat="ACCEPTED") + assert "%s" not in str(exc_info.value) + assert "nobody@example.com" in str(exc_info.value) + + +class TestFeatureSetCopyFeatureSet: + """§1.10 + §1.11: FeatureSet.copyFeatureSet() correctness bugs. + + §1.10: Merging a plain-string feature over an existing string-valued feature raised + bare AssertionError because the 'support' not in server_node guard prevented the + update branch from running. + + §1.11: An unknown feature name produced a UserWarning but was still stored in + _server_features; a later collapse()/is_supported() then hit a message-less + AssertionError far from the originating config. Unknown features must be skipped + (continue after warning) so bad keys never contaminate the feature set. + """ + + def test_string_feature_can_be_overridden(self): + """copyFeatureSet must accept a string value that overrides an existing string.""" + from caldav.compatibility_hints import FeatureSet + + fs = FeatureSet({"scheduling": "unsupported"}) + fs.copyFeatureSet({"scheduling": "fragile"}) + assert fs.is_supported("scheduling") is False # fragile → False per is_supported semantics + + def test_string_feature_full_override(self): + """Overriding 'unsupported' with 'full' must make is_supported return True.""" + from caldav.compatibility_hints import FeatureSet + + fs = FeatureSet({"scheduling": "unsupported"}) + fs.copyFeatureSet({"scheduling": "full"}) + assert fs.is_supported("scheduling") is True + + def test_unknown_feature_warns_and_does_not_store(self): + """An unknown feature name must emit UserWarning and must NOT be stored.""" + import warnings + + from caldav.compatibility_hints import FeatureSet + + fs = FeatureSet({}) + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + fs.copyFeatureSet({"totally_nonexistent_feature_xyz": "full"}) + + assert any("totally_nonexistent_feature_xyz" in str(warning.message) for warning in w) + # The bad key must NOT be in the internal feature dict + assert "totally_nonexistent_feature_xyz" not in fs._server_features + + +class TestXMLEntityHardening: + """§3.2: XML parser must not expand entity references from untrusted server data. + + etree.XMLParser without resolve_entities=False expands inline DOCTYPE + entities, allowing a malicious server to inject arbitrary text into + parsed values. With resolve_entities=False the entity reference is + left as-is (text becomes None for element content). + """ + + def test_xml_entity_not_expanded(self): + """Entity defined in DOCTYPE must NOT be expanded into element text.""" + xml = b""" +]> + + + / + + &xxe; + HTTP/1.1 200 OK + + +""" + resp = MockedDAVResponse(xml) + assert resp.tree is not None + displayname_el = resp.tree.find(".//{DAV:}displayname") + assert displayname_el is not None + assert displayname_el.text != "INJECTED", ( + "XML entity was expanded — resolve_entities=False is missing from the parser" + ) + + +class TestDAVClientCredentialPrecedence: + """§1.2: DAVClient credential handling bugs. + + - URL with username but no password (user@host) crashed with TypeError inside + urllib.parse.unquote(None). + - URL credentials had higher precedence than explicit kwargs; async client was + the opposite (explicit kwargs win). Now sync matches async: explicit kwargs win. + """ + + def test_url_with_user_but_no_password_does_not_crash(self): + """DAVClient(url='https://user@host/', password='p') must not raise TypeError.""" + client = DAVClient(url="https://user@cal.example.com/dav/", password="secret") + assert client.username == "user" + assert client.password == b"secret" + + def test_explicit_kwargs_take_precedence_over_url_credentials(self): + """Explicit username/password kwargs must override credentials embedded in the URL.""" + client = DAVClient( + url="https://urluser:urlpass@cal.example.com/dav/", + username="kwarguser", + password="kwargpass", + ) + assert client.username == "kwarguser" + assert client.password == b"kwargpass" + + def test_explicit_username_does_not_borrow_url_password(self): + """Gate finding F6: credentials are a pair, not two independent fields. + + Merging them field by field produced ``username="alice"`` with + ``password="hunter2"`` — alice's login shipping bob's password.""" + client = DAVClient( + url="https://bob:hunter2@cal.example.com/dav/", + username="alice", + ) + assert client.username == "alice" + assert client.password is None + + def test_explicit_password_still_pairs_with_the_url_username(self): + """Overriding only the password keeps the pair coherent: the username + still comes from the URL, so there is no account mismatch.""" + client = DAVClient( + url="https://bob:hunter2@cal.example.com/dav/", + password="s3cret", + ) + assert client.username == "bob" + assert client.password == b"s3cret" + + +class TestPropstatStatusValidation: + """Gate finding F7: a failing propstat status must raise, not vanish. + + Deduplicating the propstat loops dropped the per-propstat + ``validate_status()`` call, so a ``500``/``403``/``507`` propstat yielded + a silently-empty value instead of a ``ResponseError`` -- and the calendar + was then dropped from ``get_calendars()`` with only a log line. + ``_validate_status`` raises unconditionally; it is not an assert, so + ``python -O`` does not disable it either. + """ + + XML = """ + + + /dav/calendars/user/broken/ + + + HTTP/1.1 500 Internal Server Error + + + +""" + + def test_server_error_propstat_raises(self): + with pytest.raises(error.ResponseError): + MockedDAVResponse(self.XML).expand_simple_props(props=[dav.DisplayName()]) + + def test_insufficient_storage_propstat_raises(self): + xml = self.XML.replace("500 Internal Server Error", "507 Insufficient Storage") + with pytest.raises(error.ResponseError): + MockedDAVResponse(xml).expand_simple_props(props=[dav.DisplayName()]) + + def test_404_propstat_is_still_a_missing_property(self): + """The 404 quirk must survive: a 404 propstat means "not set here", + not "the request failed".""" + xml = self.XML.replace("500 Internal Server Error", "404 Not Found") + result = MockedDAVResponse(xml).expand_simple_props(props=[dav.DisplayName()]) + assert result == {"/dav/calendars/user/broken/": {"{DAV:}displayname": None}} + + +class TestAsyncDAVClientCredentialPairing: + """Gate finding F6, async twin: same field-by-field merge, same result.""" + + def test_explicit_username_does_not_borrow_url_password(self): + from caldav.async_davclient import AsyncDAVClient + + client = AsyncDAVClient( + url="https://bob:hunter2@cal.example.com/dav/", + username="alice", + ) + assert client.username == "alice" + assert client.password is None + + def test_explicit_password_still_pairs_with_the_url_username(self): + from caldav.async_davclient import AsyncDAVClient + + client = AsyncDAVClient( + url="https://bob:hunter2@cal.example.com/dav/", + password="s3cret", + ) + assert client.username == "bob" + assert client.password == "s3cret" + + +class TestPostPutRedirect: + """§1.1: 302 response to PUT must update event.url from Location header. + + Bug: `[x[1] for x in r.headers if x[0] == "location"]` iterates the + headers dict yielding key *strings*, so x[0] is the first character of + each header name — never "location". The list is always empty and any + 302 raises IndexError. + """ + + @mock.patch("caldav.davclient.requests.Session.request") + def test_302_put_updates_url_from_location_header(self, mocked): + try: + from niquests.structures import CaseInsensitiveDict + except ImportError: + from requests.structures import CaseInsensitiveDict + + new_url = "http://cal.example.com/cal/new-location.ics" + resp = mock.MagicMock() + resp.status_code = 302 + resp.headers = CaseInsensitiveDict({"Location": new_url}) + resp.reason = "Found" + resp.content = b"" + mocked.return_value = resp + + client = DAVClient(url="http://cal.example.com/") + cal = Calendar(client=client, url="http://cal.example.com/cal/") + event = Event( + client=client, + url="http://cal.example.com/cal/event.ics", + data=ev1, + parent=cal, + ) + event.save() + assert str(event.url) == new_url + + +class TestWarnUnreadableDisplayName: + """§5 (DRY): the shared helper backing the sync/async name-matching loops. + + Warn unless we positively know the server can't read DAV:displayname via + PROPFIND (propfind.displayname non-supported); warn when the feature is + supported or when there is no feature matrix to consult. + """ + + @staticmethod + def _client(features): + client = mock.MagicMock() + client.features = features + return client + + def test_warns_when_feature_supported(self, caplog): + from caldav.base_client import _warn_unreadable_display_name + from caldav.compatibility_hints import FeatureSet + + cal = mock.MagicMock() + cal.url = "http://cal.example.com/cal/" + with caplog.at_level("WARNING", logger="caldav"): + _warn_unreadable_display_name( + self._client(FeatureSet()), cal, "Work", Exception("boom") + ) + assert any("Could not read display name" in r.message for r in caplog.records) + + def test_silent_when_feature_unsupported(self, caplog): + from caldav.base_client import _warn_unreadable_display_name + from caldav.compatibility_hints import FeatureSet + + features = FeatureSet({"propfind.displayname": {"support": "unsupported"}}) + with caplog.at_level("WARNING", logger="caldav"): + _warn_unreadable_display_name( + self._client(features), mock.MagicMock(), "Work", Exception("boom") + ) + assert not caplog.records + + def test_silent_when_parent_propfind_unsupported(self, caplog): + from caldav.base_client import _warn_unreadable_display_name + from caldav.compatibility_hints import FeatureSet + + # propfind.displayname falls back to the propfind parent when not probed + features = FeatureSet({"propfind": {"support": "unsupported"}}) + with caplog.at_level("WARNING", logger="caldav"): + _warn_unreadable_display_name( + self._client(features), mock.MagicMock(), "Work", Exception("boom") + ) + assert not caplog.records + + def test_warns_when_no_feature_matrix(self, caplog): + from caldav.base_client import _warn_unreadable_display_name + + client = mock.MagicMock() + client.features = None + with caplog.at_level("WARNING", logger="caldav"): + _warn_unreadable_display_name(client, mock.MagicMock(), "Work", Exception("boom")) + assert any("Could not read display name" in r.message for r in caplog.records) + + +class TestRecurringCompleteHelpers: + """Pure (no-I/O) unit tests for the recurring-task completion helpers. + + These exercise the icalendar-mutation logic shared by the sync and async + ``complete()`` paths without touching a server. + ref https://github.com/python-caldav/caldav code-review §5.6 + """ + + def _make_todo(self, data: str = todo6) -> Todo: + client = MockedDAVClient("") + cal_url = "https://somwhere.in.the.universe.example/some/caldav/root/cal/" + calendar = Calendar(client, url=cal_url) + return Todo(client, data=data, parent=calendar, url=cal_url + "t.ics") + + def test_prepare_thisandfuture_advances_series(self) -> None: + todo = self._make_todo() + before = len(todo.icalendar_instance.subcomponents) + todo._prepare_recurring_thisandfuture(datetime(2026, 6, 14, tzinfo=timezone.utc)) + subs = todo.icalendar_instance.subcomponents + ## a completed recurrence + a new THISANDFUTURE instance were added + assert len(subs) == before + 2 + last = subs[-1] + assert last["RECURRENCE-ID"].params.get("RANGE") == "THISANDFUTURE" + ## one of the recurrences is now marked COMPLETED + assert any(str(s.get("STATUS")) == "COMPLETED" for s in subs) + + def test_build_safe_completed_returns_completed_copy(self) -> None: + todo = self._make_todo() + orig_dtstart = todo.icalendar_component["DTSTART"].dt + completed = todo._build_recurring_safe_completed(datetime(2026, 6, 14, tzinfo=timezone.utc)) + assert completed is not None + ## the standalone copy is completed and no longer recurring + assert str(completed.icalendar_component["STATUS"]) == "COMPLETED" + assert "RRULE" not in completed.icalendar_component + ## the master task advanced to its next occurrence and stays recurring + assert todo.icalendar_component["DTSTART"].dt > orig_dtstart + assert "RRULE" in todo.icalendar_component + + def test_build_safe_completed_none_when_count_one(self) -> None: + ## A recurring task with COUNT=1 is not really recurring + todo = self._make_todo() + todo.icalendar_component["RRULE"]["COUNT"] = [1] + assert ( + todo._build_recurring_safe_completed(datetime(2026, 6, 14, tzinfo=timezone.utc)) is None + ) + + def test_safe_completion_issues_two_puts(self, monkeypatch: Any) -> None: + """The standalone completed copy must not be PUT twice. + + The old async twin had drifted into saving the completed copy once + as still-pending and again as completed (3 PUTs total for the + operation). Completing in memory first means one PUT per object. + """ + saves: list[Todo] = [] + + def fake_save(self: Todo, *a: Any, **k: Any) -> Todo: + saves.append(self) + return self + + monkeypatch.setattr(Todo, "save", fake_save) + todo = self._make_todo() + todo._complete_recurring_safe(datetime(2026, 6, 14, tzinfo=timezone.utc)) + ## one PUT for the standalone completed copy, one for the advanced master + assert len(saves) == 2 + + +class TestAdoptCanonicalUrl: + """Gate finding F4: after creating a calendar on a server where + ``create-calendar.stable-url`` is unsupported, the canonical URL is + discovered by display name. When two calendars share that name the + docstring promises the requested URL is kept -- adopting one of them at + random can re-point a freshly created calendar at a *pre-existing* + unrelated calendar, so every later ``add_event()``, ``search()`` and + ``delete()`` would hit the wrong calendar and the new one would be + orphaned.""" + + REQUESTED = "https://cal.example.com/dav/requested-id/" + + def _make_calendar(self, *relocated_urls: str) -> Calendar: + client = mock.Mock(spec=DAVClient) + client.url = URL("https://cal.example.com/") + calendar = Calendar(client=client, url=self.REQUESTED) + + others = [] + for url in relocated_urls: + other = mock.Mock() + other.url = URL(url) + other.get_display_name.return_value = "My Calendar" + other.get_display_name.side_effect = None + others.append(other) + + parent = mock.Mock() + parent.calendars.return_value = others + calendar.parent = parent + return calendar + + def test_single_match_is_adopted(self) -> None: + calendar = self._make_calendar("https://cal.example.com/dav/canonical/") + calendar._adopt_canonical_url("My Calendar") + assert str(calendar.url) == "https://cal.example.com/dav/canonical/" + + def test_ambiguous_name_keeps_requested_url(self) -> None: + calendar = self._make_calendar( + "https://cal.example.com/dav/canonical/", + "https://cal.example.com/dav/some-old-calendar/", + ) + calendar._adopt_canonical_url("My Calendar") + assert str(calendar.url) == self.REQUESTED + + def _make_async_calendar(self, *relocated_urls: str) -> Calendar: + calendar = self._make_calendar(*relocated_urls) + others = calendar.parent.calendars.return_value + for other in others: + other.get_display_name = mock.AsyncMock(return_value="My Calendar") + calendar.parent.calendars = mock.AsyncMock(return_value=others) + return calendar + + def test_async_single_match_is_adopted(self) -> None: + import asyncio + + calendar = self._make_async_calendar("https://cal.example.com/dav/canonical/") + asyncio.run(calendar._async_adopt_canonical_url("My Calendar")) + assert str(calendar.url) == "https://cal.example.com/dav/canonical/" + + def test_async_ambiguous_name_keeps_requested_url(self) -> None: + import asyncio + + calendar = self._make_async_calendar( + "https://cal.example.com/dav/canonical/", + "https://cal.example.com/dav/some-old-calendar/", + ) + asyncio.run(calendar._async_adopt_canonical_url("My Calendar")) + assert str(calendar.url) == self.REQUESTED diff --git a/tests/test_compatibility_hints.py b/tests/test_compatibility_hints.py index 3277871f..79dc0a9d 100644 --- a/tests/test_compatibility_hints.py +++ b/tests/test_compatibility_hints.py @@ -207,19 +207,27 @@ def test_collapse_parent_already_exists(self) -> None: assert fs._server_features["search.text"] == {"support": "fragile"} def test_collapse_parent_exists_same_value(self) -> None: - """When parent exists with same value as subfeatures, should still collapse""" + """When parent exists with same value as subfeatures, should still collapse. + + Uses a genuine *grouping* parent (principal-search.by-name has no + explicit default); independent parents such as sync-token are + intentionally never collapsed (see + test_collapse_independent_parent_not_collapsed). by-name's parent + principal-search has a second, unset child (list-all), so the collapse + does not cascade further up. + """ fs = FeatureSet() fs._server_features = { - "sync-token": {"support": "unsupported"}, - "sync-token.delete": {"support": "unsupported"}, + "principal-search.by-name": {"support": "unsupported"}, + "principal-search.by-name.self": {"support": "unsupported"}, } fs.collapse() # All have same value, so subfeature should be removed - assert "sync-token.delete" not in fs._server_features - assert fs._server_features["sync-token"] == {"support": "unsupported"} + assert "principal-search.by-name.self" not in fs._server_features + assert fs._server_features["principal-search.by-name"] == {"support": "unsupported"} def test_collapse_empty_featureset(self) -> None: """Collapse should handle empty featureset without errors""" @@ -243,19 +251,19 @@ def test_collapse_no_parent_features(self) -> None: assert fs._server_features == {"sync-token": {"support": "full"}} def test_collapse_single_subfeature(self) -> None: - """Single subfeature should collapse since parent derives from children""" + """Single subfeature should collapse since a grouping parent derives from children""" fs = FeatureSet() - # sync-token only has one subfeature: delete + # principal-search.by-name (a grouping node) only has one subfeature: self fs._server_features = { - "sync-token.delete": {"support": "unsupported"}, + "principal-search.by-name.self": {"support": "unsupported"}, } fs.collapse() # Parent status is derived from the single child, so collapse is valid - assert "sync-token" in fs._server_features - assert "sync-token.delete" not in fs._server_features + assert "principal-search.by-name" in fs._server_features + assert "principal-search.by-name.self" not in fs._server_features def test_collapse_with_complex_dict_values(self) -> None: """Collapse should handle complex dictionary values""" @@ -263,20 +271,20 @@ def test_collapse_with_complex_dict_values(self) -> None: complex_value = { "support": "fragile", - "behaviour": "time-based", + "behaviour": "inconsistent", "extra": "metadata", } fs._server_features = { - "sync-token": complex_value.copy(), - "sync-token.delete": complex_value.copy(), + "principal-search.by-name": complex_value.copy(), + "principal-search.by-name.self": complex_value.copy(), } fs.collapse() # Both have same value, should collapse - assert "sync-token.delete" not in fs._server_features - assert fs._server_features["sync-token"] == complex_value + assert "principal-search.by-name.self" not in fs._server_features + assert fs._server_features["principal-search.by-name"] == complex_value def test_collapse_principal_search_real_scenario(self) -> None: """Test user's real scenario: principal-search subfeatures with same value should collapse""" @@ -302,140 +310,66 @@ def test_collapse_principal_search_real_scenario(self) -> None: assert "principal-search.list-all" not in fs._server_features assert "principal-search" in fs._server_features - def test_independent_subfeature_not_derived(self) -> None: - """Test that independent subfeatures (with explicit defaults) don't affect parent derivation""" - fs = FeatureSet() - - # Scenario: create-calendar.auto is set to unsupported, but it's an independent - # feature (has explicit default) and should NOT cause create-calendar to be - # derived as unsupported - fs._server_features = { - "create-calendar.auto": {"support": "unsupported"}, - } - - # create-calendar should return its default (full), NOT derive from .auto - result = fs.is_supported("create-calendar", return_type=dict) - assert result == {"support": "full"}, ( - f"create-calendar should default to 'full' when only independent " - f"subfeature .auto is set, but got {result}" - ) - - # Verify that the independent subfeature itself is still accessible - auto_result = fs.is_supported("create-calendar.auto", return_type=dict) - assert auto_result == {"support": "unsupported"} - - def test_parent_default_not_overridden_by_subfeature_derivation(self) -> None: - """Test that a parent with an explicit default is not overridden by subfeature derivation. + def test_collapse_independent_parent_not_collapsed(self) -> None: + """An independent parent (one with its own explicit default) is never + folded away by its children. - Zimbra scenario: create-calendar.set-displayname is unsupported, but - create-calendar has an explicit default of 'full'. The parent feature - represents an independent capability (calendar creation works), so the - subfeature status should not override the default. + sync-token carries a default, so even when its only child + sync-token.delete is unsupported the parent keeps its own (separately + probed) status: the two represent distinct capabilities and must not be + conflated. """ fs = FeatureSet() fs._server_features = { - "create-calendar.set-displayname": {"support": "unsupported"}, + "sync-token": {"support": "full"}, + "sync-token.delete": {"support": "unsupported"}, } - # create-calendar should return its default (full), NOT derive unsupported - # from .set-displayname - result = fs.is_supported("create-calendar", return_type=dict) - assert result == {"support": "full"}, ( - f"create-calendar should default to 'full' even when " - f".set-displayname is unsupported, but got {result}" - ) - - def test_hierarchical_vs_independent_subfeatures(self) -> None: - """Test that hierarchical subfeatures derive parent, but independent ones don't""" - fs = FeatureSet() - - # Hierarchical subfeatures: principal-search.by-name and principal-search.list-all - # These should cause parent to derive to "unknown" when mixed - fs.set_feature("principal-search.by-name", {"support": "unknown"}) - fs.set_feature("principal-search.list-all", {"support": "unsupported"}) - - # Should derive to "unknown" due to mixed hierarchical subfeatures - result = fs.is_supported("principal-search", return_type=dict) - assert result == {"support": "unknown"}, ( - f"principal-search should derive to 'unknown' from mixed hierarchical " - f"subfeatures, but got {result}" - ) - - # Now test independent subfeature: create-calendar.auto - # This should NOT affect create-calendar parent - fs2 = FeatureSet() - fs2.set_feature("create-calendar.auto", {"support": "unsupported"}) - - # Should return default, NOT derive from independent subfeature - result2 = fs2.is_supported("create-calendar", return_type=dict) - assert result2 == {"support": "full"}, ( - f"create-calendar should default to 'full' ignoring independent " - f"subfeature .auto, but got {result2}" - ) + fs.collapse() - def test_intermediate_feature_derives_from_children(self) -> None: - """Test that intermediate features (e.g. search.text) derive status from their children""" - # search.text has 4 direct children: case-sensitive, case-insensitive, - # substring, category (none have explicit defaults) + assert fs._server_features["sync-token"] == {"support": "full"} + assert fs._server_features["sync-token.delete"] == {"support": "unsupported"} - # All children set with mixed statuses -> derive "unknown" - fs = FeatureSet( - { - "search.text.case-sensitive": {"support": "unsupported"}, - "search.text.case-insensitive": {"support": "unsupported"}, - "search.text.substring": {"support": "unsupported"}, - "search.text.category": {"support": "fragile"}, - } - ) - assert not fs.is_supported("search.text") - assert fs.is_supported("search.text", return_type=dict) == {"support": "unknown"} + def test_collapse_does_not_alter_independent_sibling(self) -> None: + """collapse() must be lossless w.r.t. is_supported() for *every* + subfeature, including independent siblings. - # Partial children set with mixed non-positive statuses -> inconclusive, - # falls back to default ("full") - fs1b = FeatureSet( - { - "search.text.case-sensitive": {"support": "unsupported"}, - "search.text.category.substring": {"support": "fragile"}, - } - ) - assert fs1b.is_supported("search.text") + Regression for the save.duplicate-event compatibility-test breakage: + only save.duplicate-uid.cross-calendar was declared (ungraceful). The + grouping chain duplicate-uid -> save would have collapsed into an + explicit save=ungraceful, which the independent sibling + save.duplicate-event (own default "full") then inherited - flipping it + from "full" to "ungraceful". collapse() must not fold up to a parent + that has an independent child whose resolution would change. + """ + fs = FeatureSet({"save.duplicate-uid.cross-calendar": {"support": "ungraceful"}}) - # All children unsupported -> parent derives as "unsupported" - fs2 = FeatureSet( - { - "search.text.case-sensitive": {"support": "unsupported"}, - "search.text.case-insensitive": {"support": "unsupported"}, - "search.text.substring": {"support": "unsupported"}, - "search.text.category": {"support": "unsupported"}, - } - ) - assert not fs2.is_supported("search.text") - assert fs2.is_supported("search.text", return_type=dict) == {"support": "unsupported"} + before = fs.is_supported("save.duplicate-event", return_type=str) + assert before == "full" - # No children set -> falls back to default ("full") - fs3 = FeatureSet({}) - assert fs3.is_supported("search.text") + fs.collapse() - # Explicit parent value takes precedence over children - fs4 = FeatureSet( - { - "search.text": {"support": "full"}, - "search.text.case-sensitive": {"support": "unsupported"}, - } + after = fs.is_supported("save.duplicate-event", return_type=str) + assert after == "full", ( + f"collapse() changed save.duplicate-event from {before!r} to {after!r}" ) - assert fs4.is_supported("search.text") - + # The genuine observation is preserved either way. + assert fs.is_supported("save.duplicate-uid.cross-calendar", return_type=str) == "ungraceful" -class TestDeriveFromSubfeatures: - """Test _derive_from_subfeatures with partial and complete subfeature configs. - Uses search.recurrences which has two relevant children without defaults: - - search.recurrences.expanded - - search.recurrences.includes-implicit +class TestImplicitDerivation: + """Test is_supported() implicit derivation: parent→child, child→parent, explicit defaults. - The default for search.recurrences (a server-feature) is {"support": "full"}. + Covers: + - Children without explicit defaults derive the parent value. + - Parent set explicitly propagates down to unset children. + - Features with explicit defaults ignore subfeature derivation. + - Partial/incomplete child sets fall through to the feature's default. """ + ## TODO: the tests covering "all children" may need to be + ## protected against future additions in compatibility_hints.py + @pytest.mark.parametrize( "scenario, config, query, expected_support", [ @@ -448,6 +382,22 @@ class TestDeriveFromSubfeatures: "search.recurrences", "unsupported", ), + ( + "parent_unsupported", + { + "save-load": {"support": "unsupported"}, + }, + "save-load.event", + "unsupported", + ), + ( + "parent_with_explicit_default_unsupported", + { + "create-calendar": {"support": "unsupported"}, + }, + "create-calendar.auto", + "unsupported", + ), ( "all_children_supported", { @@ -482,6 +432,16 @@ class TestDeriveFromSubfeatures: "search.recurrences", "full", # any positive support → derive as supported ), + ( + ## Earlier logic had it that if a node has only one child, the parent should not be affected by the child, but if there are more children and all are unsupported, the parent is automatically flipped to unsupported. However, this special case logic should have been rendered obsolete by the new logic that every node having an explicit default is considered independent + "independent_feature_always_trumps", + { + "save-load.mutable.attendee-partstat": {"support": "unsupported"}, + "save-load.mutable.if-match-optional": {"support": "unsupported"}, + }, + "save-load.mutable", + "full", + ), ( "gmx_partial_unsupported_query_unset_sibling_child", { @@ -507,6 +467,47 @@ class TestDeriveFromSubfeatures: "search.recurrences.includes-implicit.todo", "full", ), + ( + "mixed_children_incomplete_unset_sibling_falls_to_default", + { + "save-load.todo": {"support": "full"}, + "save-load.journal": {"support": "unsupported"}, + }, + "save-load.event", + "full", # incomplete set: cannot derive anything about unset siblings + ), + ( + "explicit_default_overrides_children", + { + "create-calendar.auto": {"support": "unsupported"}, + "create-calendar.set-displayname": {"support": "unsupported"}, + }, + "create-calendar", + "full", # this feature does not depend on the sub-features + ), + ( + "partial_mixed_children_query_parent_falls_to_default", + { + "search.text.case-sensitive": {"support": "unsupported"}, + "search.text.case-insensitive": {"support": "full"}, + }, + "search.text", + "full", # partial+mixed: cannot conclude unsupported; default applies + ), + ( + ## Regression: setting a sibling child (search.text=full) caused + ## _derive_from_subfeatures on the grandparent "search" to return + ## full, which then bled into independent sibling features that have + ## their own explicit default. search.time-range.comp-type-optional + ## has default=unsupported and must not be overridden by a derived + ## (not explicitly set) ancestor status. + "derived_parent_does_not_bleed_into_independent_sibling", + { + "search.text": {"support": "full"}, + }, + "search.time-range.comp-type-optional", + "unsupported", # own explicit default, must not inherit derived "full" from ancestor + ), ], ids=lambda x: x if isinstance(x, str) and "_" in x else "", ) @@ -535,13 +536,16 @@ def test_string_resolves_profile(self) -> None: import caldav.compatibility_hints as ch result = _resolve_features("synology") - assert result is ch.synology + assert result == ch.synology + # deepcopy ensures the caller cannot mutate the shared profile + assert result is not ch.synology def test_string_with_prefix(self) -> None: import caldav.compatibility_hints as ch result = _resolve_features("compatibility_hints.synology") - assert result is ch.synology + assert result == ch.synology + assert result is not ch.synology def test_dict_without_base_passes_through(self) -> None: features = {"search.text": {"support": "unsupported"}} @@ -579,3 +583,50 @@ def test_base_with_prefix(self) -> None: assert result["sync-token"] == "full" # Original base feature should be overridden assert result["sync-token"] != "fragile" + + +class TestFeatureSetCompare: + """Test FeatureSet.compare(): declared (expected) vs observed feature sets.""" + + def test_matching_sets_no_mismatch(self) -> None: + expected = FeatureSet({"search.comp-type": {"support": "unsupported"}}) + observed = FeatureSet() + observed.set_feature("search.comp-type", "unsupported") + assert expected.compare(observed) == [] + + def test_subfeature_observed_default_conflicts_with_inherited_unsupported( + self, + ) -> None: + """Regression for the Infomaniak ``search.comp-type.optional`` blind spot. + + The parent ``search.comp-type`` is declared ``unsupported``, so the child + ``search.comp-type.optional`` inherits ``unsupported``. The server is + observed to *support* the child (``full``) - but ``full`` happens to be + the child's implicit default, so it is dropped from the compacted + observed dict. The mismatch must still be reported (it previously + slipped through because the feature was in neither compacted dict). + """ + expected = FeatureSet({"search.comp-type": {"support": "unsupported"}}) + observed = FeatureSet() + observed.set_feature("search.comp-type", "unsupported") + observed.set_feature("search.comp-type.optional") # -> {"support": "full"} + + mismatches = expected.compare(observed) + + by_feature = {m["feature"]: m for m in mismatches} + assert "search.comp-type.optional" in by_feature + assert by_feature["search.comp-type.optional"]["expected"] == "unsupported" + assert by_feature["search.comp-type.optional"]["observed"] == "full" + + def test_unprobed_declared_feature_is_not_flagged(self) -> None: + """A feature declared unsupported but never probed by the tester must not + be reported - we have no observation to contradict it.""" + expected = FeatureSet({"search.comp-type": {"support": "unsupported"}}) + observed = FeatureSet() # tester probed nothing + assert expected.compare(observed) == [] + + def test_fragile_and_unknown_are_ignored(self) -> None: + expected = FeatureSet({"search.comp-type": {"support": "unsupported"}}) + observed = FeatureSet() + observed.set_feature("search.comp-type", "fragile") + assert expected.compare(observed) == [] diff --git a/tests/test_discovery.py b/tests/test_discovery.py new file mode 100644 index 00000000..84b7a8a1 --- /dev/null +++ b/tests/test_discovery.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +""" +Unit tests for caldav.discovery — RFC 6764 service discovery. + +No network communication; DNS and HTTP are mocked. +""" + +from unittest import mock + +from caldav.discovery import discover_service + + +def _make_redirect_response(location: str, status_code: int = 302): + """Return a minimal mock HTTP response that redirects to *location*.""" + resp = mock.MagicMock() + resp.status_code = status_code + resp.headers = {"Location": location} + return resp + + +def _make_ok_response(): + resp = mock.MagicMock() + resp.status_code = 200 + resp.headers = {} + return resp + + +class TestRequireTLSDowngradeBlocked: + """§3.1: require_tls=True must be enforced on the well-known redirect target. + + _well_known_lookup always probes https:// but never received require_tls, + so a same-domain redirect to http:// passed the domain-validation check and + was returned as ServiceInfo(tls=False). discover_service returned it + unchecked — a misconfigured or MITM server could silently downgrade TLS. + """ + + @mock.patch("caldav.discovery.requests.get") + @mock.patch("caldav.discovery._srv_lookup", return_value=[]) + def test_http_redirect_rejected_when_require_tls(self, _srv, mock_get): + """discover_service(require_tls=True) must return None when the + well-known URI redirects to a plain-HTTP URL.""" + mock_get.return_value = _make_redirect_response( + "http://example.com/caldav/" # same domain, but HTTP + ) + + result = discover_service("example.com", require_tls=True) + + assert result is None, f"Expected None (TLS downgrade rejected), got {result}" + + @mock.patch("caldav.discovery.requests.get") + @mock.patch("caldav.discovery._srv_lookup", return_value=[]) + def test_http_redirect_accepted_when_require_tls_false(self, _srv, mock_get): + """discover_service(require_tls=False) must accept an HTTP redirect.""" + mock_get.return_value = _make_redirect_response("http://example.com/caldav/") + + result = discover_service("example.com", require_tls=False) + + assert result is not None + assert result.tls is False + assert result.url == "http://example.com/caldav/" + + @mock.patch("caldav.discovery.requests.get") + @mock.patch("caldav.discovery._srv_lookup", return_value=[]) + def test_https_redirect_accepted_when_require_tls(self, _srv, mock_get): + """HTTPS redirect is always accepted regardless of require_tls.""" + mock_get.return_value = _make_redirect_response("https://caldav.example.com/dav/") + + result = discover_service("example.com", require_tls=True) + + assert result is not None + assert result.tls is True + assert "caldav.example.com" in result.url + + @mock.patch("caldav.discovery.requests.get") + @mock.patch("caldav.discovery._srv_lookup", return_value=[]) + def test_cross_domain_http_redirect_also_rejected(self, _srv, mock_get): + """A cross-domain HTTP redirect must be rejected (domain check fires first, + but require_tls must also be a backstop).""" + mock_get.return_value = _make_redirect_response("http://evil.attacker.com/caldav/") + + result = discover_service("example.com", require_tls=True) + + assert result is None diff --git a/tests/test_examples.py b/tests/test_examples.py index 03c1adc8..0a993c07 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -52,6 +52,7 @@ def test_collation(self): with get_davclient() as dav_client: mycal = dav_client.principal().make_calendar(name="Test calendar") + assert mycal is not None collation_usage.run_examples() def test_rfc8764_test_conf(self): diff --git a/tests/test_jmap_unit.py b/tests/test_jmap_unit.py index 10e426e9..7cf316ae 100644 --- a/tests/test_jmap_unit.py +++ b/tests/test_jmap_unit.py @@ -1177,38 +1177,6 @@ def test_exdate(self): overrides = result["recurrenceOverrides"] assert any(v == {"excluded": True} for v in overrides.values()) - def test_rrule_until_utc_on_tzid_event_uses_event_local(self): - # §4.3 regression: a UTC UNTIL on a TZID event is a LocalDateTime in the - # event's own time zone per RFC 8984 — Europe/Berlin is UTC+2 in July, - # so 12:00:00Z must become 14:00:00 local, not stay 12:00:00 (which made - # the series end two hours early). - ical = _make_ical( - "DTSTART;TZID=Europe/Berlin:20240617T140000\r\n" - "DURATION:PT1H\r\n" - "SUMMARY:Recurring\r\n" - "RRULE:FREQ=WEEKLY;UNTIL=20240701T120000Z\r\n" - ) - result = ical_to_jscal(ical) - assert result["recurrenceRules"][0]["until"] == "2024-07-01T14:00:00" - # Round-trip: RFC 5545 §3.3.10 requires a UTC UNTIL for a TZID DTSTART, - # so the local until must convert back to ...120000Z (with the Z suffix). - back = jscal_to_ical(result) - assert "UNTIL=20240701T120000Z" in back - assert "UNTIL=20240701T140000" not in back - - def test_exdate_utc_on_tzid_event_uses_event_local(self): - # §4.3 regression for EXDATE: a UTC EXDATE on a TZID event becomes an - # override key in the event's local wall-clock. - ical = _make_ical( - "DTSTART;TZID=Europe/Berlin:20240617T140000\r\n" - "DURATION:PT1H\r\n" - "SUMMARY:Recurring\r\n" - "RRULE:FREQ=WEEKLY\r\n" - "EXDATE:20240624T120000Z\r\n" - ) - result = ical_to_jscal(ical) - assert "2024-06-24T14:00:00" in result["recurrenceOverrides"] - def test_valarm_relative(self): ical = _make_ical( "DTSTART:20240615T100000Z\r\n" @@ -2917,3 +2885,128 @@ def test_status_cancelled_round_trips(self): assert jscal.get("status") == "cancelled" round_tripped = jscal_to_ical(jscal) assert "STATUS:CANCELLED" in round_tripped + + +class TestLocalDateTimeIsEventLocal: + """Gate finding F3: RFC 8984 LocalDateTime slots (RRULE ``until``, + ``recurrenceOverrides`` keys) are expressed in the event's own timezone. + ``_format_local_dt()`` merely dropped the tzinfo, so a UTC value coming + off the wire was off by the UTC offset -- and the resulting floating + ``UNTIL`` against a TZID ``DTSTART`` is forbidden by RFC 5545 3.3.10.""" + + ICAL_HEAD = ( + "BEGIN:VCALENDAR\r\n" + "VERSION:2.0\r\n" + "PRODID:-//Test//Test//EN\r\n" + "BEGIN:VEVENT\r\n" + "UID:tz-local@example.com\r\n" + "DTSTAMP:20240101T000000Z\r\n" + "DTSTART;TZID=Europe/Berlin:20240615T090000\r\n" + "DURATION:PT1H\r\n" + ) + + def _convert(self, extra: str) -> dict: + return ical_to_jscal(self.ICAL_HEAD + extra + "END:VEVENT\r\nEND:VCALENDAR\r\n") + + def test_until_is_converted_to_event_timezone(self): + # 2024-06-30T07:00:00Z is 09:00 in Europe/Berlin (CEST, UTC+2). + jscal = self._convert("RRULE:FREQ=WEEKLY;UNTIL=20240630T070000Z\r\n") + assert jscal["recurrenceRules"][0]["until"] == "2024-06-30T09:00:00" + + def test_exdate_key_is_converted_to_event_timezone(self): + jscal = self._convert("RRULE:FREQ=WEEKLY\r\nEXDATE;VALUE=DATE-TIME:20240622T070000Z\r\n") + assert "2024-06-22T09:00:00" in jscal["recurrenceOverrides"] + + def test_until_round_trips_back_to_utc(self): + """The other half of the same rule: RFC 5545 3.3.10 requires a UTC UNTIL + whenever DTSTART carries a TZID, so the LocalDateTime ``until`` has to be + converted back -- not merely parsed as naive -- on the way out. Raised in + review of https://github.com/python-caldav/caldav/pull/688 ("the round-trip + back through jscal_to_ical produces UNTIL=20240701T120000 with no Z + suffix, which RFC 5545 3.3.10 forbids for TZID events").""" + jscal = self._convert("RRULE:FREQ=WEEKLY;UNTIL=20240630T070000Z\r\n") + assert jscal["recurrenceRules"][0]["until"] == "2024-06-30T09:00:00" + ical = jscal_to_ical(jscal) + assert "DTSTART;TZID=Europe/Berlin:20240615T090000" in ical + assert "UNTIL=20240630T070000Z" in ical, ( + "a TZID DTSTART requires a UTC UNTIL; got a floating one" + ) + + def test_recurrence_id_key_is_converted_to_event_timezone(self): + ical = ( + self.ICAL_HEAD + "RRULE:FREQ=WEEKLY\r\n" + "END:VEVENT\r\n" + "BEGIN:VEVENT\r\n" + "UID:tz-local@example.com\r\n" + "DTSTAMP:20240101T000000Z\r\n" + "RECURRENCE-ID:20240622T070000Z\r\n" + "DTSTART;TZID=Europe/Berlin:20240622T100000\r\n" + "SUMMARY:Moved\r\n" + "END:VEVENT\r\n" + "END:VCALENDAR\r\n" + ) + jscal = ical_to_jscal(ical) + assert "2024-06-22T09:00:00" in jscal["recurrenceOverrides"] + + def test_naive_dtstart_leaves_utc_until_alone(self): + """A floating DTSTART has no timezone to convert into; the value is + passed through rather than guessed at.""" + ical = ( + "BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//Test//Test//EN\r\n" + "BEGIN:VEVENT\r\nUID:floating@example.com\r\n" + "DTSTAMP:20240101T000000Z\r\n" + "DTSTART:20240615T090000\r\nDURATION:PT1H\r\n" + "RRULE:FREQ=WEEKLY;UNTIL=20240630T070000Z\r\n" + "END:VEVENT\r\nEND:VCALENDAR\r\n" + ) + jscal = ical_to_jscal(ical) + assert jscal["recurrenceRules"][0]["until"] == "2024-06-30T07:00:00" + + +class TestJMAPSessionRelease: + """Gate finding F9: the persistent HTTP session was released only by + ``__exit__``/``__aexit__``. The documented Quick Start does not use + ``with``, so a client built that way leaked its connection pool with no + way to release it short of dropping the object and hoping.""" + + def _make_client(self): + from caldav.jmap.client import JMAPClient + + return JMAPClient(url="https://jmap.example.com/.well-known/jmap", password="token") + + def test_close_releases_the_session(self): + client = self._make_client() + session = client._get_http_session() + assert session is not None + client.close() + assert client._http_session is None + + def test_close_is_idempotent(self): + client = self._make_client() + client._get_http_session() + client.close() + client.close() + assert client._http_session is None + + def test_context_manager_still_releases_the_session(self): + with self._make_client() as client: + client._get_http_session() + assert client._http_session is None + + def _make_async_client(self): + from caldav.jmap.async_client import AsyncJMAPClient + + return AsyncJMAPClient(url="https://jmap.example.com/.well-known/jmap", password="token") + + @pytest.mark.asyncio + async def test_aclose_releases_the_session(self): + client = self._make_async_client() + client._get_http_session() + await client.aclose() + assert client._http_session is None + + @pytest.mark.asyncio + async def test_async_context_manager_still_releases_the_session(self): + async with self._make_async_client() as client: + client._get_http_session() + assert client._http_session is None diff --git a/tests/test_protocol.py b/tests/test_protocol.py index c3a55375..9c8af001 100644 --- a/tests/test_protocol.py +++ b/tests/test_protocol.py @@ -208,6 +208,53 @@ def test_parse_sync_collection_response(self): assert result.deleted[0] == "/cal/deleted.ics" assert result.sync_token == "new-token" + def test_parse_sync_collection_generic_responsedescription(self): + """A 404 may carry an arbitrary . + + Per RFC 4918 is an optional child of + ; its text is server-defined. We must not hardcode + any particular server's wording (e.g. Stalwart's "No resources + found"). + """ + xml = b""" + + + /cal/gone.ics + HTTP/1.1 404 Not Found + The thing you asked for is not here anymore + + tok + """ + + result = DAVResponse.from_bytes(xml).parse_sync_collection() + + assert result.deleted == ["/cal/gone.ics"] + assert result.changed == [] + + def test_parse_sync_collection_generic_error(self): + """A 404 may carry an arbitrary element. + + Per RFC 4918 is an optional child of and its + children are server-defined. We must not hardcode any + particular server's error condition (e.g. purelymail's + {https://purelymail.com}does-not-exist). + """ + xml = b""" + + + /cal/gone.ics + HTTP/1.1 404 Not Found + + + tok + """ + + result = DAVResponse.from_bytes(xml).parse_sync_collection() + + assert result.deleted == ["/cal/gone.ics"] + assert result.changed == [] + def test_parse_complex_properties(self): """Parse complex properties like supported-calendar-component-set.""" xml = b""" @@ -252,3 +299,48 @@ def test_parse_complex_properties(self): # calendar-home-set - extracted href home_set = props["{urn:ietf:params:xml:ns:caldav}calendar-home-set"] assert home_set == "/calendars/user/" + + +class TestParserStackEquivalence: + """Guard the shared propstat-collection logic (code-review §5.7). + + The dataclass parsers (parse_propfind -> _extract_properties) and the + legacy _find_objects_and_props path must agree on the duplicated quirks + that used to be implemented twice: the "a 404 propstat means the property + is absent" skip and which prop elements get collected per href. + """ + + # one href with a found prop (200) and an absent prop (404 propstat), + # plus a second href that 404s entirely. + _xml = b""" + + + /cal/a/ + + A + HTTP/1.1 200 OK + + + + HTTP/1.1 404 Not Found + + + + /cal/missing/ + HTTP/1.1 404 Not Found + + """ + + def test_404_propstat_skipped_in_both_stacks(self): + dataclass_props = DAVResponse.from_bytes(self._xml).parse_propfind() + legacy = DAVResponse.from_bytes(self._xml)._find_objects_and_props() + + # dataclass stack: /cal/a/ keeps displayname, drops the 404 color prop + a_result = next(r for r in dataclass_props if r.href == "/cal/a/") + assert "{DAV:}displayname" in a_result.properties + assert "{http://apple.com/ns/ical/}calendar-color" not in a_result.properties + + # legacy stack: same set of collected prop tags for the same href + assert set(legacy["/cal/a/"].keys()) == set(a_result.properties.keys()) + # the entirely-404 href is present but carries no props in either stack + assert legacy["/cal/missing/"] == {} diff --git a/tests/test_schedule_tag.py b/tests/test_schedule_tag.py index c702b321..e325ac78 100644 --- a/tests/test_schedule_tag.py +++ b/tests/test_schedule_tag.py @@ -77,7 +77,7 @@ def _make_event_with_tag(schedule_tag='"tag-abc"'): class TestScheduleTagUnit: """ Pure unit tests — no server communication. - All tests in this class are expected to FAIL until the implementation is complete. + Covers the Schedule-Tag support added in v3.2.0 (RFC 6638 sections 3.2-3.3). """ # ------------------------------------------------------------------ # @@ -87,8 +87,6 @@ class TestScheduleTagUnit: def test_schedule_tag_property_returns_cached_value(self): """ CalendarObjectResource.schedule_tag should expose the cached tag. - - Currently fails because the property does not exist. """ event = _make_event_with_tag('"tag-xyz"') assert event.schedule_tag == '"tag-xyz"' @@ -160,16 +158,13 @@ def test_if_schedule_tag_match_header_sent_when_tag_cached(self, mocked): sent_headers = call_kwargs[1].get( "headers", call_kwargs[0][2] if len(call_kwargs[0]) > 2 else {} ) - assert "If-Schedule-Tag-Match" in sent_headers, ( - "If-Schedule-Tag-Match header was not sent; save() is still a no-op" - ) + assert "If-Schedule-Tag-Match" in sent_headers, "If-Schedule-Tag-Match header was not sent" assert sent_headers["If-Schedule-Tag-Match"] == '"tag-abc"' @mock.patch("caldav.davclient.requests.Session.request") - def test_if_schedule_tag_match_not_sent_when_flag_false(self, mocked): + def test_if_schedule_tag_match_not_sent_when_no_tag_cached(self, mocked): """ - save() without if_schedule_tag_match=True must NOT send the header, - even when a tag is cached. + save() must NOT send the header when no schedule-tag has been cached. """ ok_resp = _make_put_response(204) mocked.return_value = ok_resp @@ -197,12 +192,8 @@ def test_if_schedule_tag_match_not_sent_when_flag_false(self, mocked): def test_stale_schedule_tag_raises_mismatch_error(self, mocked): """ When the server returns 412 in response to an If-Schedule-Tag-Match - PUT, the client must raise ScheduleTagMismatchError (a subclass of - PutError). - - Currently fails: ScheduleTagMismatchError does not exist, and the - generic PutError is raised instead (or not at all because the header - is never sent). + PUT, the client must raise ScheduleTagMismatchError rather than the + generic PutError. """ mocked.return_value = _make_put_response(412) @@ -219,7 +210,7 @@ def test_412_without_schedule_tag_raises_put_error(self, mocked): mocked.return_value = _make_put_response(412) event = _make_event_with_tag(None) - # save() without if_schedule_tag_match — any 412 is a plain PutError + # No schedule-tag and no etag cached — any 412 is a plain PutError with pytest.raises(error.PutError): event.save() @@ -242,3 +233,30 @@ def test_schedule_tag_updated_in_props_after_successful_save(self, mocked): assert event.schedule_tag == new_tag, ( "schedule_tag prop not updated after successful conditional save" ) + + # ------------------------------------------------------------------ # + # 7. _post_put header handling (characterization for the dedup of # + # the block that used to be pasted twice) # + # ------------------------------------------------------------------ # + + @mock.patch("caldav.davclient.requests.Session.request") + def test_etag_captured_from_put_response(self, mocked): + """A PUT returning an Etag header must store it in props.""" + mocked.return_value = _make_put_response(201, {"Etag": '"etag-from-put"'}) + + event = _make_event_with_tag(None) + event.save() + + assert event.props[dav.GetEtag.tag] == '"etag-from-put"' + + @mock.patch("caldav.davclient.requests.Session.request") + def test_302_on_put_updates_url(self, mocked): + """A 302 in response to a PUT must follow the Location header.""" + mocked.return_value = _make_put_response( + 302, {"location": "http://cal.example.com/cal/moved.ics"} + ) + + event = _make_event_with_tag(None) + event.save() + + assert str(event.url) == "http://cal.example.com/cal/moved.ics" diff --git a/tests/test_search.py b/tests/test_search.py index 1dc7a3f2..ea1f1e42 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -10,12 +10,14 @@ from datetime import datetime, timezone from unittest import mock +from urllib.parse import quote import icalendar import pytest -from caldav import Event, Journal, Todo +from caldav import Calendar, Event, Journal, Todo from caldav.davclient import DAVClient +from caldav.lib import error from caldav.lib.url import URL from caldav.search import CalDAVSearcher @@ -140,6 +142,31 @@ END:VEVENT END:VCALENDAR""" +# Two events for §2.6 combined-is-logical-and tests +SPECIAL_EVENT = """BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Test//Test//EN +BEGIN:VEVENT +UID:special-event@example.com +DTSTAMP:20240101T120000Z +DTSTART:20240615T140000Z +DTEND:20240615T150000Z +SUMMARY:My Special Event +END:VEVENT +END:VCALENDAR""" + +UNRELATED_EVENT = """BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Test//Test//EN +BEGIN:VEVENT +UID:unrelated-event@example.com +DTSTAMP:20240101T120000Z +DTSTART:20240615T160000Z +DTEND:20240615T170000Z +SUMMARY:Unrelated Meeting +END:VEVENT +END:VCALENDAR""" + @pytest.fixture def mock_client() -> DAVClient: @@ -867,3 +894,627 @@ def mock_is_supported(feat, type_=bool): assert result == [event] calendar._request_report_build_resultlist.assert_called_once_with(full_xml, None, None) + + +class TestCompTypeLessSearchSplit: + """Gate findings F10 and F11: the comp-type split path. + + Since `search.time-range.comp-type-optional` and + `search.text.comp-type-optional` both default to `unsupported`, this + split now fires on essentially every server, so everything it gets wrong + is a common bug rather than an exotic one. + """ + + @staticmethod + def _split_client(mock_client: DAVClient) -> DAVClient: + def mock_is_supported(feat, type_=bool): + if feat == "search.comp-type.optional": + return False + if type_ is str: + return "full" + return True + + mock_client.features.is_supported = mock.Mock(side_effect=mock_is_supported) + mock_client.features.backward_compatibility_mode = False + return mock_client + + def _calendar(self, mock_client: DAVClient, objects: list) -> mock.Mock: + calendar = mock.Mock() + calendar.client = self._split_client(mock_client) + calendar._request_report_build_resultlist.return_value = (mock.Mock(), objects) + return calendar + + def test_event_false_does_not_raise_assertionerror( + self, mock_client: DAVClient, mock_url: str + ) -> None: + """F10: `search(event=False, ...)` tripped a bare, message-less + `AssertionError` -- on a perfectly legal call to a public API.""" + event = Event(client=mock_client, url=mock_url, data=SIMPLE_EVENT) + calendar = self._calendar(mock_client, [event]) + + searcher = CalDAVSearcher(event=False) + result = searcher.search(calendar) + + assert result == [event] + + def test_results_are_deduplicated_by_url(self, mock_client: DAVClient, mock_url: str) -> None: + """F11: one query per component type is sent, and a resource that + legally holds both a VEVENT and a VTODO comes back from more than one + of them. The `include_completed` split deduplicates by URL; this one + did not, so the object was returned twice.""" + event = Event(client=mock_client, url=mock_url, data=SIMPLE_EVENT) + calendar = self._calendar(mock_client, [event]) + + searcher = CalDAVSearcher() + result = searcher.search(calendar) + + assert calendar._request_report_build_resultlist.call_count > 1, ( + "expected one REPORT per component type" + ) + assert [o.url for o in result] == [event.url] + + def test_searcher_is_not_mutated_by_the_split( + self, mock_client: DAVClient, mock_url: str + ) -> None: + """F11: the split set `include_completed` on the caller's searcher -- + the issue-#650 class of bug, where a reused searcher silently changes + meaning between calls.""" + event = Event(client=mock_client, url=mock_url, data=SIMPLE_EVENT) + calendar = self._calendar(mock_client, [event]) + + searcher = CalDAVSearcher() + assert searcher.include_completed is None + searcher.search(calendar) + + assert searcher.include_completed is None, "search() mutated the caller's searcher" + + +class TestCompTypeOptionalTimeRange: + """Regression tests for https://github.com/python-caldav/caldav/issues/681. + + A CALDAV:time-range filter is only valid inside a comp-filter for + VEVENT/VTODO/VJOURNAL/VFREEBUSY/VALARM (RFC 4791 section 9.7), never + directly under the VCALENDAR comp-filter. When no component type is + specified, the library must NOT emit a under VCALENDAR - + it must split the search into one query per component type instead. + + SabreDAV-based servers (Baikal, Nextcloud, ...) reject the illegal query + with HTTP 400 "You cannot add time-range filters on the VCALENDAR + component". + """ + + _NS = {"C": "urn:ietf:params:xml:ns:caldav"} + + def _vcalendar_timerange_children(self, xml): + """Return any elements that are direct children of the + VCALENDAR comp-filter (i.e. the RFC-illegal placement).""" + from lxml import etree + + x = xml.xmlelement() if hasattr(xml, "xmlelement") else None + if x is None: + return [] + return x.xpath('//C:comp-filter[@name="VCALENDAR"]/C:time-range', namespaces=self._NS) + + def test_untyped_timerange_search_splits_per_comptype( + self, mock_client: DAVClient, mock_url: str + ) -> None: + """Backward-compat mode: an untyped time-range search must split into + per-component queries rather than placing under VCALENDAR.""" + from caldav.compatibility_hints import FeatureSet + + mock_client.features = FeatureSet(None) # default / backward-compat (what end users get) + + calls = [] + calendar = mock.Mock() + calendar.client = mock_client + + def rep(xml, comp_cls, props=None): + calls.append(xml) + return (mock.Mock(), []) + + calendar._request_report_build_resultlist.side_effect = rep + + searcher = CalDAVSearcher( + start=datetime(2024, 1, 1, tzinfo=timezone.utc), + end=datetime(2024, 2, 1, tzinfo=timezone.utc), + ) + searcher.search(calendar) + + assert calls, "no REPORT was issued" + for xml in calls: + assert not self._vcalendar_timerange_children(xml), ( + "time-range must not be placed directly under VCALENDAR" + ) + ## split into one query per component type (VEVENT/VTODO/VJOURNAL) + assert len(calls) == 3 + + def test_reactive_workaround_on_vcalendar_timerange_rejection( + self, mock_client: DAVClient, mock_url: str + ) -> None: + """If the feature is (mis)configured as supported and the server rejects + the comp-type-less time-range query with a 400, the library must retry + by splitting into per-component queries.""" + from caldav.compatibility_hints import FeatureSet + + ## Feature explicitly configured as supported, so the library optimistically + ## sends the comp-type-less time-range query that SabreDAV rejects. + mock_client.features = FeatureSet( + {"search.time-range.comp-type-optional": {"support": "full"}} + ) + + event = Event(client=mock_client, url=mock_url, data=SIMPLE_EVENT) + calendar = mock.Mock() + calendar.client = mock_client + + def rep(xml, comp_cls, props=None): + if self._vcalendar_timerange_children(xml): + raise error.ReportError( + "400 Bad Request - You cannot add time-range filters on the VCALENDAR component" + ) + return (mock.Mock(), [event] if comp_cls is Event else []) + + calendar._request_report_build_resultlist.side_effect = rep + + searcher = CalDAVSearcher( + start=datetime(2024, 1, 1, tzinfo=timezone.utc), + end=datetime(2024, 2, 1, tzinfo=timezone.utc), + ) + result = searcher.search(calendar) + + assert result == [event] + + def test_compatibility_workarounds_false_sends_raw_query( + self, mock_client: DAVClient, mock_url: str + ) -> None: + """compatibility_workarounds=False must disable the comp-type split and send + the comp-type-less time-range query verbatim (single REPORT), so the + compatibility checker can observe the raw server behaviour.""" + from caldav.compatibility_hints import FeatureSet + + mock_client.features = FeatureSet(None) + + calls = [] + calendar = mock.Mock() + calendar.client = mock_client + + def rep(xml, comp_cls, props=None): + calls.append(xml) + return (mock.Mock(), []) + + calendar._request_report_build_resultlist.side_effect = rep + + searcher = CalDAVSearcher( + start=datetime(2024, 1, 1, tzinfo=timezone.utc), + end=datetime(2024, 2, 1, tzinfo=timezone.utc), + ) + searcher.search(calendar, post_filter=False, compatibility_workarounds=False) + + ## exactly one report, sent verbatim with the (RFC-questionable) time-range + ## directly under VCALENDAR - no splitting + assert len(calls) == 1 + assert self._vcalendar_timerange_children(calls[0]) + + +class TestCompTypeOptionalPropFilter: + """Regression tests for https://github.com/python-caldav/caldav/issues/681. + + A CALDAV:prop-filter (CATEGORIES, SUMMARY, ...) placed directly under the + VCALENDAR comp-filter filters on VCALENDAR's own properties, which do not + include component properties like CATEGORIES. Servers (e.g. Xandikos, and + SabreDAV-based servers) therefore match nothing. When no component type is + specified, the library must split the search into one query per component + type so the prop-filter lands inside a VEVENT/VTODO/VJOURNAL comp-filter. + """ + + _NS = {"C": "urn:ietf:params:xml:ns:caldav"} + + def _vcalendar_propfilter_children(self, xml): + """Return any elements that are direct children of the + VCALENDAR comp-filter (i.e. filtering on a non-existent VCALENDAR prop).""" + x = xml.xmlelement() if hasattr(xml, "xmlelement") else None + if x is None: + return [] + return x.xpath('//C:comp-filter[@name="VCALENDAR"]/C:prop-filter', namespaces=self._NS) + + def test_untyped_propfilter_search_splits_per_comptype( + self, mock_client: DAVClient, mock_url: str + ) -> None: + """Backward-compat mode: an untyped property-filter search must split into + per-component queries rather than placing under VCALENDAR.""" + from caldav.compatibility_hints import FeatureSet + + mock_client.features = FeatureSet(None) # default / backward-compat + + calls = [] + calendar = mock.Mock() + calendar.client = mock_client + + def rep(xml, comp_cls, props=None): + calls.append(xml) + return (mock.Mock(), []) + + calendar._request_report_build_resultlist.side_effect = rep + + searcher = CalDAVSearcher() + searcher.add_property_filter("SUMMARY", "meeting") + searcher.search(calendar) + + assert calls, "no REPORT was issued" + for xml in calls: + assert not self._vcalendar_propfilter_children(xml), ( + "prop-filter must not be placed directly under VCALENDAR" + ) + ## split into one query per component type (VEVENT/VTODO/VJOURNAL) + assert len(calls) == 3 + + +class TestSearchDriverExceptionHandling: + """The search() driver runs the generator's yielded actions and must feed any + exception raised by an action back INTO the generator (via gen.throw()) so the + search logic's own try/except blocks can act on it. Without this, the + generator's error-handling branches (issue #681 fallback, per-object load + error handling, ...) would be dead code. + """ + + def _mock_features_all_supported(self, mock_client): + def mock_is_supported(feat, type_=bool): + if type_ is str: + return "full" + return True + + mock_client.features.is_supported = mock.Mock(side_effect=mock_is_supported) + mock_client.features.backward_compatibility_mode = False + + def test_unloaded_object_not_returned_by_batch_is_excluded( + self, mock_client: DAVClient, mock_url: str + ) -> None: + """Objects that remain unloaded after _batch_load_objects are excluded from results. + + With the old per-object LOAD_OBJECT loop, exceptions were thrown into the generator + to skip objects. With the new LOAD_OBJECTS_BATCH approach, _batch_load_objects + handles errors internally; objects it cannot populate remain unloaded and are + filtered out by the post-batch is_loaded()/has_component() check. + """ + self._mock_features_all_supported(mock_client) + + good = Event(client=mock_client, url=mock_url + "/good", data=SIMPLE_EVENT) + bad = Event(client=mock_client, url=mock_url + "/bad") # unloaded: server skipped it + + calendar = mock.Mock() + calendar.client = mock_client + calendar._request_report_build_resultlist.return_value = (mock.Mock(), [good, bad]) + # Default mock._batch_load_objects does nothing: bad remains unloaded + + searcher = CalDAVSearcher(event=True) + result = searcher.search(calendar) + + assert good in result + assert bad not in result + + def test_unhandled_action_exception_propagates( + self, mock_client: DAVClient, mock_url: str + ) -> None: + """An exception the generator does NOT catch must still propagate out of + search() (gen.throw re-raises it) rather than being swallowed.""" + self._mock_features_all_supported(mock_client) + + calendar = mock.Mock() + calendar.client = mock_client + calendar._request_report_build_resultlist.side_effect = RuntimeError("boom") + + searcher = CalDAVSearcher(event=True) + with pytest.raises(RuntimeError, match="boom"): + searcher.search(calendar) + + +class TestCombinedIsLogicalAndWorkaround: + """§2.6: combined-is-logical-and workaround must apply property filters client-side. + + When search.combined-is-logical-and is False, the workaround strips all + property filters from the server query (sending only the time range) and + must apply them client-side afterward. The bug passed the ambient + post_filter=None instead of True, so _filter_search_results short-circuited + and returned all objects in the time range unfiltered. + """ + + def _make_calendar_with_features(self) -> "tuple": + """Return (client, calendar) with combined-is-logical-and: unsupported.""" + from caldav import Calendar + from caldav.compatibility_hints import FeatureSet + + features = FeatureSet( + { + "search.combined-is-logical-and": "unsupported", + "search.text": "full", + "search.text.substring": "full", + "search.text.case-sensitive": "full", + "search.time-range.accurate": "full", + "search.unlimited-time-range": "full", + } + ) + from caldav.davclient import DAVClient + + client = DAVClient(url="https://cal.example.com/") + client.features = features + cal = Calendar(client=client, url="https://cal.example.com/cal/") + return client, cal + + def test_summary_filter_applied_client_side(self) -> None: + """Time-range + SUMMARY filter on combined-is-logical-and:unsupported server + must return only events whose SUMMARY matches — not every event in the range.""" + client, cal = self._make_calendar_with_features() + + special = Event( + client=client, + url="https://cal.example.com/cal/special.ics", + data=SPECIAL_EVENT, + parent=cal, + ) + unrelated = Event( + client=client, + url="https://cal.example.com/cal/unrelated.ics", + data=UNRELATED_EVENT, + parent=cal, + ) + + mock_response = mock.MagicMock() + cal._request_report_build_resultlist = mock.Mock( + return_value=(mock_response, [special, unrelated]) + ) + + start = datetime(2024, 6, 15, 0, 0, tzinfo=timezone.utc) + end = datetime(2024, 6, 16, 0, 0, tzinfo=timezone.utc) + searcher = CalDAVSearcher(event=True, start=start, end=end) + searcher.add_property_filter("SUMMARY", "Special", operator="contains") + + results = searcher.search(cal) + + summaries = [str(r.icalendar_component["SUMMARY"]) for r in results] + assert len(results) == 1, f"Expected 1 result, got {len(results)}: {summaries}" + assert "Special" in summaries[0] + + +class TestExactMatchOperator: + """§2.8: operator='==' must be enforced client-side via post-filtering. + + The docstring documents that '==' means exact match enforced client-side, + but no code path inspected the '==' operator — post_filter was never set + for '==' searches, so server substring semantics leaked through. + """ + + _exact_match_event = """BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Test//Test//EN +BEGIN:VEVENT +UID:exact-event@example.com +DTSTAMP:20240101T120000Z +DTSTART:20240615T140000Z +DTEND:20240615T150000Z +SUMMARY:rain +END:VEVENT +END:VCALENDAR""" + + _substring_event = """BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Test//Test//EN +BEGIN:VEVENT +UID:substring-event@example.com +DTSTAMP:20240101T120000Z +DTSTART:20240615T160000Z +DTEND:20240615T170000Z +SUMMARY:Training session +END:VEVENT +END:VCALENDAR""" + + def _make_calendar_with_features(self): + from caldav.lib.url import URL + + client = mock.Mock(spec=DAVClient) + client.url = URL("https://cal.example.com/") + features = mock.Mock() + features.is_supported = mock.Mock(return_value=False) + client.features = features + cal = mock.Mock() + cal.client = client + cal.url = URL("https://cal.example.com/cal/") + return client, cal + + def test_exact_match_excludes_substrings(self): + """operator='==' must exclude events where the value is a substring of the summary.""" + client, cal = self._make_calendar_with_features() + + exact_ev = Event( + client=client, + url="https://cal.example.com/cal/exact.ics", + data=self._exact_match_event, + parent=cal, + ) + substring_ev = Event( + client=client, + url="https://cal.example.com/cal/substring.ics", + data=self._substring_event, + parent=cal, + ) + + cal._request_report_build_resultlist = mock.Mock( + return_value=(mock.MagicMock(), [exact_ev, substring_ev]) + ) + + searcher = CalDAVSearcher(event=True) + searcher.add_property_filter("SUMMARY", "rain", operator="==") + results = searcher.search(cal) + + summaries = [str(r.icalendar_component["SUMMARY"]) for r in results] + assert len(results) == 1, f"Expected 1 result (exact), got {len(results)}: {summaries}" + assert results[0].icalendar_component["SUMMARY"] == "rain" + + +class TestBatchLoadObjects: + """Calendar._batch_load_objects fetches N objects with one _multiget REPORT. + + Replaces the per-object LOAD_OBJECT loop in search post-processing (issue #5.4). + Before this fix, a 200-event search triggered 200 individual GET requests; + after, one batched calendar-multiget REPORT is sent. + """ + + CAL_URL = "https://cal.example.com/cal/" + + def _make_calendar(self) -> Calendar: + client = mock.Mock(spec=DAVClient) + client.url = URL("https://cal.example.com/") + return Calendar(client=client, url=self.CAL_URL) + + def test_batch_load_calls_multiget_once(self) -> None: + """_batch_load_objects calls _multiget exactly once for N unloaded objects.""" + cal = self._make_calendar() + ev1 = Event(client=cal.client, url=self.CAL_URL + "ev1.ics") + ev2 = Event(client=cal.client, url=self.CAL_URL + "ev2.ics") + + cal._multiget = mock.Mock( + return_value=iter([("/cal/ev1.ics", SIMPLE_EVENT), ("/cal/ev2.ics", SIMPLE_EVENT)]) + ) + + cal._batch_load_objects([ev1, ev2]) + + assert cal._multiget.call_count == 1 + + def test_batch_load_populates_object_data(self) -> None: + """_batch_load_objects sets data on matched unloaded objects.""" + cal = self._make_calendar() + ev = Event(client=cal.client, url=self.CAL_URL + "ev1.ics") + assert not ev.is_loaded() + + cal._multiget = mock.Mock(return_value=iter([("/cal/ev1.ics", SIMPLE_EVENT)])) + + cal._batch_load_objects([ev]) + + assert ev.is_loaded() + + def test_batch_load_skips_already_loaded_in_multiget_request(self) -> None: + """Already-loaded objects are not included in the multiget URL list.""" + cal = self._make_calendar() + loaded = Event(client=cal.client, url=self.CAL_URL + "loaded.ics", data=SIMPLE_EVENT) + unloaded = Event(client=cal.client, url=self.CAL_URL + "unloaded.ics") + + cal._multiget = mock.Mock(return_value=iter([("/cal/unloaded.ics", SIMPLE_EVENT)])) + + cal._batch_load_objects([loaded, unloaded]) + + assert cal._multiget.called + urls_passed = [str(u) for u in cal._multiget.call_args[0][0]] + assert not any("loaded.ics" in u and "unloaded" not in u for u in urls_passed) + + def test_batch_load_matches_href_with_at_sign(self) -> None: + """UIDs are conventionally `@` and the resource is + conventionally named `.ics`, so an "@" in the href is the common + case. The result keys were normalised with `safe="/:@"` (literal "@") + while the object URLs come out of `quote()` with the default + `safe="/"` (percent-encoded "%40"), so nothing matched, nothing was + assigned, and the objects were then silently dropped by the + `has_component()` filter -- `search()` returned [] with no exception + and no log line.""" + cal = self._make_calendar() + href = "/cal/" + quote("20200516T060000Z-123401@example.com.ics") + assert "%40" in href + ev = Event(client=cal.client, url=cal.url.join(href)) + + cal._multiget = mock.Mock(return_value=iter([(href, SIMPLE_EVENT)])) + + cal._batch_load_objects([ev]) + + assert ev.is_loaded() + + def test_batch_load_matches_unencoded_at_sign_in_href(self) -> None: + """The same, for a server that echoes the "@" back unencoded.""" + cal = self._make_calendar() + ev = Event( + client=cal.client, + url=cal.url.join("/cal/" + quote("20200516T060000Z-123401@example.com.ics")), + ) + + cal._multiget = mock.Mock( + return_value=iter([("/cal/20200516T060000Z-123401@example.com.ics", SIMPLE_EVENT)]) + ) + + cal._batch_load_objects([ev]) + + assert ev.is_loaded() + + def test_batch_load_fallback_on_multiget_error(self) -> None: + """When _multiget raises, _batch_load_objects falls back to individual obj.load().""" + cal = self._make_calendar() + ev = Event(client=cal.client, url=self.CAL_URL + "ev1.ics") + ev.load = mock.Mock() + cal._multiget = mock.Mock(side_effect=RuntimeError("network error")) + + cal._batch_load_objects([ev]) + + ev.load.assert_called() + + +class TestSearchBatchLoadIntegration: + """search() must use one batched _multiget call instead of N individual load() calls. + + The fix replaces the per-object LOAD_OBJECT loops in _search_impl with a single + LOAD_OBJECTS_BATCH action that delegates to Calendar._batch_load_objects. + """ + + def _make_calendar_all_supported(self) -> "tuple[mock.Mock, Calendar]": + """Return (client, calendar) with all features marked supported.""" + client = mock.Mock(spec=DAVClient) + client.url = URL("https://cal.example.com/") + + def mock_is_supported(feat: str, type_: type = bool): + return "full" if type_ is str else True + + client.features.is_supported = mock.Mock(side_effect=mock_is_supported) + client.features.backward_compatibility_mode = False + + cal = Calendar(client=client, url="https://cal.example.com/cal/") + return client, cal + + def test_search_calls_multiget_once_not_n_individual_loads(self) -> None: + """For N unloaded search results, search() must call _multiget once, not load() N times. + + With the old per-object LOAD_OBJECT loop, 2 unloaded objects caused 2 GET requests. + With the new LOAD_OBJECTS_BATCH action, a single calendar-multiget REPORT is issued. + """ + client, cal = self._make_calendar_all_supported() + + ev1 = Event(client=client, url="https://cal.example.com/cal/ev1.ics", parent=cal) + ev2 = Event(client=client, url="https://cal.example.com/cal/ev2.ics", parent=cal) + ev1.load = mock.Mock(return_value=ev1) + ev2.load = mock.Mock(return_value=ev2) + + cal._request_report_build_resultlist = mock.Mock(return_value=(mock.Mock(), [ev1, ev2])) + cal._multiget = mock.Mock( + return_value=iter( + [ + ("/cal/ev1.ics", SIMPLE_EVENT), + ("/cal/ev2.ics", SIMPLE_EVENT), + ] + ) + ) + + searcher = CalDAVSearcher(event=True) + searcher.search(cal) + + assert cal._multiget.call_count == 1, ( + f"Expected one batched _multiget call, got {cal._multiget.call_count}. " + "search() is still loading unloaded objects one-by-one." + ) + + def test_search_results_populated_after_batch_load(self) -> None: + """search() returns populated objects when batch-loading succeeds.""" + client, cal = self._make_calendar_all_supported() + + ev = Event(client=client, url="https://cal.example.com/cal/ev1.ics", parent=cal) + ev.load = mock.Mock(return_value=ev) + + cal._request_report_build_resultlist = mock.Mock(return_value=(mock.Mock(), [ev])) + cal._multiget = mock.Mock(return_value=iter([("/cal/ev1.ics", SIMPLE_EVENT)])) + + searcher = CalDAVSearcher(event=True) + results = searcher.search(cal) + + assert len(results) == 1 diff --git a/tests/test_servers.yaml.example b/tests/test_servers.yaml.example deleted file mode 100644 index c07af076..00000000 --- a/tests/test_servers.yaml.example +++ /dev/null @@ -1,138 +0,0 @@ -# Test server configuration for caldav tests -# -# Copy this file to test_servers.yaml and customize for your setup. -# See tests/README.md for documentation. -# -# Environment variables can be used with ${VAR} or ${VAR:-default} syntax. - -test-servers: - # ========================================================================= - # Embedded servers (run in-process, no external setup required) - # ========================================================================= - - radicale: - type: embedded - enabled: true - host: ${RADICALE_HOST:-localhost} - port: ${RADICALE_PORT:-5232} - username: user1 - password: "" - - xandikos: - type: embedded - enabled: true - host: ${XANDIKOS_HOST:-localhost} - port: ${XANDIKOS_PORT:-8993} - username: sometestuser - password: "" - - # ========================================================================= - # Docker servers (require docker-compose, see docker-test-servers/) - # ========================================================================= - # - # Set enabled to: - # - true: always enable - # - false: always disable - # - "auto": enable if docker is available (default for docker servers) - - baikal: - type: docker - enabled: ${TEST_BAIKAL:-auto} - host: ${BAIKAL_HOST:-localhost} - port: ${BAIKAL_PORT:-8800} - username: ${BAIKAL_USERNAME:-testuser} - password: ${BAIKAL_PASSWORD:-testpass} - # Path within the CalDAV server - # path: /dav.php - - nextcloud: - type: docker - enabled: ${TEST_NEXTCLOUD:-false} - host: ${NEXTCLOUD_HOST:-localhost} - port: ${NEXTCLOUD_PORT:-8801} - username: ${NEXTCLOUD_USERNAME:-testuser} - password: ${NEXTCLOUD_PASSWORD:-testpass} - - cyrus: - type: docker - enabled: ${TEST_CYRUS:-false} - host: ${CYRUS_HOST:-localhost} - port: ${CYRUS_PORT:-8802} - username: ${CYRUS_USERNAME:-testuser@test.local} - password: ${CYRUS_PASSWORD:-testpassword} - - sogo: - type: docker - enabled: ${TEST_SOGO:-false} - host: ${SOGO_HOST:-localhost} - port: ${SOGO_PORT:-8803} - username: ${SOGO_USERNAME:-testuser} - password: ${SOGO_PASSWORD:-testpassword} - - bedework: - type: docker - enabled: ${TEST_BEDEWORK:-false} - host: ${BEDEWORK_HOST:-localhost} - port: ${BEDEWORK_PORT:-8804} - username: ${BEDEWORK_USERNAME:-admin} - password: ${BEDEWORK_PASSWORD:-bedework} - - davical: - type: docker - enabled: ${TEST_DAVICAL:-false} - host: ${DAVICAL_HOST:-localhost} - port: ${DAVICAL_PORT:-8805} - username: ${DAVICAL_USERNAME:-admin} - password: ${DAVICAL_PASSWORD:-davical} - - # ========================================================================= - # External/private servers (your own CalDAV server) - # ========================================================================= - # - # Uncomment and configure to test against your own server: - - # my-server: - # type: external - # enabled: true - # url: ${CALDAV_URL:-https://caldav.example.com/dav/} - # username: ${CALDAV_USERNAME} - # password: ${CALDAV_PASSWORD} - # # Optional: SSL verification (default: true) - # ssl_verify: true - # # Optional: specify server limitations/features - # features: - # - no-expand # Server doesn't support EXPAND - # - no-sync-token # Server doesn't support sync tokens - # - no-freebusy # Server doesn't support freebusy queries - -# ========================================================================= -# RFC6638 scheduling test users (optional) -# ========================================================================= -# -# For testing calendar scheduling (meeting invites, etc.), define at least -# three users on the same CalDAV server that can send invites to each other. -# This section lives at the TOP LEVEL (not under test-servers). -# -# Cyrus (pre-creates user1-user5 with password 'x'): -# rfc6638_users: -# - url: http://localhost:8802/dav/calendars/user/user1 -# username: user1 -# password: x -# - url: http://localhost:8802/dav/calendars/user/user2 -# username: user2 -# password: x -# - url: http://localhost:8802/dav/calendars/user/user3 -# username: user3 -# password: x -# -# Baikal (user1-user3 are in the pre-seeded db.sqlite, passwords testpass1-3): -# rfc6638_users: -# - url: http://localhost:8800/dav.php/ -# username: user1 -# password: testpass1 -# - url: http://localhost:8800/dav.php/ -# username: user2 -# password: testpass2 -# - url: http://localhost:8800/dav.php/ -# username: user3 -# password: testpass3 diff --git a/tests/test_servers/registry.py b/tests/test_servers/registry.py index 4ebd8bea..dccf331c 100644 --- a/tests/test_servers/registry.py +++ b/tests/test_servers/registry.py @@ -187,6 +187,16 @@ def load_from_config(self, config: dict) -> None: continue if not server_config.get("enabled", True): + # auto_discover() has already registered every + # docker-test-servers/* directory by the time this runs, so + # merely skipping the config entry would leave the server + # enabled - the opposite of what the config file says. + # Disable the registered instance instead. + already_registered = next( + (k for k in self._servers if k.lower() == name.lower()), None + ) + if already_registered is not None: + self._servers[already_registered].config["enabled"] = False continue # Keys that only carry test-specific metadata, not connection config. diff --git a/tests/test_servers/test_config_loader.py b/tests/test_servers/test_config_loader.py index 97230ba9..3d59df8f 100644 --- a/tests/test_servers/test_config_loader.py +++ b/tests/test_servers/test_config_loader.py @@ -124,3 +124,56 @@ def test_rfc6638_users_only_config(self, tmp_path: Path) -> None: cfg = load_test_server_config(str(config_file)) assert "rfc6638_users" in cfg assert cfg["rfc6638_users"][0]["url"] == "http://localhost:8802/dav/calendars/user/user1" + + +class TestEnabledFalseDisablesRegisteredServer: + """Gate finding F16: `enabled: false` was a no-op for docker servers. + + ``auto_discover()`` registers every ``docker-test-servers/*`` directory + before ``load_from_config()`` runs, and the ``enabled`` check only skipped + the *config entry* -- the already-registered server stayed enabled. So a + user who copied the example file and had Docker got every server run + anyway, the opposite of what the file says. + """ + + def _registry_with_registered_server(self): + from .base import TestServer + from .registry import ServerRegistry + + class _FakeDocker(TestServer): + server_type = "docker" + + def start(self) -> None: ... + + def stop(self) -> None: ... + + def is_running(self) -> bool: + return True + + def is_accessible(self) -> bool: + return True + + @property + def url(self) -> str: + return "http://localhost:8800/" + + registry = ServerRegistry() + registry.register(_FakeDocker({"name": "baikal", "host": "localhost", "port": 8800})) + return registry + + def test_enabled_false_disables_an_autodiscovered_server(self, monkeypatch) -> None: + monkeypatch.delenv("PYTHON_CALDAV_TEST_DOCKER", raising=False) + registry = self._registry_with_registered_server() + assert [s.name for s in registry.enabled_servers()] == ["baikal"] + + registry.load_from_config({"baikal": {"type": "docker", "enabled": False}}) + + assert registry.enabled_servers() == [] + + def test_enabled_true_leaves_it_alone(self, monkeypatch) -> None: + monkeypatch.delenv("PYTHON_CALDAV_TEST_DOCKER", raising=False) + registry = self._registry_with_registered_server() + + registry.load_from_config({"baikal": {"type": "docker", "enabled": True}}) + + assert [s.name for s in registry.enabled_servers()] == ["baikal"] diff --git a/tests/test_substring_workaround.py b/tests/test_substring_workaround.py index 42123415..656dbc18 100644 --- a/tests/test_substring_workaround.py +++ b/tests/test_substring_workaround.py @@ -128,6 +128,8 @@ def capture_xml(xml, *args, **kwargs): # This SHOULD trigger the workaround result = searcher.search(calendar) + ## the capturing stub returns no objects, so the post-filter has none to keep + assert result == [] # Verify that at least one query was sent assert len(xml_queries) >= 1 @@ -169,10 +171,12 @@ def test_mixed_explicit_and_implicit_operators() -> None: def check_build(*args, **kwargs): xml, comp_class = original_build(*args, **kwargs) xml_str = str(xml) - # SUMMARY should be in the query (implicit, server decides) - # LOCATION should NOT be in query (explicit contains, removed) - # STATUS should be in the query (explicit ==, supported) - # Note: Properties are lowercased in internal storage + ## These three were comments describing what the query should look like, + ## with nothing checking any of it. Asserted now. Note that properties + ## are lowercased in internal storage but emitted uppercased. + assert 'name="SUMMARY"' in xml_str, "implicit operator: the server decides" + assert 'name="LOCATION"' not in xml_str, "explicit contains: filtered client-side" + assert 'name="STATUS"' in xml_str, "explicit ==: the server can do this one" return xml, comp_class searcher.build_search_xml_query = check_build diff --git a/tests/test_vcal.py b/tests/test_vcal.py index 4593864a..1ec0c484 100644 --- a/tests/test_vcal.py +++ b/tests/test_vcal.py @@ -131,6 +131,23 @@ def create_and_validate(**args): ) assert re.search(b"DTSTART(;VALUE=DATE-TIME)?:20321010T101010Z", some_ical) + ## ical_fragment with alarm_* props: fragment must land in VEVENT, not VALARM (§2.2) + raw_ical = create_ical( + summary="alarm-test", + dtstart=datetime(2032, 10, 10, 10, 10, 10, tzinfo=utc), + duration=timedelta(hours=1), + alarm_action="DISPLAY", + alarm_description="reminder", + alarm_trigger=timedelta(minutes=-15), + ical_fragment="RRULE:FREQ=DAILY;COUNT=3", + ) + raw_bytes = to_wire(raw_ical) + assert b"RRULE:FREQ=DAILY" in raw_bytes, "ical_fragment must appear in output" + assert b"BEGIN:VALARM" in raw_bytes, "alarm must be present" + end_valarm_pos = raw_bytes.index(b"END:VALARM") + rrule_pos = raw_bytes.index(b"RRULE:FREQ=DAILY") + assert rrule_pos > end_valarm_pos, "RRULE must not be inside VALARM" + def test_vcal_fixups(self): """ There is an obscure function lib.vcal that attempts to fix up @@ -281,6 +298,124 @@ def test_vcal_fixups(self): for ical in non_broken_ical: assert vcal.fix(ical) == ical + def test_trailing_whitespace_stripped_per_line(self) -> None: + """Bug §2.3: re.sub(' *$', '', fixed) without re.MULTILINE only strips + trailing spaces at the very end of the document, leaving per-line + trailing spaces intact. Trailing whitespace on a line that is not + continued by a folded line cannot be part of the value, so it is + stripped.""" + ical = ( + "BEGIN:VCALENDAR\n" + "VERSION:2.0\n" + "BEGIN:VEVENT\n" + "UID:test\n" + "DTSTAMP:20190103T070319Z\n" + "DTSTART:20190117T180000Z\n" + "SUMMARY:test \n" + "LOCATION:somewhere \n" + "END:VEVENT\n" + "END:VCALENDAR\n" + ) + + fixed = vcal.fix(ical) + assert not re.search(r" +\n", fixed), ( + "fix() must strip trailing spaces from each line, not just the document end" + ) + + def test_fold_before_space_is_not_corrupted(self) -> None: + """RFC 5545 3.1 folds blind at 75 octets, so a fold may land right + after a space that is part of the value. Stripping trailing + whitespace per line joins the two words together, silently corrupting + every folded description loaded from a server -- and, since save() + writes the result back, corrupting it on the server as well.""" + ical = ( + "BEGIN:VCALENDAR\n" + "VERSION:2.0\n" + "BEGIN:VEVENT\n" + "UID:test\n" + "DTSTAMP:20190103T070319Z\n" + "DTSTART:20190117T180000Z\n" + "DESCRIPTION:Please bring the following \n" + " items and also a pen\n" + "END:VEVENT\n" + "END:VCALENDAR\n" + ) + + fixed = vcal.fix(ical) + event = list(icalendar.Calendar.from_ical(fixed).walk("VEVENT"))[0] + assert str(event["DESCRIPTION"]) == "Please bring the following items and also a pen", ( + "fix() must not strip whitespace that a fold made trailing" + ) + + def test_compliant_ical_with_folded_lines_is_left_alone(self) -> None: + """Modifying compliant data also fires the rate-limited "your calendar + server breaks the icalendar standard" warning at servers that did + nothing wrong.""" + ical = ( + "BEGIN:VCALENDAR\n" + "VERSION:2.0\n" + "BEGIN:VEVENT\n" + "UID:test\n" + "DTSTAMP:20190103T070319Z\n" + "DTSTART:20190117T180000Z\n" + "DESCRIPTION:Please bring the following \n" + " items and also a pen\n" + "END:VEVENT\n" + "END:VCALENDAR\n" + ) + assert vcal.fix(ical) == ical + + def test_backslash_unescape_single_and_double_quotes(self) -> None: + """Bug §2.4: re.sub(r"\\+('\")", r"\1", fixed) used a group ('\"') + which matches only the literal two-char sequence '\" — not a character + class. Backslash before a lone single quote or lone double quote was + therefore not unescaped.""" + ical_single = ( + "BEGIN:VCALENDAR\n" + "VERSION:2.0\n" + "BEGIN:VEVENT\n" + "UID:test\n" + "DTSTAMP:20190103T070319Z\n" + "DTSTART:20190117T180000Z\n" + "SUMMARY:it\\'s here\n" + "END:VEVENT\n" + "END:VCALENDAR\n" + ) + ical_double = ical_single.replace("\\'", '\\"') + fixed_single = vcal.fix(ical_single) + fixed_double = vcal.fix(ical_double) + assert "SUMMARY:it's here" in fixed_single, "fix() must strip backslash before single quote" + assert 'SUMMARY:it"s here' in fixed_double, "fix() must strip backslash before double quote" + + def test_completed_date_fixup_preserves_next_property(self) -> None: + """Bug §2.1: COMPLETED date fixup regex consumed the trailing newline, + merging the next property line into COMPLETED and destroying it.""" + ical = """BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Example Corp.//CalDAV Client//EN +BEGIN:VTODO +UID:20070313T123432Z-456553@example.com +DTSTAMP:20070313T123432Z +COMPLETED:20070501 +SUMMARY:Submit Quebec Income Tax Return for 2006 +STATUS:NEEDS-ACTION +END:VTODO +END:VCALENDAR""" + fixed = vcal.fix(ical) + cal = icalendar.Calendar.from_ical(fixed) + todo = list(cal.walk("VTODO"))[0] + assert str(todo["SUMMARY"]) == "Submit Quebec Income Tax Return for 2006", ( + "SUMMARY was destroyed by COMPLETED fixup (newline consumed)" + ) + ## The COMPLETED date is given without a time, so fix() adds one; + ## asserting the resulting value is what actually pins the regex down. + ## (The previous check -- "SUMMARY" not in str(todo["COMPLETED"].dt) -- + ## could not fail: .dt is a datetime, whose str() is never going to + ## contain a summary.) + assert todo["COMPLETED"].dt == datetime(2007, 5, 1, 12, 0, 0, tzinfo=utc), ( + "COMPLETED was not parsed as the fixed-up 20070501T120000Z" + ) + def test_missing_dtstamp_fix(self) -> None: """ Test that missing DTSTAMP is added by the fix function. @@ -355,3 +490,60 @@ def test_missing_dtstamp_fix(self) -> None: # Verify the fixed ical is valid self.verifyICal(fixed) + + def test_fix_does_not_crash_on_truncated_input(self) -> None: + """§1.12: vcal.fix() must not raise AssertionError on truncated/garbage iCalendar. + + Truncated data (no END: line) previously triggered a bare assert on line 93 + which gave no useful error message and failed silently under python -O. + """ + truncated = "BEGIN:VCALENDAR\nVERSION:2.0\nBEGIN:VEVENT\nUID:trunc@example.com\n" + # Must not raise — return something (possibly unchanged input) + result = vcal.fix(truncated) + assert result is not None + + +class TestParseIcal(TestCase): + """vcal.parse_ical() - the guard in front of icalendar.Calendar.from_ical(). + + A server may hand us a body with no iCalendar in it at all. from_ical() + answers that with `ValueError: Found no components where exactly one is + required`, which says nothing about where the data came from. + """ + + def test_valid_calendar_parses(self) -> None: + cal = vcal.parse_ical(ev) + assert isinstance(cal, icalendar.Calendar) + assert [c.name for c in cal.subcomponents] == ["VEVENT"] + + def test_empty_data_raises_a_caldav_error(self) -> None: + from caldav.lib import error + + for empty in ("", " ", "\r\n\r\n", None): + with pytest.raises(error.ResponseError): + vcal.parse_ical(empty) + + def test_data_without_any_component_raises_a_caldav_error(self) -> None: + """An HTML error page, a bare header, a JSON body - anything with no BEGIN:.""" + from caldav.lib import error + + with pytest.raises(error.ResponseError): + vcal.parse_ical("404 not found") + + def test_the_error_says_what_arrived_and_where_from(self) -> None: + from caldav.lib import error + + with pytest.raises(error.ResponseError) as excinfo: + vcal.parse_ical("404", context="https://cal.example.com/inbox/1.ics") + message = str(excinfo.value) + assert "https://cal.example.com/inbox/1.ics" in message + assert "404" in message + + def test_malformed_but_recognisable_ical_is_left_to_icalendar(self) -> None: + """The guard is deliberately narrow: data that does contain a component + keeps whatever icalendar makes of it, rather than being reclassified.""" + with pytest.raises(ValueError) as excinfo: + vcal.parse_ical("BEGIN:VCALENDAR\r\nBEGIN:VEVENT\r\n") + from caldav.lib import error + + assert not isinstance(excinfo.value, error.ResponseError) diff --git a/tests/tools/check_dist.py b/tests/tools/check_dist.py new file mode 100644 index 00000000..df950575 --- /dev/null +++ b/tests/tools/check_dist.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +""" +Build the sdist and the wheel, and refuse to let stray local files ship. + +Usage: + python tests/tools/check_dist.py [--outdir DIR] + +Run by the ``package`` tox environment and by CI. It exists because +hatchling's VCS-ignore support only honours the *root* ``.gitignore``: a file +hidden from ``git status`` by a nested ``.gitignore``, or by the developer's +global git ignore file, is invisible locally and still ends up in the +tarball. caldav-3.2.1.tar.gz shipped ``.claude/settings.json`` and 1755 +files under ``venv/`` exactly that way, and nothing in CI ever built an sdist, +so nothing noticed. + +The sdist file list is checked against the working tree's tracked files: the +sdist may leave tracked files out (that is a packaging choice), but it must +never contain a file git does not track. That is also what turns "build the +release from a clean checkout" from a rule someone has to remember into one +the build enforces. +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +import tarfile +import zipfile +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent + +## Generated by the build, so absent from git but legitimately in the dist. +GENERATED = frozenset( + { + "PKG-INFO", + "caldav/_version.py", + } +) + + +def _tracked_files() -> set[str]: + out = subprocess.run( + ["git", "-C", str(REPO_ROOT), "ls-files"], + check=True, + capture_output=True, + text=True, + ) + return set(out.stdout.split()) + + +def _sdist_members(sdist: Path) -> set[str]: + with tarfile.open(sdist) as tar: + ## every member is prefixed with "-/" + return {name.split("/", 1)[1] for name in tar.getnames() if "/" in name} + + +def _wheel_members(wheel: Path) -> set[str]: + with zipfile.ZipFile(wheel) as zf: + return set(zf.namelist()) + + +def _build(outdir: Path) -> tuple[Path, Path]: + subprocess.run( + [sys.executable, "-m", "build", "--outdir", str(outdir), str(REPO_ROOT)], + check=True, + ) + (sdist,) = outdir.glob("*.tar.gz") + (wheel,) = outdir.glob("*.whl") + return sdist, wheel + + +def _twine_check(outdir: Path) -> None: + subprocess.run( + [sys.executable, "-m", "twine", "check", "--strict", *map(str, outdir.iterdir())], + check=True, + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--outdir", default="dist", help="where to write the built artifacts") + args = parser.parse_args() + + outdir = Path(args.outdir).resolve() + outdir.mkdir(parents=True, exist_ok=True) + for stale in outdir.iterdir(): + stale.unlink() + + sdist, wheel = _build(outdir) + _twine_check(outdir) + + tracked = _tracked_files() + untracked_in_sdist = sorted(_sdist_members(sdist) - tracked - GENERATED) + if untracked_in_sdist: + print( + f"{len(untracked_in_sdist)} file(s) in {sdist.name} are not tracked by git:", + file=sys.stderr, + ) + for name in untracked_in_sdist[:50]: + print(f" {name}", file=sys.stderr) + if len(untracked_in_sdist) > 50: + print(f" ... and {len(untracked_in_sdist) - 50} more", file=sys.stderr) + print( + "\nEither the release is being built from a dirty tree (build from a " + "clean checkout - see docs/design/RELEASE-HOWTO.md), or these want an " + "entry in [tool.hatch.build.targets.sdist] exclude in pyproject.toml.", + file=sys.stderr, + ) + return 1 + + ## The wheel is the package and nothing else. + stray_in_wheel = sorted( + name + for name in _wheel_members(wheel) + if not name.startswith("caldav/") and ".dist-info/" not in name + ) + if stray_in_wheel: + print(f"unexpected files in {wheel.name}:", file=sys.stderr) + for name in stray_in_wheel: + print(f" {name}", file=sys.stderr) + return 1 + + print(f"{sdist.name}: {len(_sdist_members(sdist))} files, all accounted for") + print(f"{wheel.name}: {len(_wheel_members(wheel))} files, all accounted for") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tox.ini b/tox.ini index a8131b94..703cd349 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,8 @@ -[tox:tox] -envlist = y39,py310,py311,py312,py313,py314,docs,style,deptry +## [tox:tox] is the section name to use when the configuration lives in +## setup.cfg; in a tox.ini it has to be [tox], or the settings below are +## silently ignored (which they were, until 2026-08). +[tox] +envlist = py310,py311,py312,py313,py314,docs,style,deptry,audit,package [testenv] deps = --editable .[test] @@ -30,7 +33,10 @@ deps = sphinx-copybutton pydata-sphinx-theme sphinx-autodoc-typehints -environment = +## "environment" is not a tox key (it is passenv/setenv), so tox silently +## ignored it and the doctests never saw PYTHON_CALDAV_USE_TEST_SERVER - +## the same class of bug as the [tox:tox] one fixed above. +passenv = PYTHON_CALDAV_USE_TEST_SERVER commands = sphinx-build -b doctest docs/source docs/build/doctest @@ -45,6 +51,40 @@ basepython = python3.13 deps = --editable .[test] commands = deptry caldav --known-first-party caldav +[testenv:audit] +## Audits the resolved runtime dependency tree against the PyPI advisory +## database. We declare open-ended version ranges, so a vulnerable release is +## normally resolved away by itself - the value here is catching the case where +## one of our *upper* bounds (currently only icalendar-searcher<2) would hold +## users back on a release with a known vulnerability. Test-only +## dependencies are deliberately not covered: pip-audit reads pyproject.toml +## metadata and skips the optional extras. +## +## No basepython pin: the audit is essentially interpreter-independent, and +## pinning would make this env unrunnable on machines lacking that version. +## Needs network access (PyPI advisory lookups). The socket timeout is raised +## from the 15s default because the advisory lookups are one request per +## dependency and a single slow response is enough to fail the whole run. +skip_install = true +deps = pip-audit +## A dedicated HTTP cache dir is used because pip-audit otherwise shares pip's +## cache, where it hits entries it cannot deserialize and logs a screenful of +## warnings on every run - noise is the enemy of a job nobody looks at. +commands = pip-audit --strict --desc on --timeout 30 --cache-dir {toxworkdir}/pip-audit-cache {toxinidir} + +[testenv:package] +## Builds the release artifacts and checks that nothing local leaked into +## them. Without this nothing ever built an sdist outside a release, so the +## contents of the tarball were only ever discovered after publishing - see +## tests/tools/check_dist.py for what went wrong before. +skip_install = true +deps = + build + hatch-vcs + hatchling>=1.27.0 + twine +commands = python {toxinidir}/tests/tools/check_dist.py --outdir {envtmpdir}/dist + [build_sphinx] source-dir = docs/source build-dir = docs/build