diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..a47ce0b --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,70 @@ +name: Test + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + protocol: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@1.88 + with: + components: rustfmt, clippy + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: npm + cache-dependency-path: tests/conformance/package-lock.json + - run: cargo fmt --all -- --check + - run: cargo clippy --locked --all-targets --all-features -- -D warnings + - run: cargo test --locked --all-features + - uses: actions/checkout@v4 + if: ${{ vars.CONFORMANCE_ENGINE_URL == '' && vars.CONFORMANCE_ENGINE_VERSION == '' }} + with: + repository: rivet-dev/rivet + ref: 78cf1288ba44ebc6ea769be42c525b4be4819338 + path: .conformance-engine + persist-credentials: false + - name: Build matching OSS Engine + if: ${{ vars.CONFORMANCE_ENGINE_URL == '' && vars.CONFORMANCE_ENGINE_VERSION == '' }} + working-directory: .conformance-engine + env: + CARGO_PROFILE_DEV_DEBUG: "0" + CARGO_INCREMENTAL: "0" + CARGO_BUILD_JOBS: "2" + run: | + rustup toolchain install stable --profile minimal + RUSTUP_TOOLCHAIN=stable cargo build --locked -p rivet-engine --bin rivet-engine + - name: Run conformance + run: | + if [[ -x .conformance-engine/target/debug/rivet-engine ]]; then + export CONFORMANCE_ENGINE_BINARY="$PWD/.conformance-engine/target/debug/rivet-engine" + export CONFORMANCE_ENGINE_SHA256="$(sha256sum "$CONFORMANCE_ENGINE_BINARY" | cut -d ' ' -f 1)" + fi + bash scripts/test-conformance.sh + env: + CONFORMANCE_KEEP_LOGS: "1" + # A published compatible Engine overrides the pinned source build. + CONFORMANCE_ENGINE_URL: ${{ vars.CONFORMANCE_ENGINE_URL }} + CONFORMANCE_ENGINE_SHA256: ${{ vars.CONFORMANCE_ENGINE_SHA256 }} + CONFORMANCE_ENGINE_VERSION: ${{ vars.CONFORMANCE_ENGINE_VERSION }} + - uses: actions/upload-artifact@v4 + if: always() + with: + name: protocol-test-logs + path: | + /tmp/rivet-durable-streams-conformance.*/conformance.log + /tmp/rivet-durable-streams-conformance.*/engine.log + /tmp/rivet-durable-streams-conformance.*/server.log + /tmp/rivet-durable-streams-conformance.*/e2e.log + /tmp/rivet-durable-streams-conformance.*/large-snapshot.log + /tmp/rivet-durable-streams-conformance.*/recovery.log + /tmp/rivet-durable-streams-conformance.*/recovery/*.log + /tmp/rivet-durable-streams-conformance.*/recovery/migration/*.log diff --git a/.gitignore b/.gitignore index fedaa2b..331898e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /target .env +__pycache__/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ee63745..3992e70 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,6 +9,7 @@ RIVETKIT_ENGINE_AUTO_DOWNLOAD=1 cargo run -- --host 0.0.0.0 ``` To use an existing engine instead, set `RIVET_ENDPOINT`, `RIVET_NAMESPACE`, and `RIVET_TOKEN`. +Subscriptions additionally require an Engine build with the purpose-specific signing APIs and a scoped ACL credential; see [subscription configuration](docs/subscriptions.md#configure-access-and-public-urls). ## Talk to it @@ -24,8 +25,21 @@ curl 'http://127.0.0.1:8787/v1/stream/demo?offset=-1' ## Test it ```sh -just test -just test-conformance +cargo test --locked +CONFORMANCE_ENGINE_BINARY=/path/to/reviewed/rivet-engine \ +CONFORMANCE_ENGINE_SHA256= \ +bash scripts/test-conformance.sh ``` -`test-conformance` builds the pinned Rivet Engine from source, starts it with the server, and runs the official Durable Streams conformance suite against it. +The conformance script verifies the supplied Engine build, provisions an isolated namespace signing credential, starts the server, and runs the complete official `@durable-streams/server-conformance-tests` 0.3.7 suite, including all six subscription cases. Engine 2.3.16 lacks the required signing APIs. A compatible download can instead use `CONFORMANCE_ENGINE_URL` with its expected checksum; once released, pin `CONFORMANCE_ENGINE_VERSION` to use the published checksum manifest. CI accepts the same repository variables. The isolated npm harness pins Vitest 4.0.18 and its dependency lockfile. Use Node.js 22.12+ or 24+ and npm. + +SSE requires Engine's streaming gateway. The script sets `RIVET__FEATURES__GUARD_GATEWAY_V3__MODE=on` and `RIVET__FEATURES__GUARD_GATEWAY_V3__PERCENTAGE=100`; use the same settings for your Engine deployment. The default gateway buffers streaming responses until completion. + +The script enables unauthenticated subscription management and localhost HTTP webhooks only for its local test server. Failures retain Engine, server, and conformance logs. Set `CONFORMANCE_KEEP_LOGS=1` to retain passing logs too. Set `CONFORMANCE_LEGACY_BINARY` to a previous DS executable with local signing to also test live capability migration. + +To run against an existing local test server, export its `RIVET_ENDPOINT`, `RIVET_NAMESPACE`, `RIVET_TOKEN`, and `CONFORMANCE_ENGINE_ADMIN_TOKEN` first. The added signing tests use the operator credential only to provision test namespaces and exercise key rotation: + +```sh +npm --prefix tests/conformance ci +CONFORMANCE_TEST_URL=http://127.0.0.1:17878 npm --prefix tests/conformance test +``` diff --git a/Cargo.lock b/Cargo.lock index 77f8cef..5e30bbf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -980,6 +980,19 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonwebtoken" +version = "9.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde" +dependencies = [ + "base64", + "js-sys", + "ring", + "serde", + "serde_json", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -1555,8 +1568,10 @@ dependencies = [ "futures-util", "http", "include_dir", + "jsonwebtoken", "percent-encoding", "reqwest", + "ring", "rivetkit", "rivetkit-client", "serde", diff --git a/Cargo.toml b/Cargo.toml index e6a0c97..50f5966 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,9 +17,14 @@ include = [ "Cargo.toml", "Cargo.lock", "README.md", + "docs/subscriptions.md", "LICENSE", ] +[features] +# Deterministic crash windows for the real-Engine recovery harness only. +subscription-fault-injection = [] + [dependencies] anyhow = "1" async-trait = "0.1" @@ -31,9 +36,11 @@ clap = { version = "4.3", features = ["derive"] } futures-util = "0.3" http = "1.3" include_dir = "0.7" +jsonwebtoken = { version = "9", default-features = false } percent-encoding = "2.3" # RivetKit returns reqwest responses; enable their streaming body API. -reqwest = { version = "0.12.22", default-features = false, features = ["stream"] } +reqwest = { version = "0.12.22", default-features = false, features = ["stream", "rustls-tls"] } +ring = "0.17" rivetkit = "=2.3.16" rivetkit-client = "=2.3.16" serde = { version = "1", features = ["derive"] } diff --git a/README.md b/README.md index 5e66b56..d206f50 100644 --- a/README.md +++ b/README.md @@ -113,3 +113,5 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for running the server from source and te ## License Apache 2.0 + +Subscription delivery, authentication, and recovery are described in [Subscriptions](docs/subscriptions.md). diff --git a/docs/subscriptions.md b/docs/subscriptions.md new file mode 100644 index 0000000..a35f692 --- /dev/null +++ b/docs/subscriptions.md @@ -0,0 +1,91 @@ +# Durable Streams subscriptions + +Subscriptions track processed offsets across streams and notify workers through signed webhooks or an ordinary wake stream. This server implements the upstream draft at [`461b40267aabd644558f9b19dbb9507dd5f691cf`](https://github.com/durable-streams/durable-streams/blob/461b40267aabd644558f9b19dbb9507dd5f691cf/PROTOCOL.md#6-reserved-subscription-apis), checked against upstream on September 24, 2026. The conformance harness pins `@durable-streams/server-conformance-tests` 0.3.7 and Vitest 4.0.18, with subscriptions explicitly enabled. + +The [Node implementation](https://github.com/durable-streams/durable-streams/blob/461b40267aabd644558f9b19dbb9507dd5f691cf/packages/server/src/subscription-manager.ts) informed wire compatibility, and [Caddy's crypto implementation](https://github.com/durable-streams/durable-streams/blob/461b40267aabd644558f9b19dbb9507dd5f691cf/packages/caddy-plugin/webhook/crypto.go) provided a second signing reference. Both references keep subscription state in memory; this implementation persists delivery state and recovery intents. Caddy uses an older subscription protocol and is not the reference for pull-wake behavior. + +## Configure access and public URLs + +Set `PUBLIC_ORIGIN` to the externally reachable origin, for example `https://streams.gameinc.io`. The server adds its stream root (`/v1/stream` in the standalone binary) when constructing callback and JWKS URLs. Forwarded host headers do not control these URLs. + +Set `DS_SUBSCRIPTIONS_MANAGEMENT_TOKEN` and send it as `Authorization: Bearer ` on subscription management and claim requests. Callback, ack and release requests require their own wake capability token. JWKS is public. Engine credentials authenticate internal actor traffic; they are not public client credentials. Protect ordinary stream operations through your application's existing access-control boundary. + +The same `RIVET_ENDPOINT`, `RIVET_NAMESPACE`, and `RIVET_TOKEN` connection supplies Engine signing. Issue a namespace-scoped Engine JWT through `/auth/tokens` with `application_token/create` and `webhook_signing/create`, plus the actor and runner grants needed by your deployment. Add `namespace/read` for namespace lookup and `datacenter/list` for RivetKit routing. The JWT's namespace must match `RIVET_NAMESPACE`. An Engine build containing the purpose-specific application-token and webhook-signing APIs is required; released Engine 2.3.16 does not provide them. + +JWT verification trusts the configured Engine endpoint as issuer by default. Set `DS_ENGINE_ISSUER` to the trusted leader's issuer when connecting through a proxy or follower with a different URL. The issuer never comes from unverified token claims. + +The library's `SubscriptionConfig` supports the same policy. An embedding application that already authenticates management requests may explicitly enable `allow_unauthenticated` inside that protected router. The standalone equivalent, `DS_SUBSCRIPTIONS_ALLOW_UNAUTHENTICATED=1`, is intended for local tests. Management access is denied by default. + +Use `DS_TENANT_SCOPE` to isolate standalone applications sharing an Engine namespace. In library integrations, use `DurableStreamsConfig.tenant_scope`; choose the tenant from authenticated application context, not a caller-controlled header. + +Webhooks require HTTPS and public IP addresses. Each attempt resolves and validates DNS, pins the connection to the validated addresses, and disables proxies and redirects. For local receivers only, set `DS_SUBSCRIPTIONS_ALLOW_LOCAL_HTTP=1` on every actor runner and enable the library's `allow_local_http` option when embedding. This permits localhost, `127.0.0.x`, and `::1`; other private destinations remain rejected. + +## Create and consume + +Create the wake stream before using pull-wake delivery: + +```sh +curl -X PUT "$BASE/v1/stream/wake/workers" \ + -H 'Content-Type: application/json' --data '[]' + +curl -X PUT "$BASE/v1/stream/__ds/subscriptions/processor" \ + -H "Authorization: Bearer $MANAGEMENT_TOKEN" \ + -H 'Content-Type: application/json' \ + --data '{"type":"pull-wake","pattern":"events/**","wake_stream":"wake/workers"}' +``` + +Read `wake/workers` normally. Claim through `POST .../__ds/subscriptions/processor/claim` with `{"worker":"worker-1"}`. Read each linked stream starting from its `acked_offset`, then acknowledge the returned `Stream-Next-Offset` verbatim. Do not increment or interpret offsets. Send the claim token to `/ack`, with the returned `wake_id`, `generation`, `acks`, and `done:true` when finished. Omit `done` to heartbeat; `/release` gives up ownership without acknowledging. + +For webhooks, replace `type` with `webhook` and `wake_stream` with `"webhook":{"url":"https://worker.gameinc.io/hooks"}`. Verify the exact request bytes using `Webhook-Signature` and the advertised JWKS. Returning `{"done":true}` acknowledges only that delivery's snapshot. For asynchronous work, use the supplied callback URL and bearer capability token. + +The signature header is `Webhook-Signature: t=,kid=,ed25519=`. Decode the unpadded base64url signature and verify Ed25519 over the timestamp's ASCII bytes, a literal `.` byte, and the unchanged request body. Reject timestamps outside a five-minute freshness window. Select the key using the header's `kid`; subscription metadata can race a rotation. + +## Recovery and lifecycle + +Configuration, memberships, cursors, active wakes, retry deadlines, leases and notification intents live in actor SQLite storage. Engine owns webhook private keys and application-token issuance. A committed append is retried into the catalog after a crash. Delivery is at least once; receivers must tolerate repeated wake IDs. Pull-wake publication uses a durable producer identity to deduplicate retries, and claims remain serialized across restarts. + +A crash during signing or delivery can defer another attempt until the persisted 60-second dispatch reservation expires. This reservation covers bounded signing and receiver work; restarting the server does not bypass it. + +The standalone server imports existing stream actors before the first pattern subscription. Library hosts upgrading an existing deployment should configure `SubscriptionConfig.catalog_api` with the authenticated Engine endpoint, namespace and token. Explicit membership also refreshes the addressed stream directly. All actor runners should run the upgraded implementation before the initial catalog import. Import failures fail registration instead of silently omitting streams. + +Webhook keys belong to the immutable Engine namespace and may be shared by multiple DS tenants. Callback and claim application tokens use Engine's separate authentication key ring; DS verifies their signature, issuer, audience, purpose, namespace, tenant, public stream root and live wake identity locally. Keep Engine storage durable and protected. Signing outages leave delivery pending; there is no local signing fallback. + +Permanent Engine signing failures (400, 401, 403 or 413) leave the subscription visibly `failed` with `delivery_error` and preserve its pending offsets. They do not retry automatically. After correcting credentials, request configuration or body limits, re-confirm the same subscription configuration with PUT to resume in a new generation; membership additions or removals also resume processing. Temporary failures retain bounded retry backoff. + +Engine's webhook body limit is 32 MiB. Snapshots exceeding that limit fail visibly; the server never truncates membership or acknowledges undelivered data. Reduce the subscription's membership before resuming. Rate-limited delivery honors `Retry-After`. + +Operators manage webhook keys through Engine's `GET /namespaces/{namespace}/auth/webhook-keys`, `POST .../rotate`, and `POST .../revoke` APIs with separately authorized read/update/delete permissions. A normal rotation stages a public key before activation. DS fetches authoritative webhook keys for its own `__ds/jwks.json` response and preserves its public URL and five-minute cache policy. Emergency revocation cannot recall keys already cached by receivers; refresh once on an unfamiliar `kid`, and reject if a trusted verification key is unavailable. + +Upgrades preserve already-issued local capabilities only until their recorded expiry, bounded by 24 hours after cutover. Old webhook public keys remain available for 365 seconds after cutover. Legacy private signing keys are discarded, verification material is removed after these windows, and every newly issued token or signature uses Engine. Upgrade all runners before retiring the previous deployment. + +The draft leaves several lifecycle details open. This implementation uses these rules: + +- Re-confirmation compares the original normalized creation configuration. Adding or removing memberships does not change that configuration hash or reset progress. +- Existing memberships start at the current tail. New matching streams include their initial PUT data. Forks begin after inherited data, including a copied fork sub-offset prefix; newly appended fork data is pending. +- Deletion or recreation of a linked stream fences its active wake. A new incarnation resets that stream's cursor; offsets from the old incarnation cannot acknowledge replacement data. +- Retries use the original transmitted snapshot. Expired generations cannot process late HTTP responses. When expiration overlaps failed delivery, the next generation inherits the retry count and deadline. The delivery lease begins immediately before contacting the receiver, after Engine signing completes. +- Heartbeats extend the persisted lease. DS requests a 24-hour capability lifetime but honors Engine's actual expiry, which may be shorter because of the caller credential. After expiry the worker must acquire the newly fenced wake. Leases otherwise default to 30 seconds and accept 1–600 seconds. +- Missing, closed or expired wake streams leave a durable retry intent. The server never creates them implicitly. +- A subscription's own generated wake records do not recursively trigger it. Their offsets are skipped only after preceding application data has been acknowledged. Other subscriptions can consume those wake records normally. Cross-subscription feedback remains coalesced behind active wakes; applications should avoid cycles that workers continually acknowledge into new work. + +Subscriptions do not pin stream retention. Keep processing within your stream retention policy. + +## Validation + +`bash scripts/test-conformance.sh` runs the pinned upstream suite, including all six subscription cases, additional HTTP race and signing-security coverage, the Rust end-to-end test, a 4096-stream snapshot storage check, and crash/restart recovery tests. Until an Engine release containing the signing APIs is pinned, supply a reviewed executable and its expected SHA-256: + +```sh +CONFORMANCE_ENGINE_BINARY=/path/to/rivet-engine \ +CONFORMANCE_ENGINE_SHA256= \ +CONFORMANCE_KEEP_LOGS=1 bash scripts/test-conformance.sh +``` + +Alternatively, set `CONFORMANCE_ENGINE_URL` and the expected checksum, or pin a compatible released `CONFORMANCE_ENGINE_VERSION` to verify against its published checksum manifest. CI builds the pinned OSS Engine revision when neither variable is configured. The harness creates an isolated Engine, provisions a namespace-scoped JWT, and uses that same connection for DS, signing tests and recovery. `CONFORMANCE_KEEP_LOGS=1` retains successful logs as well as failures. The streaming gateway remains enabled for the pinned Rust runtime. + +Set `CONFORMANCE_LEGACY_BINARY` to a previous DS executable with local subscription signing to additionally exercise a live upgrade: existing callback/claim tokens remain valid, their public webhook keys remain discoverable, and subsequent wakes use Engine application JWTs. + +HTTP test files run sequentially so large subscription fixtures do not consume the upstream tests' short TTL timing windows. Each test has a 180-second harness timeout for randomized cases that perform hundreds of real-Engine operations. Upstream assertions, generated cases and concurrency within its test file are unchanged. + +The recovery harness `tests/subscription_recovery.py` runs separate server processes against an existing test Engine using `RIVET_TOKEN` and `RIVET_NAMESPACE`. Build with `cargo build --locked --features subscription-fault-injection` before running it. That opt-in feature exposes no HTTP endpoint; environment-selected checkpoints terminate the test process at committed-data and delivery-intent boundaries. Production builds leave it disabled. + +The test Engine also uses a 250 ms actor rescheduling base delay and maximum backoff exponent of 1. Engine 2.3.16's default allocation backoff can delay actors for several minutes after repeated runner restarts; these test-only settings let the crash tests finish promptly. Subscription retry deadlines and lease durations retain their normal values. The tests do not establish a recovery-time bound for production Engine defaults. diff --git a/scripts/test-conformance.sh b/scripts/test-conformance.sh index bb29243..9a7e6fa 100644 --- a/scripts/test-conformance.sh +++ b/scripts/test-conformance.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -Eeuo pipefail -readonly RIVET_VERSION=2.3.12 +readonly RIVET_VERSION="${CONFORMANCE_ENGINE_VERSION:-}" readonly ENGINE_PORT=17420 readonly SERVER_PORT=17878 readonly RUN_DIR="$(mktemp -d "${TMPDIR:-/tmp}/rivet-durable-streams-conformance.XXXXXX")" @@ -22,10 +22,11 @@ stop_group() { cleanup() { local status="$?" - trap - EXIT INT TERM + trap - EXIT + trap '' INT TERM stop_group "$server_pid" stop_group "$engine_pid" - if (( status == 0 )); then + if (( status == 0 )) && [[ "${CONFORMANCE_KEEP_LOGS:-0}" != 1 ]]; then rm -rf -- "$RUN_DIR" else echo "Logs retained in $RUN_DIR" >&2 @@ -59,40 +60,74 @@ case "$(uname -s)-$(uname -m)" in *) echo "error: unsupported platform $(uname -s)-$(uname -m)" >&2; exit 1 ;; esac -engine_cache="$PWD/target/conformance/engine-$RIVET_VERSION" -engine_binary="$engine_cache/$engine_artifact" -release_base="https://releases.rivet.dev/rivet/$RIVET_VERSION/engine" -manifest="$(curl --fail --silent --show-error "$release_base/SHA256SUMS")" -expected_checksum="$(awk -v artifact="$engine_artifact" '$2 == artifact { print $1 }' <<<"$manifest")" -[[ -n "$expected_checksum" ]] || { echo "error: missing Engine checksum" >&2; exit 1; } - -if [[ ! -x "$engine_binary" ]] \ - || [[ "$(sha256sum "$engine_binary" | awk '{ print $1 }')" != "$expected_checksum" ]]; then - downloaded_engine="$RUN_DIR/$engine_artifact" - curl --fail --silent --show-error "$release_base/$engine_artifact" -o "$downloaded_engine" - received_checksum="$(sha256sum "$downloaded_engine" | awk '{ print $1 }')" - [[ "$received_checksum" == "$expected_checksum" ]] \ - || { echo "error: Engine checksum mismatch" >&2; exit 1; } - mkdir -p "$engine_cache" - install -m 755 "$downloaded_engine" "$engine_binary" +if [[ -n "${CONFORMANCE_ENGINE_BINARY:-}" ]]; then + engine_binary="$(realpath "$CONFORMANCE_ENGINE_BINARY")" + expected_checksum="${CONFORMANCE_ENGINE_SHA256:?Set CONFORMANCE_ENGINE_SHA256 for the supplied Engine binary}" + [[ -x "$engine_binary" && "$(sha256sum "$engine_binary" | awk '{ print $1 }')" == "$expected_checksum" ]] \ + || { echo "error: supplied Engine binary checksum mismatch" >&2; exit 1; } +elif [[ -n "${CONFORMANCE_ENGINE_URL:-}" ]]; then + expected_checksum="${CONFORMANCE_ENGINE_SHA256:?Set CONFORMANCE_ENGINE_SHA256 for the supplied Engine URL}" + engine_binary="$RUN_DIR/$engine_artifact" + curl --fail --silent --show-error "$CONFORMANCE_ENGINE_URL" -o "$engine_binary" + [[ "$(sha256sum "$engine_binary" | awk '{ print $1 }')" == "$expected_checksum" ]] \ + || { echo "error: supplied Engine download checksum mismatch" >&2; exit 1; } + chmod 755 "$engine_binary" +else + [[ -n "$RIVET_VERSION" ]] || { + echo "error: Engine signing APIs require a compatible build; set CONFORMANCE_ENGINE_BINARY or CONFORMANCE_ENGINE_URL with CONFORMANCE_ENGINE_SHA256, or pin CONFORMANCE_ENGINE_VERSION" >&2 + exit 1 + } + engine_cache="$PWD/target/conformance/engine-$RIVET_VERSION" + engine_binary="$engine_cache/$engine_artifact" + release_base="https://releases.rivet.dev/rivet/$RIVET_VERSION/engine" + manifest="$(curl --fail --silent --show-error "$release_base/SHA256SUMS")" + expected_checksum="$(awk -v artifact="$engine_artifact" '$2 == artifact { print $1 }' <<<"$manifest")" + [[ -n "$expected_checksum" ]] || { echo "error: missing Engine checksum" >&2; exit 1; } + if [[ ! -x "$engine_binary" ]] \ + || [[ "$(sha256sum "$engine_binary" | awk '{ print $1 }')" != "$expected_checksum" ]]; then + downloaded_engine="$RUN_DIR/$engine_artifact" + curl --fail --silent --show-error "$release_base/$engine_artifact" -o "$downloaded_engine" + received_checksum="$(sha256sum "$downloaded_engine" | awk '{ print $1 }')" + [[ "$received_checksum" == "$expected_checksum" ]] \ + || { echo "error: Engine checksum mismatch" >&2; exit 1; } + mkdir -p "$engine_cache" + install -m 755 "$downloaded_engine" "$engine_binary" + fi fi -echo "Using Rivet Engine $RIVET_VERSION" -cargo build --quiet --locked --bin rivet-durable-streams +echo "Using checksum-verified Engine: $expected_checksum" +"$engine_binary" --version +cargo build --quiet --locked --bin rivet-durable-streams --features subscription-fault-injection +npm --prefix tests/conformance ci --no-audit --no-fund +export RIVET_ENDPOINT="http://127.0.0.1:$ENGINE_PORT" RIVET_NAMESPACE=default +export CONFORMANCE_ENGINE_ADMIN_TOKEN="${CONFORMANCE_ENGINE_ADMIN_TOKEN:-dev}" +RIVET__AUTH__ADMIN_TOKEN="$CONFORMANCE_ENGINE_ADMIN_TOKEN" \ +RIVET__TOPOLOGY__DATACENTER_LABEL=1 \ +RIVET__TOPOLOGY__DATACENTERS__DEFAULT__DATACENTER_LABEL=1 \ +RIVET__TOPOLOGY__DATACENTERS__DEFAULT__IS_LEADER=true \ +RIVET__TOPOLOGY__DATACENTERS__DEFAULT__PUBLIC_URL="$RIVET_ENDPOINT" \ +RIVET__TOPOLOGY__DATACENTERS__DEFAULT__PEER_URL="http://127.0.0.1:$((ENGINE_PORT + 1))" \ RIVET__GUARD__HOST=127.0.0.1 \ RIVET__GUARD__PORT="$ENGINE_PORT" \ RIVET__API_PEER__HOST=127.0.0.1 \ RIVET__API_PEER__PORT="$((ENGINE_PORT + 1))" \ RIVET__METRICS__HOST=127.0.0.1 \ RIVET__METRICS__PORT="$((ENGINE_PORT + 10))" \ +RIVET__FEATURES__GUARD_GATEWAY_V3__MODE=on \ +RIVET__FEATURES__GUARD_GATEWAY_V3__PERCENTAGE=100 \ +RIVET__PEGBOARD__BASE_RETRY_TIMEOUT=250 \ +RIVET__PEGBOARD__RESCHEDULE_BACKOFF_MAX_EXPONENT=1 \ RIVET__FILE_SYSTEM__PATH="$RUN_DIR/engine" \ setsid "$engine_binary" start >"$RUN_DIR/engine.log" 2>&1 & engine_pid="$!" wait_ready "Rivet Engine" "http://127.0.0.1:$ENGINE_PORT/health" "$engine_pid" "$RUN_DIR/engine.log" +RIVET_TOKEN="$(python3 tests/conformance/provision_signing.py)" +export RIVET_TOKEN -RIVET_ENDPOINT="http://127.0.0.1:$ENGINE_PORT" \ -RIVET_NAMESPACE=default RIVET_TOKEN=dev RIVET_POOL_NAME=durable-streams-conformance \ +RIVET_POOL_NAME=durable-streams-conformance \ +PUBLIC_ORIGIN="http://127.0.0.1:$SERVER_PORT" RUST_LOG="${RUST_LOG:-warn}" \ +DS_SUBSCRIPTIONS_ALLOW_LOCAL_HTTP=1 DS_SUBSCRIPTIONS_ALLOW_UNAUTHENTICATED=1 \ setsid target/debug/rivet-durable-streams --host 0.0.0.0 --port "$SERVER_PORT" \ >"$RUN_DIR/server.log" 2>&1 & server_pid="$!" @@ -100,15 +135,16 @@ wait_ready "Durable Streams" \ "http://127.0.0.1:$SERVER_PORT/v1/stream/__conformance_readiness" \ "$server_pid" "$RUN_DIR/server.log" -# The 0.3.6 CLI's runner filename misses Vitest 4's default include pattern. -# A temporary copy plus this config runs the published tests unchanged. -CONFORMANCE_RUN_DIR="$RUN_DIR" CONFORMANCE_URL="http://127.0.0.1:$SERVER_PORT" \ - npx --yes --package=@durable-streams/server-conformance-tests@0.3.6 --call ' - runner=$(readlink -f "$(command -v server-conformance-tests)") - package=$(dirname "$(dirname "$runner")") - cp -R "$package" "$CONFORMANCE_RUN_DIR/conformance" - ln -s "$(dirname "$(dirname "$package")")" "$CONFORMANCE_RUN_DIR/conformance/node_modules" - cd "$CONFORMANCE_RUN_DIR/conformance" - printf %s\\n "export default { test: { include: [\"dist/test-runner.js\"], testTimeout: 60000 } }" > vitest.config.js - ./dist/cli.js --run "$CONFORMANCE_URL" - ' 2>&1 | tee "$RUN_DIR/conformance.log" +# The official CLI omits subscriptions; our isolated harness enables all groups. +if [[ "${CONFORMANCE_RECOVERY_ONLY:-0}" != "1" ]]; then + CONFORMANCE_TEST_URL="http://127.0.0.1:$SERVER_PORT" \ + npm --prefix tests/conformance test -- "$@" 2>&1 | tee "$RUN_DIR/conformance.log" + RIVET_TEST_ENDPOINT="http://127.0.0.1:$ENGINE_PORT" \ + cargo test --locked --features subscription-fault-injection --test e2e -- --ignored --nocapture 2>&1 | tee "$RUN_DIR/e2e.log" + RIVET_TEST_ENDPOINT="http://127.0.0.1:$ENGINE_PORT" \ + cargo test --locked --features subscription-fault-injection --lib maximum_snapshot_persists_through_real_engine -- --ignored --nocapture 2>&1 | tee "$RUN_DIR/large-snapshot.log" +fi +if [[ "${CONFORMANCE_SKIP_RECOVERY:-0}" != "1" ]]; then + python3 tests/subscription_recovery.py --engine "http://127.0.0.1:$ENGINE_PORT" \ + --logs "$RUN_DIR/recovery" 2>&1 | tee "$RUN_DIR/recovery.log" +fi diff --git a/src/actor.rs b/src/actor.rs index 244b686..e1562cc 100644 --- a/src/actor.rs +++ b/src/actor.rs @@ -39,6 +39,7 @@ use producer::{ close_success, closed_by_matches, parse_producer, producer_failure, producer_success, validate_producer, }; +use subscriptions::wake_request; mod producer; @@ -61,7 +62,10 @@ pub struct DurableStreamState { } pub struct DurableStreamActor { + client: rivetkit_client::Client, lifecycle: Mutex<()>, + subscription_flush: Mutex>, + subscription_recovery_armed: Mutex, changes: watch::Sender, live_readers: Arc, readers: Arc, @@ -103,7 +107,6 @@ struct ForkCreateOptions<'a> { expires_at: Option<&'a str>, closed: bool, body: &'a [u8], - now_ms: i64, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -187,11 +190,44 @@ impl Action for Maintenance { const NAME: &'static str = "__durableStreamMaintenance"; } +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SubscriptionSnapshot; + +impl Action for SubscriptionSnapshot { + type Output = SubscriptionSnapshotResult; + const NAME: &'static str = "__durableStreamSubscriptionSnapshot"; +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct AppendWake { + pub wake_id: String, + pub event: serde_json::Value, + pub origin_sub_key: String, +} + +impl Action for AppendWake { + type Output = u16; + const NAME: &'static str = "__durableStreamAppendWake"; +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SubscriptionSnapshotResult { + pub tenant_scope: String, + pub entry: Option, +} + #[async_trait] impl Actor for DurableStreamActor { type State = DurableStreamState; type Input = DurableStreamInput; - type Actions = (ForkAcquire, ForkRelease, ReadRange, Maintenance); + type Actions = ( + ForkAcquire, + ForkRelease, + ReadRange, + Maintenance, + SubscriptionSnapshot, + AppendWake, + ); type Events = (); type Queue = (); type ConnParams = (); @@ -226,7 +262,10 @@ impl Actor for DurableStreamActor { store::ensure_schema(ctx.sql()).await?; let (changes, _) = watch::channel(0); Ok(Self { + client: crate::client::actor_client(ctx)?, lifecycle: Mutex::new(()), + subscription_flush: Mutex::new(None), + subscription_recovery_armed: Mutex::new(false), changes, live_readers: Arc::new(Semaphore::new(64)), readers: Arc::new(Semaphore::new(32)), @@ -238,10 +277,18 @@ impl Actor for DurableStreamActor { ctx: Ctx, request: Request, ) -> Result { - match *request.method() { + let mutates = matches!( + *request.method(), + http::Method::PUT | http::Method::POST | http::Method::DELETE + ); + if mutates { + // Persist recovery before any data/outbox transaction can commit. + self.arm_subscription_recovery(&ctx).await?; + } + let result = match *request.method() { http::Method::PUT => self.handle_put(&ctx, &request).await.map(Into::into), http::Method::POST => self.handle_post(&ctx, &request).await.map(Into::into), - http::Method::GET => self.handle_get(ctx, request).await, + http::Method::GET => self.clone().handle_get(ctx.clone(), request).await, http::Method::HEAD => self.handle_head(&ctx).await.map(Into::into), http::Method::DELETE => self.handle_delete(&ctx).await.map(Into::into), _ => response( @@ -250,10 +297,24 @@ impl Actor for DurableStreamActor { Vec::new(), ) .map(Into::into), + }; + if mutates { + if result.as_ref().is_ok_and(|response| match response { + ActorHttpResponse::Buffered(response) => response.status().is_success(), + _ => false, + }) { + crate::subscriptions::fault::checkpoint("after_stream_commit"); + } + // Handlers have released lifecycle before catalog RPCs, allowing + // registration to read snapshots while an append is completing. + self.flush_subscriptions_best_effort(&ctx).await; } + result } async fn on_start(self: Arc, ctx: Ctx) -> Result<()> { + // Become callable before notifying the catalog: a coordinator may be + // starting this actor in order to request a backfill snapshot. self.run_maintenance(&ctx).await } } @@ -301,7 +362,40 @@ impl Handles for DurableStreamActor { type Future = BoxActionFuture<()>; fn handle(self: Arc, ctx: Ctx, _action: Maintenance) -> Self::Future { - Box::pin(async move { self.run_maintenance(&ctx).await }) + Box::pin(async move { + let result = self.run_maintenance(&ctx).await; + if result.is_err() { + self.arm_subscription_recovery(&ctx).await?; + } + self.defer_subscription_flush(&ctx); + result + }) + } +} + +impl Handles for DurableStreamActor { + type Future = BoxActionFuture; + + fn handle(self: Arc, ctx: Ctx, _action: SubscriptionSnapshot) -> Self::Future { + Box::pin(async move { self.subscription_snapshot(&ctx).await }) + } +} + +impl Handles for DurableStreamActor { + type Future = BoxActionFuture; + + fn handle(self: Arc, ctx: Ctx, action: AppendWake) -> Self::Future { + Box::pin(async move { + self.arm_subscription_recovery(&ctx).await?; + let origin = action.origin_sub_key.clone(); + let request = wake_request(action)?; + let result = self.handle_post_inner(&ctx, &request, Some(&origin)).await; + if result.as_ref().is_ok_and(|r| r.status().is_success()) { + crate::subscriptions::fault::checkpoint("after_wake_append_commit"); + } + self.defer_subscription_flush(&ctx); + result.map(|response| response.status().as_u16()) + }) } } @@ -309,6 +403,7 @@ mod fork; mod lifecycle; mod maintenance; mod read; +mod subscriptions; mod write; fn query_values(request: &Request, name: &str) -> Vec { diff --git a/src/actor/fork.rs b/src/actor/fork.rs index dadf357..0f60eac 100644 --- a/src/actor/fork.rs +++ b/src/actor/fork.rs @@ -4,7 +4,7 @@ impl DurableStreamActor { pub(super) fn source_handle(&self, ctx: &Ctx, path: &str) -> Result { let state = ctx.state(); let key = stream_actor_key(&state.tenant_scope, path); - ctx.client()?.get_or_create( + self.client.get_or_create( ACTOR_NAME, vec![key], GetOrCreateOptions { @@ -300,13 +300,14 @@ impl DurableStreamActor { tx.rollback().await?; return Err(anyhow!("stream was created during fork acquisition")); } + let created_at = now_ms(); create_meta( &tx, Some(&resolved_content_type), effective_ttl, effective_expires_at.as_deref(), options.closed, - options.now_ms, + created_at, Some(options.parent_path), Some(&fork_offset), options @@ -320,15 +321,22 @@ impl DurableStreamActor { let mut current_offset = fork_offset.clone(); if let Some(prefix) = sub_offset_prefix { current_offset = - store::append_message(&tx, ¤t_offset, prefix, options.now_ms).await?; + store::append_message(&tx, ¤t_offset, prefix, created_at).await?; + // A partial inherited record is historical fork data too. + store::set_subscription_start(&tx, ¤t_offset).await?; } if let Some(payload) = initial_payload && !payload.is_empty() { current_offset = - store::append_message(&tx, ¤t_offset, payload, options.now_ms).await?; + store::append_message(&tx, ¤t_offset, payload, created_at).await?; } delete_fork_intent(&tx, &edge_id).await?; + // Acquiring the parent and copying a prefix can take most of a + // short TTL. Start it when the new stream is ready to commit. + if effective_ttl.is_some() { + touch(&tx, now_ms()).await?; + } tx.commit().await?; Ok::<_, anyhow::Error>(current_offset) } diff --git a/src/actor/maintenance.rs b/src/actor/maintenance.rs index 085e02c..8bac868 100644 --- a/src/actor/maintenance.rs +++ b/src/actor/maintenance.rs @@ -24,6 +24,7 @@ impl DurableStreamActor { } pub(super) async fn run_maintenance(&self, ctx: &Ctx) -> Result<()> { + *self.subscription_recovery_armed.lock().await = false; if get_meta(ctx.sql()).await?.is_none() { self.reconcile_fork_intents(ctx).await?; } @@ -33,6 +34,12 @@ impl DurableStreamActor { { self.schedule_expiry(ctx, &meta).await?; } + if store::subscription_outbox(ctx.sql()) + .await? + .is_some_and(|outbox| outbox.dirty) + { + self.arm_subscription_recovery(ctx).await?; + } Ok(()) } diff --git a/src/actor/read.rs b/src/actor/read.rs index f80a1d5..c129d20 100644 --- a/src/actor/read.rs +++ b/src/actor/read.rs @@ -153,14 +153,16 @@ impl DurableStreamActor { } else { offset }; - let mut snapshot = self + let Some(mut snapshot) = self .read_snapshot( &ctx, effective_offset, Some(initial_meta.generation.as_str()), ) .await? - .ok_or_else(|| anyhow!("stream generation changed during read"))?; + else { + return text_response(404, "Stream not found", [] as [(&str, &str); 0]).map(Into::into); + }; let caught_up = effective_offset == Some(initial_meta.current_offset.as_str()); if live.as_deref() == Some("long-poll") && caught_up @@ -445,29 +447,44 @@ impl DurableStreamActor { let _reader = Arc::clone(&self.readers) .try_acquire_owned() .map_err(|_| anyhow!("too many concurrent reads"))?; - let Some(meta) = self.visible_meta(ctx, now_ms()).await? else { + let Some(mut meta) = self.visible_meta(ctx, now_ms()).await? else { return Ok(None); }; if expected_generation.is_some_and(|generation| generation != meta.generation) { return Ok(None); } - let batch = self - .read_stitched( - ctx, - &meta, - offset, - None, - crate::protocol::MAX_READ_ROWS, - crate::protocol::MAX_READ_BYTES, - ) - .await?; - let Some(fresh) = get_meta(ctx.sql()).await? else { - return Ok(None); - }; - if fresh.generation != meta.generation { - return Ok(None); - } + // Reserve sliding TTL before an inherited fork read can await its source. self.touch_access(ctx, now_ms()).await?; - Ok(Some(ReadSnapshot { meta: fresh, batch })) + let mut retries = 0; + loop { + let mut batch = self + .read_stitched( + ctx, + &meta, + offset, + None, + crate::protocol::MAX_READ_ROWS, + crate::protocol::MAX_READ_BYTES, + ) + .await?; + let Some(fresh) = get_meta(ctx.sql()).await? else { + return Ok(None); + }; + if fresh.generation != meta.generation { + return Ok(None); + } + if batch.messages.is_empty() && fresh.current_offset != meta.current_offset { + // An append landed between the message read and the metadata read. + // Do not advance the cursor past data that this batch never saw. + if retries < 2 { + retries += 1; + meta = fresh; + continue; + } + batch.capped = true; + return Ok(Some(ReadSnapshot { meta, batch })); + } + return Ok(Some(ReadSnapshot { meta: fresh, batch })); + } } } diff --git a/src/actor/subscriptions.rs b/src/actor/subscriptions.rs new file mode 100644 index 0000000..653548f --- /dev/null +++ b/src/actor/subscriptions.rs @@ -0,0 +1,143 @@ +use super::*; + +use crate::subscriptions::model::CatalogEntry; + +impl DurableStreamActor { + pub(super) fn defer_subscription_flush(self: &Arc, ctx: &Ctx) { + let actor = self.clone(); + let work_ctx = ctx.clone(); + // Wake delivery returns once its append commits. Publishing back to + // the coordinator runs independently, outside the stream lifecycle lock. + ctx.wait_until(async move { actor.flush_subscriptions_best_effort(&work_ctx).await }); + } + + pub(super) async fn arm_subscription_recovery(&self, ctx: &Ctx) -> Result<()> { + let mut armed = self.subscription_recovery_armed.lock().await; + if !*armed { + self.schedule_maintenance_after(ctx, MAINTENANCE_RETRY_MS) + .await?; + *armed = true; + } + Ok(()) + } + + pub(super) async fn subscription_snapshot( + &self, + ctx: &Ctx, + ) -> Result { + let _guard = self.lifecycle.lock().await; + self.visible_meta_while_locked(ctx, now_ms()).await?; + let tenant_scope = ctx.state().tenant_scope.clone(); + Ok(SubscriptionSnapshotResult { + tenant_scope, + entry: store::subscription_outbox(ctx.sql()) + .await? + .map(|outbox| catalog_entry(ctx, outbox)), + }) + } + + pub(super) async fn flush_subscriptions_best_effort(&self, ctx: &Ctx) { + if let Err(error) = self.flush_subscriptions(ctx).await { + // Stream writes already committed. A temporary coordinator failure + // leaves the durable intent for the pre-armed maintenance event. + tracing::warn!( + ?error, + actor_id = ctx.actor_id(), + "subscription catalog notification deferred" + ); + if let Err(error) = self.arm_subscription_recovery(ctx).await { + tracing::warn!( + ?error, + actor_id = ctx.actor_id(), + "subscription recovery scheduling failed" + ); + } + } + } + + pub(super) async fn flush_subscriptions(&self, ctx: &Ctx) -> Result<()> { + let mut retry_after = self.subscription_flush.lock().await; + if retry_after.is_some_and(|deadline| Instant::now() < deadline) { + // Queued appends share one failed attempt instead of each waiting + // for a coordinator timeout during an outage. + return self.arm_subscription_recovery(ctx).await; + } + let result = self.publish_subscription_outbox(ctx).await; + *retry_after = result + .is_err() + .then(|| Instant::now() + Duration::from_millis(MAINTENANCE_RETRY_MS as u64)); + result + } + + async fn publish_subscription_outbox(&self, ctx: &Ctx) -> Result<()> { + let entry = { + let _guard = self.lifecycle.lock().await; + let Some(outbox) = store::subscription_outbox(ctx.sql()).await? else { + return Ok(()); + }; + if !outbox.dirty { + return Ok(()); + } + catalog_entry(ctx, outbox) + }; + self.arm_subscription_recovery(ctx).await?; + let tenant_scope = ctx.state().tenant_scope.clone(); + let handle = self.client.get_or_create( + "durableStreamSubscriptions", + vec![stream_actor_key(&tenant_scope, "__ds/subscriptions")], + GetOrCreateOptions { + create_with_input: Some(serde_json::json!({ + "tenant_scope": tenant_scope, + })), + ..GetOrCreateOptions::default() + }, + )?; + tokio::time::timeout( + Duration::from_secs(5), + handle.action( + "__durableStreamCatalogChanged", + vec![serde_json::to_value(&entry)?], + ), + ) + .await??; + // A notification racing a new write must not clear that write's + // intent. That writer's own flush (or maintenance) publishes it next. + store::acknowledge_subscription_outbox(ctx.sql(), entry.version).await + } +} + +fn catalog_entry(ctx: &Ctx, outbox: store::SubscriptionOutbox) -> CatalogEntry { + CatalogEntry { + path: ctx.state().path.clone(), + version: outbox.version, + incarnation: outbox.incarnation, + tail: outbox.tail, + start: outbox.start, + exists: outbox.exists, + wakes: outbox.wakes, + } +} + +pub(super) fn wake_request(action: AppendWake) -> Result { + use base64::engine::general_purpose::URL_SAFE_NO_PAD; + let headers = [ + ("content-type", "application/json".to_owned()), + (PRODUCER_ID, format!("__ds_wake_{}", action.wake_id)), + (PRODUCER_EPOCH, "0".to_owned()), + (PRODUCER_SEQ, "0".to_owned()), + ] + .into_iter() + .map(|(name, value)| { + ( + format!("{}{name}", crate::protocol::INTERNAL_HEADER_PREFIX), + URL_SAFE_NO_PAD.encode(value), + ) + }) + .collect(); + Request::from_parts( + "POST", + "/protocol", + headers, + serde_json::to_vec(&action.event)?, + ) +} diff --git a/src/actor/write.rs b/src/actor/write.rs index a583a43..6f43f26 100644 --- a/src/actor/write.rs +++ b/src/actor/write.rs @@ -134,7 +134,6 @@ impl DurableStreamActor { expires_at: expires_at.as_deref(), closed: create_closed, body: request.body(), - now_ms: now, }, ) .await; @@ -182,6 +181,9 @@ impl DurableStreamActor { if !payload.is_empty() { current_offset = store::append_message(&tx, ¤t_offset, payload, now).await?; } + if ttl_seconds.is_some() { + touch(&tx, now_ms()).await?; + } tx.commit().await?; self.notify_change(); if let Some(meta) = get_meta(ctx.sql()).await? { @@ -200,6 +202,15 @@ impl DurableStreamActor { } pub(super) async fn handle_post(&self, ctx: &Ctx, request: &Request) -> Result { + self.handle_post_inner(ctx, request, None).await + } + + pub(super) async fn handle_post_inner( + &self, + ctx: &Ctx, + request: &Request, + subscription_wake: Option<&str>, + ) -> Result { if request.body().len() > MAX_BODY_BYTES { return text_response(413, "Payload too large", [] as [(&str, &str); 0]); } @@ -301,6 +312,9 @@ impl DurableStreamActor { } let new_offset = store::append_message(&tx, &fresh_meta.current_offset, payload, now).await?; + if let Some(origin) = subscription_wake { + store::record_subscription_wake(&tx, &fresh_meta.current_offset, origin).await?; + } if let (Some(producer), Some(state)) = (&producer, &producer_state) { commit_producer_state(&tx, producer.id.clone(), state).await?; } diff --git a/src/client.rs b/src/client.rs new file mode 100644 index 0000000..d70708b --- /dev/null +++ b/src/client.rs @@ -0,0 +1,33 @@ +//! Preserve the runtime's effective connection while supporting authenticated +//! management lookups in RivetKit 2.3.16 (which only sets X-Rivet-Token). +use anyhow::{Result, anyhow}; +use rivetkit::{Actor, Ctx}; +use rivetkit_client::{Client, ClientConfig, EncodingKind, TransportKind}; + +pub(crate) fn actor_client(ctx: &Ctx) -> Result { + let runtime = ctx.inner(); + let mut config = ClientConfig::new( + runtime + .client_endpoint() + .ok_or_else(|| anyhow!("Missing actor client endpoint"))?, + ) + .namespace( + runtime + .client_namespace() + .ok_or_else(|| anyhow!("Missing actor client namespace"))?, + ) + .pool_name( + runtime + .client_pool_name() + .ok_or_else(|| anyhow!("Missing actor client pool"))?, + ) + .encoding(EncodingKind::Bare) + .transport(TransportKind::WebSocket) + .disable_metadata_lookup(true); + if let Some(token) = runtime.client_token() { + config = config + .token(token) + .header("authorization", format!("Bearer {token}")); + } + Ok(Client::new(config)) +} diff --git a/src/facade.rs b/src/facade.rs index d15d7c1..c467b7d 100644 --- a/src/facade.rs +++ b/src/facade.rs @@ -1,4 +1,6 @@ +use std::sync::Arc; use std::time::Duration; +use tokio::sync::OnceCell; use axum::Router; use axum::body::Body; @@ -21,6 +23,7 @@ const EXPOSE_HEADERS: &str = "Stream-Next-Offset, Stream-Cursor, Stream-Up-To-Da #[derive(Clone, Debug)] pub struct DurableStreamsConfig { + pub subscriptions: crate::subscriptions::facade::SubscriptionConfig, /// Separates deterministic actor keys between applications or tenants. pub tenant_scope: String, /// Public origin used to construct the `Location` header on create. @@ -35,6 +38,7 @@ pub struct DurableStreamsConfig { impl Default for DurableStreamsConfig { fn default() -> Self { Self { + subscriptions: Default::default(), tenant_scope: "default".to_owned(), public_origin: None, body_timeout: Duration::from_secs(15), @@ -45,6 +49,7 @@ impl Default for DurableStreamsConfig { #[derive(Clone)] struct FacadeState { + catalog_initialized: Arc>, client: Client, config: DurableStreamsConfig, } @@ -58,7 +63,11 @@ pub fn durable_streams_router(client: Client, config: DurableStreamsConfig) -> R Router::new() .route("/", any(handle_root)) .route("/{*path}", any(handle)) - .with_state(FacadeState { client, config }) + .with_state(FacadeState { + client, + config, + catalog_initialized: Arc::new(OnceCell::new()), + }) } async fn handle_root(method: Method) -> Response { @@ -92,6 +101,16 @@ async fn handle( return response; } + // Resolve control routing and actor identity within the same mount. Reject + // traversal before URL canonicalization can move a stream into __ds or out + // of the router's stream root. + if uri.path().split('/').any(|segment| { + let decoded = percent_encoding::percent_decode_str(segment).decode_utf8_lossy(); + decoded == "." || decoded == ".." || decoded.contains('\\') + }) { + return cors_response(StatusCode::BAD_REQUEST, Body::from("Invalid stream path")); + } + // Actor identity uses the complete public path. This deliberately matches // Stream-Forked-From, whose value is a path relative to this same server. let canonical_path = match canonical_stream_path(original_uri.path()) { @@ -101,9 +120,6 @@ async fn handle( } }; let routed_path = uri.path(); - if routed_path.ends_with("/__ds") || routed_path.contains("/__ds/") { - return cors_response(StatusCode::NOT_FOUND, Body::from("Stream not found")); - } let body = match tokio::time::timeout( state.config.body_timeout, @@ -129,6 +145,21 @@ async fn handle( } }; + let decoded_path = percent_encoding::percent_decode_str(routed_path).decode_utf8_lossy(); + if decoded_path.trim_start_matches('/').split('/').next() == Some("__ds") { + return crate::subscriptions::facade::route( + &state.client, + &state.config, + &state.catalog_initialized, + original_uri.path(), + routed_path, + method, + &headers, + &body, + ) + .await; + } + let actor_key = stream_actor_key(&state.config.tenant_scope, &canonical_path); let actor = if method == Method::PUT { state.client.get_or_create( @@ -136,7 +167,7 @@ async fn handle( vec![actor_key], GetOrCreateOptions { create_with_input: Some(json!({ - "path": canonical_path, + "path": canonical_path.clone(), "tenant_scope": state.config.tenant_scope.clone(), })), ..GetOrCreateOptions::default() @@ -192,12 +223,26 @@ async fn handle( } } }; - let actor_headers = match bridge_request_headers(&headers) { + let mut actor_headers = match bridge_request_headers(&headers) { Ok(headers) => headers, Err(message) => { return cors_response(StatusCode::BAD_REQUEST, Body::from(message)); } }; + // Guard verifies this server-owned identity against the actor key and uses + // it to authorize reads from a fork's source stream. + for (name, value) in [ + ("x-rivet-stream-path", canonical_path.as_str()), + ("x-rivet-stream-tenant", state.config.tenant_scope.as_str()), + ] { + let Ok(value) = HeaderValue::from_str(value) else { + return cors_response( + StatusCode::INTERNAL_SERVER_ERROR, + Body::from("Invalid stream identity"), + ); + }; + actor_headers.insert(name, value); + } let actor_path = match uri.query() { Some(query) => format!("protocol?{query}"), None => "protocol".to_owned(), @@ -287,7 +332,7 @@ async fn collect_bounded_body(body: Body, limit: usize) -> Result, Ok((!too_large).then(|| collected.freeze())) } -fn bridge_request_headers(headers: &HeaderMap) -> Result { +pub(crate) fn bridge_request_headers(headers: &HeaderMap) -> Result { let mut bridged = HeaderMap::new(); for name in BRIDGED_REQUEST_HEADERS { let header_name = HeaderName::from_static(name); @@ -323,7 +368,7 @@ pub(crate) fn stream_actor_key(tenant_scope: &str, path: &str) -> String { format!("{:x}", hash.finalize()) } -fn cors_response(status: StatusCode, body: Body) -> Response { +pub(crate) fn cors_response(status: StatusCode, body: Body) -> Response { let mut response = Response::builder() .status(status) .body(body) diff --git a/src/lib.rs b/src/lib.rs index 904928c..ffe2757 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,9 +2,13 @@ //! RivetKit actors. mod actor; +mod client; mod facade; mod protocol; mod store; +mod subscriptions; +pub use subscriptions::catalog_import::CatalogApi; +pub use subscriptions::facade::SubscriptionConfig; pub use actor::{ACTOR_NAME, DurableStreamActor, DurableStreamInput, DurableStreamState}; pub use facade::{DurableStreamsConfig, durable_streams_router}; @@ -38,6 +42,17 @@ pub fn register_with_inspector(registry: &mut Registry, inspector_root: Option

( + subscriptions::ACTOR_NAME, + ActorConfig { + has_database: true, + has_state: true, + max_incoming_message_size: 6 * 1024 * 1024, + // A 4096-stream snapshot with percent-encoded maximum paths is ~25 MiB. + max_outgoing_message_size: 32 * 1024 * 1024, + ..ActorConfig::default() + }, + ); let inspector_tabs = inspector_root.map(inspector_tabs).unwrap_or_default(); registry.register_actor_with::( ACTOR_NAME, diff --git a/src/main.rs b/src/main.rs index 16f98cc..0281cfe 100644 --- a/src/main.rs +++ b/src/main.rs @@ -43,6 +43,21 @@ async fn main() -> anyhow::Result<()> { let serve_config = normalize_serve_config(ServeConfig::from_env())?; let client = Client::new(client_config(&serve_config)?); let streams_config = DurableStreamsConfig { + tenant_scope: std::env::var("DS_TENANT_SCOPE").unwrap_or_else(|_| "default".into()), + subscriptions: rivet_durable_streams::SubscriptionConfig { + management_token: std::env::var("DS_SUBSCRIPTIONS_MANAGEMENT_TOKEN") + .ok() + .filter(|v| !v.is_empty()), + allow_unauthenticated: std::env::var("DS_SUBSCRIPTIONS_ALLOW_UNAUTHENTICATED") + .is_ok_and(|v| v == "1"), + allow_local_http: std::env::var("DS_SUBSCRIPTIONS_ALLOW_LOCAL_HTTP") + .is_ok_and(|v| v == "1"), + catalog_api: Some(rivet_durable_streams::CatalogApi { + endpoint: serve_config.endpoint.clone(), + namespace: serve_config.namespace.clone(), + token: serve_config.token.clone(), + }), + }, public_origin: std::env::var("PUBLIC_ORIGIN").ok(), ..DurableStreamsConfig::default() }; diff --git a/src/store.rs b/src/store.rs index c574903..f6df385 100644 --- a/src/store.rs +++ b/src/store.rs @@ -84,12 +84,49 @@ CREATE TABLE IF NOT EXISTS gc_releases ( "#, ]; +const MIGRATION_2: &[&str] = &[ + r#" +CREATE TABLE subscription_outbox ( + singleton INTEGER PRIMARY KEY CHECK (singleton = 1), + version INTEGER NOT NULL, + dirty INTEGER NOT NULL, + incarnation TEXT NOT NULL, + tail TEXT NOT NULL, + start_offset TEXT NOT NULL, + stream_exists INTEGER NOT NULL +) STRICT +"#, + // Existing streams enter the catalog at their deployment-time tail. The + // row survives deletion, so delayed notifications cannot revive an old + // incarnation after the same stream path has been recreated. + r#" +INSERT INTO subscription_outbox + (singleton, version, dirty, incarnation, tail, start_offset, stream_exists) +SELECT 1, 1, 1, gen, current_offset, current_offset, 1 - soft_deleted FROM meta +"#, + r#" +CREATE TABLE subscription_wake_writes ( + version INTEGER PRIMARY KEY, + before_offset TEXT NOT NULL, + after_offset TEXT NOT NULL, + incarnation TEXT NOT NULL, + origin_sub_key TEXT NOT NULL +) STRICT +"#, +]; + // Entries are immutable once released. Schema changes append a migration with // the next version instead of modifying or reordering existing entries. -const MIGRATIONS: &[Migration] = &[Migration { - version: 1, - statements: MIGRATION_1, -}]; +const MIGRATIONS: &[Migration] = &[ + Migration { + version: 1, + statements: MIGRATION_1, + }, + Migration { + version: 2, + statements: MIGRATION_2, + }, +]; #[derive(Clone, Debug, PartialEq, Eq)] pub struct Meta { @@ -167,6 +204,92 @@ pub struct GcRelease { pub source_generation: Option, } +#[derive(Clone, Debug)] +pub struct SubscriptionOutbox { + pub version: i64, + pub dirty: bool, + pub incarnation: String, + pub tail: String, + pub start: String, + pub exists: bool, + pub wakes: Vec, +} + +pub async fn subscription_outbox(db: &SqliteDb) -> Result> { + let result = db + .query( + "SELECT o.version, o.dirty, o.incarnation, o.tail, o.start_offset, o.stream_exists, \ + w.before_offset, w.after_offset, w.incarnation, w.origin_sub_key FROM subscription_outbox AS o \ + LEFT JOIN subscription_wake_writes AS w ON w.incarnation = o.incarnation \ + WHERE o.singleton = 1 ORDER BY w.version", + None, + ) + .await?; + let Some(row) = result.rows.first() else { + return Ok(None); + }; + let wakes = result + .rows + .iter() + .filter(|row| !matches!(row.get(6), Some(ColumnValue::Null))) + .map(|row| { + Ok(crate::subscriptions::model::WakeInterval { + before: text(row.get(6))?, + after: text(row.get(7))?, + incarnation: text(row.get(8))?, + origin: text(row.get(9))?, + }) + }) + .collect::>>()?; + Ok(Some(SubscriptionOutbox { + version: int(row.first())?, + dirty: int(row.get(1))? != 0, + incarnation: text(row.get(2))?, + tail: text(row.get(3))?, + start: text(row.get(4))?, + exists: int(row.get(5))? != 0, + wakes, + })) +} + +pub async fn acknowledge_subscription_outbox(db: &SqliteDb, version: i64) -> Result<()> { + let tx = begin_mutation(db).await?; + tx.execute( + "UPDATE subscription_outbox SET dirty = 0 WHERE singleton = 1 AND version = ?", + Some(vec![BindParam::Integer(version)]), + ) + .await?; + tx.execute( + "DELETE FROM subscription_wake_writes WHERE version <= ?", + Some(vec![BindParam::Integer(version)]), + ) + .await?; + tx.commit().await +} + +pub async fn record_subscription_wake( + tx: &SqliteTransaction, + before: &str, + origin: &str, +) -> Result<()> { + tx.execute( + "INSERT INTO subscription_wake_writes (version, before_offset, after_offset, incarnation,origin_sub_key) \ + SELECT version, ?, tail, incarnation,? FROM subscription_outbox WHERE singleton = 1", + Some(vec![BindParam::Text(before.to_owned()),BindParam::Text(origin.to_owned())]), + ) + .await?; + Ok(()) +} + +pub async fn set_subscription_start(tx: &SqliteTransaction, offset: &str) -> Result<()> { + tx.execute( + "UPDATE subscription_outbox SET start_offset = ? WHERE singleton = 1", + Some(vec![BindParam::Text(offset.to_owned())]), + ) + .await?; + Ok(()) +} + pub async fn ensure_schema(db: &SqliteDb) -> Result<()> { db.exec(SCHEMA_VERSION_TABLE) .await @@ -318,6 +441,16 @@ pub async fn create_meta( ]), ) .await?; + tx.execute( + "INSERT INTO subscription_outbox \ + (singleton, version, dirty, incarnation, tail, start_offset, stream_exists) \ + SELECT 1, 1, 1, gen, current_offset, current_offset, 1 FROM meta WHERE id = 1 \ + ON CONFLICT(singleton) DO UPDATE SET version = subscription_outbox.version + 1, \ + dirty = 1, incarnation = excluded.incarnation, tail = excluded.tail, \ + start_offset = excluded.start_offset, stream_exists = 1", + None, + ) + .await?; Ok(()) } @@ -351,6 +484,12 @@ pub async fn append_message( Some(vec![BindParam::Text(next.clone())]), ) .await?; + tx.execute( + "UPDATE subscription_outbox SET version = version + 1, dirty = 1, tail = ? \ + WHERE singleton = 1", + Some(vec![BindParam::Text(next.clone())]), + ) + .await?; Ok(next) } @@ -382,6 +521,7 @@ pub async fn set_closed(tx: &SqliteTransaction, closed_by: Option) -> Re } pub async fn purge(tx: &SqliteTransaction) -> Result<()> { + mark_subscription_deleted(tx).await?; for statement in [ "DELETE FROM meta", "DELETE FROM messages", @@ -394,11 +534,22 @@ pub async fn purge(tx: &SqliteTransaction) -> Result<()> { } pub async fn set_soft_deleted(tx: &SqliteTransaction) -> Result<()> { + mark_subscription_deleted(tx).await?; tx.execute("UPDATE meta SET soft_deleted = 1 WHERE id = 1", None) .await?; Ok(()) } +async fn mark_subscription_deleted(tx: &SqliteTransaction) -> Result<()> { + tx.execute( + "UPDATE subscription_outbox SET version = version + 1, dirty = 1, stream_exists = 0 \ + WHERE singleton = 1 AND stream_exists = 1", + None, + ) + .await?; + Ok(()) +} + pub async fn fork_edge_count(tx: &SqliteTransaction) -> Result { let result = tx.execute("SELECT COUNT(*) FROM fork_edges", None).await?; result diff --git a/src/subscriptions/catalog_import.rs b/src/subscriptions/catalog_import.rs new file mode 100644 index 0000000..ceba306 --- /dev/null +++ b/src/subscriptions/catalog_import.rs @@ -0,0 +1,339 @@ +//! Import actors created before the durable stream catalog existed. +use std::{collections::HashMap, time::Duration}; + +use anyhow::{Context, Result, ensure}; +use futures_util::{StreamExt, TryStreamExt, stream}; +use rivetkit_client::{Client, GetOptions}; +use serde::Deserialize; + +use super::model::CatalogEntry; + +const PAGE_SIZE: usize = 256; +const MAX_PAGE_SIZE: usize = 8_192; +const MAX_RESPONSE_BYTES: usize = 16 * 1024 * 1024; + +#[derive(Clone)] +pub struct CatalogApi { + pub endpoint: String, + pub namespace: String, + pub token: Option, +} + +#[derive(Deserialize)] +struct ActorPage { + actors: Vec, + pagination: Pagination, +} + +#[derive(Deserialize)] +struct ListedActor { + actor_id: String, + name: String, + create_ts: i64, +} + +#[derive(Deserialize)] +struct Pagination { + cursor: Option, +} + +#[derive(Deserialize)] +struct Snapshot { + tenant_scope: String, + entry: Option, +} + +pub(crate) async fn import(client: &Client, api: &CatalogApi, tenant: &str) -> Result<()> { + let mut url = url::Url::parse(&api.endpoint).context("parse catalog Engine endpoint")?; + ensure!( + matches!(url.scheme(), "http" | "https") + && url.username().is_empty() + && url.password().is_none(), + "catalog Engine endpoint must be HTTP(S) with credentials configured separately" + ); + url.set_path(&format!("{}/actors", url.path().trim_end_matches('/'))); + url.set_query(None); + url.set_fragment(None); + let http = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(30)) + .build()?; + let manager = super::handle(client, tenant)?; + let ready = manager + .action( + "__durableStreamCatalogReady", + vec![serde_json::json!({"complete":false})], + ) + .await?; + if ready == serde_json::Value::Bool(true) { + return Ok(()); + } + let mut pager = Pager::default(); + + loop { + let mut request = http.get(url.clone()).query(&[ + ("namespace", api.namespace.clone()), + ("name", crate::actor::ACTOR_NAME.to_owned()), + ("include_destroyed", "false".to_owned()), + ("limit", pager.limit.to_string()), + ]); + if let Some(cursor) = pager.cursor { + request = request.query(&[("cursor", cursor.to_string())]); + } + if let Some(token) = &api.token { + request = request.bearer_auth(token); + } + let response = request + .send() + .await + .context("list existing stream actors")? + .error_for_status() + .context("Engine rejected catalog actor listing")?; + ensure!( + response + .content_length() + .is_none_or(|n| n <= MAX_RESPONSE_BYTES as u64), + "Engine catalog listing exceeds response size limit" + ); + let mut body = Vec::new(); + let mut chunks = response.bytes_stream(); + while let Some(chunk) = chunks.try_next().await? { + ensure!( + body.len() + chunk.len() <= MAX_RESPONSE_BYTES, + "Engine catalog listing exceeds response size limit" + ); + body.extend_from_slice(&chunk); + } + let page = serde_json::from_slice(&body).context("decode Engine catalog actor listing")?; + let (ids, complete) = pager.accept(page)?; + let manager = &manager; + stream::iter(ids) + .map(|id| async move { + tokio::time::timeout(Duration::from_secs(60), async { + let actor = + client.get_for_id(crate::actor::ACTOR_NAME, &id, GetOptions::default())?; + let value = actor + .action("__durableStreamSubscriptionSnapshot", vec![]) + .await?; + let snapshot: Snapshot = serde_json::from_value(value) + .context("decode stream subscription snapshot")?; + if snapshot.tenant_scope == tenant + && let Some(entry) = snapshot.entry + { + manager + .action( + "__durableStreamCatalogChanged", + vec![serde_json::to_value(entry)?], + ) + .await?; + } + Ok::<_, anyhow::Error>(()) + }) + .await + .context("catalog actor import timed out")? + .with_context(|| format!("import stream actor {id}")) + }) + .buffer_unordered(8) + .try_collect::>() + .await?; + if complete { + manager + .action( + "__durableStreamCatalogReady", + vec![serde_json::json!({"complete":true})], + ) + .await?; + return Ok(()); + } + } +} + +struct Pager { + cursor: Option, + limit: usize, + seen: HashMap, +} + +impl Default for Pager { + fn default() -> Self { + Self { + cursor: None, + limit: PAGE_SIZE, + seen: HashMap::new(), + } + } +} + +impl Pager { + fn accept(&mut self, page: ActorPage) -> Result<(Vec, bool)> { + ensure!( + page.actors.len() <= self.limit, + "Engine exceeded catalog page limit" + ); + ensure!( + page.actors + .windows(2) + .all(|pair| pair[0].create_ts >= pair[1].create_ts), + "Engine catalog listing is not ordered by creation time" + ); + ensure!( + page.actors + .iter() + .all(|actor| actor.name == crate::actor::ACTOR_NAME + && !actor.actor_id.is_empty() + && self.cursor.is_none_or(|cursor| actor.create_ts <= cursor)), + "Engine returned an actor outside the catalog query" + ); + let last_ts = page.actors.last().map(|actor| actor.create_ts); + let returned_cursor = page + .pagination + .cursor + .as_deref() + .map(str::parse::) + .transpose() + .context("invalid Engine catalog pagination cursor")?; + ensure!( + returned_cursor == last_ts, + "Engine catalog cursor does not match its page" + ); + let complete = page.actors.len() < self.limit; + if !complete { + // Engine 2.3.12 cursors contain only milliseconds, without an actor-ID + // tiebreaker. Overlap both inclusive and exclusive implementations to + // avoid dropping actors sharing the last timestamp in a full page. + let next = last_ts + .context("missing full catalog page timestamp")? + .checked_add(1) + .context("catalog timestamp cannot be paginated")?; + if self.cursor.is_some_and(|cursor| next >= cursor) { + ensure!( + self.limit < MAX_PAGE_SIZE, + "catalog migration cannot advance past a timestamp boundary within {MAX_PAGE_SIZE} actors" + ); + self.limit = (self.limit * 2).min(MAX_PAGE_SIZE); + } else { + self.cursor = Some(next); + self.limit = PAGE_SIZE; + } + } + let ids = page + .actors + .into_iter() + .filter_map(|actor| { + self.seen + .insert(actor.actor_id.clone(), actor.create_ts) + .is_none() + .then_some(actor.actor_id) + }) + .collect(); + if let Some(cursor) = self.cursor { + // Only the overlapping boundary can reappear. Keep migration memory + // bounded by a page instead of the total number of existing actors. + self.seen.retain(|_, timestamp| *timestamp <= cursor); + } + Ok((ids, complete)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashSet; + + fn page(actors: Vec) -> ActorPage { + let cursor = actors.last().map(|actor| actor.create_ts.to_string()); + ActorPage { + actors, + pagination: Pagination { cursor }, + } + } + + fn actor(id: usize, create_ts: i64) -> ListedActor { + ListedActor { + actor_id: id.to_string(), + name: crate::actor::ACTOR_NAME.to_owned(), + create_ts, + } + } + + #[test] + fn pagination_keeps_all_timestamp_ties_for_both_engine_boundary_rules() { + let timestamps: Vec<_> = (0..900).map(|i| 100 - i / 300).collect(); + for inclusive in [false, true] { + let mut pager = Pager::default(); + let mut imported = HashSet::new(); + let mut complete = false; + for _ in 0..20 { + let rows = timestamps + .iter() + .enumerate() + .filter(|(_, ts)| { + pager.cursor.is_none_or(|cursor| { + if inclusive { + **ts <= cursor + } else { + **ts < cursor + } + }) + }) + .take(pager.limit) + .map(|(id, ts)| actor(id, *ts)) + .collect(); + let (ids, done) = pager.accept(page(rows)).unwrap(); + for id in ids { + assert!(imported.insert(id), "duplicate import"); + } + if done { + complete = true; + break; + } + } + assert!(complete, "pagination did not terminate"); + assert_eq!(imported.len(), timestamps.len()); + } + } + + #[test] + fn pagination_handles_empty_and_exact_size_final_pages() { + let mut pager = Pager::default(); + assert!(pager.accept(page(vec![])).unwrap().1); + let rows = (0..PAGE_SIZE) + .map(|id| actor(id, 1000 - id as i64)) + .collect(); + assert!(!pager.accept(page(rows)).unwrap().1); + let (ids, complete) = pager + .accept(page(vec![actor( + PAGE_SIZE - 1, + 1000 - (PAGE_SIZE - 1) as i64, + )])) + .unwrap(); + assert!(ids.is_empty()); + assert!(complete); + } + + #[test] + fn pagination_fails_instead_of_skipping_an_unbounded_timestamp_group() { + let mut pager = Pager { + cursor: Some(101), + limit: MAX_PAGE_SIZE, + seen: HashMap::new(), + }; + let rows = (0..MAX_PAGE_SIZE).map(|id| actor(id, 100)).collect(); + assert!( + pager + .accept(page(rows)) + .unwrap_err() + .to_string() + .contains("cannot advance") + ); + } + + #[test] + fn pagination_rejects_inconsistent_engine_results() { + let mut pager = Pager::default(); + assert!(pager.accept(page(vec![actor(1, 1), actor(2, 2)])).is_err()); + let mut response = page(vec![actor(1, 1)]); + response.pagination.cursor = Some("2".into()); + assert!(pager.accept(response).is_err()); + } +} diff --git a/src/subscriptions/db.rs b/src/subscriptions/db.rs new file mode 100644 index 0000000..29cbe8f --- /dev/null +++ b/src/subscriptions/db.rs @@ -0,0 +1,473 @@ +use anyhow::{Result, anyhow}; +use rivetkit::{BindParam, ColumnValue, SqliteDb, SqliteTransaction}; + +use super::{ + legacy::{LegacyKeys, TOKEN_RETENTION_MS}, + model::{CatalogEntry, Subscription}, +}; + +pub async fn begin(db: &SqliteDb) -> Result { + // Maximum snapshots require hundreds of bounded Engine RPCs; a replacement + // takes about seven seconds locally. Keep ordinary stream writes at five. + db.begin_transaction(Some(std::time::Duration::from_secs(10))) + .await +} + +pub async fn ensure(db: &SqliteDb) -> Result<()> { + for sql in [ + "CREATE TABLE IF NOT EXISTS subscription_keys (id INTEGER PRIMARY KEY CHECK(id=1), value TEXT NOT NULL)", + "CREATE TABLE IF NOT EXISTS subscription_signing_state (id INTEGER PRIMARY KEY CHECK(id=1), cutover INTEGER NOT NULL)", + "CREATE TABLE IF NOT EXISTS subscription_catalog (path TEXT PRIMARY KEY, version INTEGER NOT NULL, value TEXT NOT NULL)", + "CREATE TABLE IF NOT EXISTS subscriptions (key TEXT PRIMARY KEY, root TEXT NOT NULL, pattern TEXT, due INTEGER, value TEXT NOT NULL)", + "CREATE INDEX IF NOT EXISTS subscriptions_due ON subscriptions(due) WHERE due IS NOT NULL", + "CREATE INDEX IF NOT EXISTS subscriptions_pattern_keys ON subscriptions(root,key) WHERE pattern IS NOT NULL", + "CREATE TABLE IF NOT EXISTS subscription_links (path TEXT NOT NULL, sub_key TEXT NOT NULL, PRIMARY KEY(path,sub_key))", + "CREATE INDEX IF NOT EXISTS subscription_links_sub ON subscription_links(sub_key)", + "CREATE TABLE IF NOT EXISTS subscription_fanout (path TEXT PRIMARY KEY, version INTEGER NOT NULL, cursor TEXT NOT NULL, discover INTEGER NOT NULL, value TEXT NOT NULL)", + "CREATE TABLE IF NOT EXISTS subscription_catalog_status (id INTEGER PRIMARY KEY CHECK(id=1), ready INTEGER NOT NULL)", + "INSERT OR IGNORE INTO subscription_catalog_status(id,ready) VALUES(1,0)", + ] { + db.exec(sql).await?; + } + let tx = crate::store::begin_mutation(db).await?; + if tx + .execute( + "SELECT cutover FROM subscription_signing_state WHERE id=1", + None, + ) + .await? + .rows + .is_empty() + { + let cutover = super::now_ms(); + let rows = tx + .execute("SELECT value FROM subscription_keys WHERE id=1", None) + .await?; + if let Some(row) = rows.rows.first() { + let mut latest_expiry = cutover; + let mut after = String::new(); + loop { + // Extract only bounded metadata; a subscription's full snapshot + // can be several MiB and is unnecessary for this one-time scan. + let page = tx.execute( + "SELECT key,json_extract(value,'$.wake.token_expires'),json_extract(value,'$.wake.issued_at') FROM subscriptions WHERE key>? ORDER BY key LIMIT 128", + Some(vec![text(&after)]), + ).await?; + if page.rows.is_empty() { + break; + } + for entry in &page.rows { + after = string(entry, 0)?.to_owned(); + if let (Some(ColumnValue::Integer(expiry)), Some(ColumnValue::Integer(issued))) = + (entry.get(1), entry.get(2)) + && *issued < cutover + { + latest_expiry = latest_expiry + .max((*expiry).min(cutover.saturating_add(TOKEN_RETENTION_MS))); + } + } + } + let legacy = LegacyKeys::migrate(string(row, 0)?, cutover, latest_expiry)?; + save_legacy(&tx, &legacy).await?; + } + tx.execute( + "INSERT INTO subscription_signing_state(id,cutover) VALUES(1,?)", + Some(vec![BindParam::Integer(cutover)]), + ) + .await?; + } + tx.commit().await +} +pub fn text(s: impl Into) -> BindParam { + BindParam::Text(s.into()) +} +pub fn string(row: &[ColumnValue], i: usize) -> Result<&str> { + match row.get(i) { + Some(ColumnValue::Text(v)) => Ok(v), + _ => Err(anyhow!("Expected subscription text column")), + } +} +pub fn key(root: &str, id: &str) -> String { + format!("{root}/{id}") +} +pub async fn legacy(db: &SqliteDb) -> Result> { + let result = db + .query("SELECT value FROM subscription_keys WHERE id=1", None) + .await?; + result + .rows + .first() + .map(|row| Ok(serde_json::from_str(string(row, 0)?)?)) + .transpose() +} + +async fn save_legacy(tx: &SqliteTransaction, keys: &LegacyKeys) -> Result<()> { + if keys.next_deadline().is_none() { + tx.execute("DELETE FROM subscription_keys WHERE id=1", None) + .await?; + return Ok(()); + } + let value = serde_json::to_string(keys)?; + let inline = value.len() <= 60 * 1024; + tx.execute( + "UPDATE subscription_keys SET value=? WHERE id=1", + Some(vec![text(if inline { &value } else { "" })]), + ) + .await?; + if !inline { + for chunk in value.as_bytes().chunks(60 * 1024) { + tx.execute( + "UPDATE subscription_keys SET value=CAST(value || ? AS TEXT) WHERE id=1", + Some(vec![BindParam::Blob(chunk.to_vec())]), + ) + .await?; + } + } + Ok(()) +} + +/// Call under the coordinator's mutation lock and schedule the returned deadline. +pub async fn cleanup_legacy(db: &SqliteDb, now: i64) -> Result> { + let Some(mut keys) = legacy(db).await? else { + return Ok(None); + }; + if keys.next_deadline().is_some_and(|deadline| deadline > now) { + return Ok(keys.next_deadline()); + } + keys.cleanup(now); + let tx = crate::store::begin_mutation(db).await?; + save_legacy(&tx, &keys).await?; + tx.commit().await?; + Ok(keys.next_deadline()) +} +pub async fn load(tx: &SqliteTransaction, key: &str) -> Result> { + tx.execute( + "SELECT value FROM subscriptions WHERE key=?", + Some(vec![text(key)]), + ) + .await? + .rows + .first() + .map(|r| Ok(serde_json::from_str(string(r, 0)?)?)) + .transpose() +} +pub async fn save(tx: &SqliteTransaction, sub: &Subscription) -> Result<()> { + let value = stage(tx, sub).await?; + write(tx, sub, value, None).await +} + +/// Start or renew a lease after uploading the snapshot, not before that work. +pub async fn save_with_lease(tx: &SqliteTransaction, sub: &mut Subscription) -> Result<()> { + let value = stage(tx, sub).await?; + let now = super::now_ms(); + let wake = sub + .wake + .as_mut() + .ok_or_else(|| anyhow!("Cannot lease a subscription without a wake"))?; + if wake.token_expires <= now { + return Err(anyhow::Error::new(super::model::ApiError::new( + 401, + "TOKEN_EXPIRED", + "Subscription capability expired before lease persistence", + ))); + } + let deadline = now + .saturating_add(sub.config.lease_ttl_ms) + .min(wake.token_expires); + wake.deadline = Some(deadline); + write(tx, sub, value, Some(deadline)).await +} + +async fn stage(tx: &SqliteTransaction, sub: &Subscription) -> Result { + let value = serde_json::to_string(sub)?; + let inline = value.len() <= 60 * 1024; + if !inline { + // Stage bounded blobs, then assemble once. Repeated value || chunk + // copies the entire growing snapshot and becomes quadratic at MiB sizes. + // Engine limits each statement's decoded binds to 128 KiB; leave room + // for the part number while halving the serialized RPC round trips. + // Temporary staging avoids committing a second copy of the snapshot. + tx.execute( + "CREATE TEMP TABLE IF NOT EXISTS subscription_value_parts (part INTEGER PRIMARY KEY, value BLOB NOT NULL)", + None, + ).await?; + for (part, chunk) in value.as_bytes().chunks(120 * 1024).enumerate() { + tx.execute( + "INSERT INTO temp.subscription_value_parts(part,value) VALUES(?,?)", + Some(vec![ + BindParam::Integer(part as i64), + BindParam::Blob(chunk.to_vec()), + ]), + ) + .await?; + } + } + Ok(value) +} + +async fn write( + tx: &SqliteTransaction, + sub: &Subscription, + value: String, + deadline: Option, +) -> Result<()> { + let inline = value.len() <= 60 * 1024; + let mut params = vec![ + text(key(&sub.root, &sub.id)), + text(&sub.root), + sub.config + .pattern + .as_ref() + .filter(|_| !sub.deleted) + .map_or(BindParam::Null, text), + sub.due().map_or(BindParam::Null, BindParam::Integer), + ]; + let value_sql = if inline { + params.push(text(value)); + "?" + } else { + "CAST((SELECT group_concat(value,'') FROM (SELECT value FROM temp.subscription_value_parts ORDER BY part)) AS TEXT)" + }; + let value_sql = if let Some(deadline) = deadline { + params.push(BindParam::Integer(deadline)); + format!("json_set({value_sql},'$.wake.deadline',?)") + } else { + value_sql.to_owned() + }; + // Keep the previous snapshot intact until its replacement is assembled. + // Clearing it first needlessly writes and frees all its pages twice. + tx.execute(format!("INSERT INTO subscriptions(key,root,pattern,due,value) VALUES(?,?,?,?,{value_sql}) ON CONFLICT(key) DO UPDATE SET pattern=excluded.pattern,due=excluded.due,value=excluded.value"), Some(params)).await?; + if !inline { + tx.execute("DELETE FROM temp.subscription_value_parts", None) + .await?; + } + Ok(()) +} +pub async fn sync_links(tx: &SqliteTransaction, sub: &Subscription) -> Result<()> { + let key = key(&sub.root, &sub.id); + let rows = tx + .execute( + "SELECT path FROM subscription_links WHERE sub_key=?", + Some(vec![text(&key)]), + ) + .await?; + let old = rows + .rows + .iter() + .map(|r| string(r, 0).map(str::to_owned)) + .collect::>>()?; + let new = if sub.deleted { + Default::default() + } else { + sub.paths() + }; + for paths in link_batches(old.difference(&new), key.len() + 8, 8)? { + let mut params = vec![text(&key)]; + params.extend(paths.iter().map(|path| text(*path))); + tx.execute( + format!( + "DELETE FROM subscription_links WHERE sub_key=? AND path IN ({})", + vec!["?"; paths.len()].join(",") + ), + Some(params), + ) + .await?; + } + for paths in link_batches(new.difference(&old), 0, key.len() + 16)? { + let params = paths + .iter() + .flat_map(|path| [text(*path), text(&key)]) + .collect(); + tx.execute( + format!( + "INSERT INTO subscription_links(path,sub_key) VALUES {}", + vec!["(?,?)"; paths.len()].join(",") + ), + Some(params), + ) + .await?; + } + Ok(()) +} + +fn link_batches<'a>( + paths: impl IntoIterator, + base: usize, + per_row: usize, +) -> Result>> { + // Keep both encoded binds and SQLite's parameter count bounded. The size + // allowance includes the string length tags used by the remote transport. + const BUDGET: usize = 60 * 1024; + let mut batches = Vec::new(); + let mut batch = Vec::new(); + let mut bytes = base; + for path in paths { + let size = path.len() + per_row; + if base + size > BUDGET { + return Err(anyhow!("Subscription link exceeds parameter budget")); + } + if bytes + size > BUDGET || batch.len() == 256 { + batches.push(std::mem::take(&mut batch)); + bytes = base; + } + bytes += size; + batch.push(path); + } + if !batch.is_empty() { + batches.push(batch); + } + Ok(batches) +} +pub async fn catalog(tx: &SqliteTransaction, path: &str) -> Result> { + tx.execute( + "SELECT value FROM subscription_catalog WHERE path=?", + Some(vec![text(path)]), + ) + .await? + .rows + .first() + .map(|r| Ok(serde_json::from_str(string(r, 0)?)?)) + .transpose() +} +pub async fn save_catalog(tx: &SqliteTransaction, entry: &CatalogEntry) -> Result { + let old = catalog(tx, &entry.path).await?; + if old.as_ref().is_some_and(|old| old.version >= entry.version) { + return Ok(false); + } + let active = tx + .execute( + "SELECT version FROM subscription_fanout WHERE path=?", + Some(vec![text(&entry.path)]), + ) + .await?; + let mut latest = entry.clone(); + if !active.rows.is_empty() + && let Some(old) = &old + && old.incarnation == entry.incarnation + { + // A newer tail cannot discard synthetic spans before the prior fanout + // has reached every subscriber. Delivery versions remain immutable. + let mut offsets = latest + .wakes + .iter() + .map(|s| s.after.clone()) + .collect::>(); + for span in &old.wakes { + if offsets.insert(span.after.clone()) { + latest.wakes.push(span.clone()); + } + } + } + write_catalog(tx, &latest).await?; + if active.rows.is_empty() { + let discover = old + .as_ref() + .is_none_or(|old| !old.exists || old.incarnation != entry.incarnation); + save_fanout(tx, &latest, discover).await?; + } + Ok(true) +} + +async fn write_catalog(tx: &SqliteTransaction, entry: &CatalogEntry) -> Result<()> { + let value = serde_json::to_string(entry)?; + let inline = value.len() <= 60 * 1024; + tx.execute("INSERT INTO subscription_catalog(path,version,value) VALUES(?,?,?) ON CONFLICT(path) DO UPDATE SET version=excluded.version,value=excluded.value",Some(vec![text(&entry.path),BindParam::Integer(entry.version),text(if inline{&value}else{""})])).await?; + if !inline { + for chunk in value.as_bytes().chunks(60 * 1024) { + tx.execute( + "UPDATE subscription_catalog SET value=CAST(value || ? AS TEXT) WHERE path=?", + Some(vec![BindParam::Blob(chunk.to_vec()), text(&entry.path)]), + ) + .await?; + } + } + Ok(()) +} + +pub struct Fanout { + pub entry: CatalogEntry, + pub cursor: String, + pub discover: bool, +} + +pub async fn fanout(tx: &SqliteTransaction, path: Option<&str>) -> Result> { + let rows = match path { + Some(path) => { + tx.execute( + "SELECT value,cursor,discover FROM subscription_fanout WHERE path=?", + Some(vec![text(path)]), + ) + .await? + } + None => { + tx.execute( + "SELECT value,cursor,discover FROM subscription_fanout ORDER BY rowid LIMIT 1", + None, + ) + .await? + } + }; + rows.rows + .first() + .map(|r| { + Ok(Fanout { + entry: serde_json::from_str(string(r, 0)?)?, + cursor: string(r, 1)?.to_owned(), + discover: matches!(r.get(2), Some(ColumnValue::Integer(1))), + }) + }) + .transpose() +} + +async fn save_fanout(tx: &SqliteTransaction, entry: &CatalogEntry, discover: bool) -> Result<()> { + let value = serde_json::to_string(entry)?; + let inline = value.len() <= 60 * 1024; + tx.execute("INSERT INTO subscription_fanout(path,version,cursor,discover,value) VALUES(?,?,'',?,?) ON CONFLICT(path) DO UPDATE SET version=excluded.version,cursor='',discover=excluded.discover,value=excluded.value",Some(vec![text(&entry.path),BindParam::Integer(entry.version),BindParam::Integer(i64::from(discover)),text(if inline{&value}else{""})])).await?; + if !inline { + for chunk in value.as_bytes().chunks(60 * 1024) { + tx.execute( + "UPDATE subscription_fanout SET value=CAST(value || ? AS TEXT) WHERE path=?", + Some(vec![BindParam::Blob(chunk.to_vec()), text(&entry.path)]), + ) + .await?; + } + } + Ok(()) +} + +pub async fn advance_fanout( + tx: &SqliteTransaction, + job: &Fanout, + cursor: &str, + complete: bool, +) -> Result<()> { + if !complete { + tx.execute( + "UPDATE subscription_fanout SET cursor=? WHERE path=?", + Some(vec![text(cursor), text(&job.entry.path)]), + ) + .await?; + return Ok(()); + } + let mut latest = catalog(tx, &job.entry.path) + .await? + .ok_or_else(|| anyhow!("Fanout catalog entry missing"))?; + // Delete/reinsert rotates a busy path behind other pending paths. + tx.execute( + "DELETE FROM subscription_fanout WHERE path=?", + Some(vec![text(&job.entry.path)]), + ) + .await?; + if latest.version > job.entry.version { + let discover = !job.entry.exists || job.entry.incarnation != latest.incarnation; + latest.wakes.retain(|span| { + span.incarnation != job.entry.incarnation || span.after > job.entry.tail + }); + write_catalog(tx, &latest).await?; + save_fanout(tx, &latest, discover).await?; + } else if !latest.wakes.is_empty() { + latest.wakes.clear(); + write_catalog(tx, &latest).await?; + } + Ok(()) +} diff --git a/src/subscriptions/engine.rs b/src/subscriptions/engine.rs new file mode 100644 index 0000000..a653135 --- /dev/null +++ b/src/subscriptions/engine.rs @@ -0,0 +1,1781 @@ +//! Engine owns issuance and signing. This adapter only caches public JWT keys. +use std::{ + collections::{HashMap, HashSet}, + fmt, + sync::Arc, + time::{Duration, Instant}, +}; + +use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; +use bytes::Bytes; +use futures_util::StreamExt; +use jsonwebtoken::{Algorithm, DecodingKey, Validation}; +use reqwest::{ + Client, RequestBuilder, + header::{AGE, CACHE_CONTROL}, +}; +use serde::{Deserialize, Serialize, de::DeserializeOwned}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use tokio::sync::{Mutex, OnceCell, RwLock}; + +use super::catalog_import::CatalogApi; + +const MAX_RESPONSE: usize = 256 * 1024; +const MAX_TOKEN: usize = 6144; +const MAX_LIFETIME: u64 = 86400; +const KEY_REFRESH_INTERVAL: Duration = Duration::from_secs(5); +const MAX_BODY_LIMIT: usize = 128 * 1024 * 1024; + +/// Messages never include an upstream body, URL, credential, or signed artifact. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EngineError { + pub status: u16, + pub code: &'static str, + pub retry_after: Option, +} +impl EngineError { + fn new(status: u16, code: &'static str) -> Self { + Self { + status, + code, + retry_after: None, + } + } + fn invalid() -> Self { + Self::new(401, "TOKEN_INVALID") + } + fn unavailable() -> Self { + Self::new(503, "ENGINE_UNAVAILABLE") + } + fn response() -> Self { + Self::new(503, "ENGINE_INVALID_RESPONSE") + } +} +impl fmt::Display for EngineError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{} (HTTP {})", self.code, self.status) + } +} +impl std::error::Error for EngineError {} +type Result = std::result::Result; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CapabilityKind { + Callback, + Claim, +} +impl CapabilityKind { + pub fn purpose(self) -> &'static str { + match self { + Self::Callback => "durable-streams.callback", + Self::Claim => "durable-streams.claim", + } + } + fn operations(self) -> &'static [&'static str] { + match self { + Self::Callback => &["callback"], + Self::Claim => &["ack", "release"], + } + } +} + +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct CapabilityClaims { + pub root: String, + pub tenant_scope: String, + pub subscription_id: String, + pub wake_id: String, + pub generation: i64, + pub operations: Vec, +} + +#[derive(Deserialize)] +pub struct IssuedToken { + pub token: String, + pub issued_ts: i64, + pub expires_ts: i64, +} + +pub struct VerifiedCapability { + pub claims: CapabilityClaims, + pub expires_ts: i64, +} + +#[derive(Clone, Deserialize)] +pub struct WebhookMetadata { + pub namespace_id: String, + pub alg: String, + pub kid: String, + pub jwks_url: String, + pub generation: u64, +} + +#[derive(Deserialize)] +pub struct Signature { + pub timestamp: String, + pub kid: String, + pub signature: String, +} +impl Signature { + pub fn header(&self) -> String { + format!( + "t={},kid={},ed25519={}", + self.timestamp, self.kid, self.signature + ) + } +} + +/// Preserve `max-age=300` on the DS response and propagate this age in `Age`. +pub struct FreshJwks { + pub value: Value, + pub age_secs: u64, +} + +#[derive(Clone)] +pub struct EngineClient { + inner: Arc, +} +struct Inner { + http: Client, + endpoint: url::Url, + issuer: String, + namespace: String, + token: Option, + namespace_id: OnceCell, + jwt_keys: RwLock, + jwt_refresh: Mutex<()>, + webhook_identity: OnceCell, + webhook_kids: RwLock>, +} +#[derive(Default)] +struct KeyCache { + keys: HashMap, + expires: Option, + last_attempt: Option, + last_error: Option, +} + +impl EngineClient { + /// `issuer` is deployment configuration, never a value learned from a token. + pub fn new(api: &CatalogApi, issuer: Option<&str>) -> Result { + let endpoint = trusted_url(&api.endpoint)?; + let issuer = trusted_url(issuer.unwrap_or(&api.endpoint))? + .as_str() + .trim_end_matches('/') + .to_owned(); + if issuer.len() > 512 || api.namespace.is_empty() { + return Err(EngineError::new(503, "ENGINE_CONFIGURATION")); + } + let http = Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(35)) + .connect_timeout(Duration::from_secs(5)) + .pool_max_idle_per_host(8) + .build() + .map_err(|_| EngineError::unavailable())?; + Ok(Self { + inner: Arc::new(Inner { + http, + endpoint, + issuer, + namespace: api.namespace.clone(), + token: api.token.clone(), + namespace_id: OnceCell::new(), + jwt_keys: RwLock::new(KeyCache::default()), + jwt_refresh: Mutex::new(()), + webhook_identity: OnceCell::new(), + webhook_kids: RwLock::new(HashSet::new()), + }), + }) + } + + fn url(&self, path: &str) -> url::Url { + let mut url = self.inner.endpoint.clone(); + url.set_path(&format!("{}{}", url.path().trim_end_matches('/'), path)); + url + } + fn authenticate(&self, request: RequestBuilder) -> Result { + let token = self + .inner + .token + .as_deref() + .filter(|v| !v.is_empty()) + .ok_or_else(|| EngineError::new(401, "ENGINE_CREDENTIAL_REQUIRED"))?; + Ok(request.bearer_auth(token)) + } + + /// Resolve the existing configured namespace once; callers persist this ID to + /// prevent namespace-name reuse from silently changing subscription authority. + pub async fn namespace_id(&self) -> Result { + self.inner + .namespace_id + .get_or_try_init(|| async { + #[derive(Deserialize)] + struct Namespace { + namespace_id: String, + name: String, + } + #[derive(Deserialize)] + struct Namespaces { + namespaces: Vec, + } + let request = self.authenticate( + self.inner + .http + .get(self.url("/namespaces")) + .query(&[("name", self.inner.namespace.as_str())]) + .timeout(Duration::from_secs(5)), + )?; + let response: Namespaces = self.json(request).await?; + let mut matches = response + .namespaces + .into_iter() + .filter(|n| n.name == self.inner.namespace); + let namespace = matches + .next() + .ok_or_else(|| EngineError::new(404, "ENGINE_NAMESPACE_NOT_FOUND"))?; + if matches.next().is_some() || !identifier(&namespace.namespace_id) { + return Err(EngineError::response()); + } + Ok(namespace.namespace_id) + }) + .await + .cloned() + } + + pub async fn ensure(&self) -> Result { + self.inner + .webhook_identity + .get_or_try_init(|| async { + let namespace = self.namespace_id().await?; + let request = self.authenticate( + self.inner + .http + .post(self.url("/auth/webhooks/ensure")) + .timeout(Duration::from_secs(10)), + )?; + let metadata = self.checked_metadata(self.json(request).await?, &namespace)?; + *self.inner.webhook_kids.write().await = HashSet::from([metadata.kid.clone()]); + Ok(metadata) + }) + .await + .cloned() + } + + pub async fn metadata(&self, namespace: &str) -> Result { + if self.ensure().await?.namespace_id != namespace { + return Err(EngineError::response()); + } + // Ensure binds this credential before advertising even an existing ring. + // Metadata still comes from a fresh read, not the cached ensure response. + self.metadata_read(namespace).await + } + + async fn metadata_read(&self, namespace: &str) -> Result { + let path = namespace_path(namespace)?; + let request = self + .inner + .http + .get(self.url(&path)) + .timeout(Duration::from_secs(5)); + self.checked_metadata(self.json(request).await?, namespace) + } + + fn checked_metadata( + &self, + metadata: WebhookMetadata, + namespace: &str, + ) -> Result { + if metadata.namespace_id != namespace + || metadata.alg != "ed25519" + || !webhook_kid(&metadata.kid) + || metadata.generation == 0 + || trusted_url(&metadata.jwks_url).is_err() + { + return Err(EngineError::response()); + } + // Discovery always uses our configured Engine origin, never this advisory URL. + Ok(metadata) + } + + /// No private adapter cache: every DS public response needs an authoritative read. + pub async fn webhook_jwks(&self, namespace: &str) -> Result { + if self.namespace_id().await? != namespace { + return Err(EngineError::response()); + } + self.ensure().await?; + let path = format!("{}/jwks.json", namespace_path(namespace)?); + let request = || { + self.inner + .http + .get(self.url(&path)) + .header(CACHE_CONTROL, "no-cache") + .timeout(Duration::from_secs(5)) + }; + // The ensured identity is cached; the public key snapshot never is. + let reply = response(request()).await?; + let jwks: JwkSet = decode(&reply.body)?; + validate_jwks(&jwks, true)?; + let age_secs = reply.age_secs(); + if age_secs >= 300 { + return Err(EngineError::unavailable()); + } + // This bounded set records namespace ownership only; it never serves + // discovery responses or takes over Engine's rotation/revocation policy. + *self.inner.webhook_kids.write().await = + jwks.keys.iter().map(|key| key.kid.clone()).collect(); + Ok(FreshJwks { + value: serde_json::to_value(jwks).map_err(|_| EngineError::response())?, + age_secs, + }) + } + + pub async fn sign(&self, body: Bytes) -> Result { + self.sign_with_limit(body, MAX_BODY_LIMIT).await + } + + pub async fn sign_with_limit(&self, body: Bytes, limit: usize) -> Result { + if !(1024 * 1024..=128 * 1024 * 1024).contains(&limit) { + return Err(EngineError::new(503, "ENGINE_CONFIGURATION")); + } + if body.len() > limit { + return Err(EngineError::new(413, "ENGINE_PAYLOAD_TOO_LARGE")); + } + let namespace = self.ensure().await?.namespace_id; + let request = self.authenticate( + self.inner + .http + .post(self.url("/auth/webhooks/sign")) + .header("content-type", "application/octet-stream") + .body(body), + )?; + let signature: Signature = self.json(request).await?; + if signature + .timestamp + .parse::() + .ok() + .filter(|v| v.to_string() == signature.timestamp) + .is_none() + || !webhook_kid(&signature.kid) + || URL_SAFE_NO_PAD + .decode(&signature.signature) + .map_or(true, |v| v.len() != 64) + { + return Err(EngineError::response()); + } + if !self + .inner + .webhook_kids + .read() + .await + .contains(&signature.kid) + { + // ACL roles are mutable. A new signature kid must belong to the + // configured namespace, even after an earlier authenticated ensure. + let fresh = self.webhook_jwks(&namespace).await?; + if !fresh.value["keys"] + .as_array() + .is_some_and(|keys| keys.iter().any(|key| key["kid"] == signature.kid)) + { + return Err(EngineError::response()); + } + } + Ok(signature) + } + + pub async fn issue( + &self, + claims: &CapabilityClaims, + kind: CapabilityKind, + namespace: &str, + ) -> Result { + if !valid_capability(claims, kind) { + return Err(EngineError::new(400, "ENGINE_INVALID_CAPABILITY")); + } + let app = json!({"durable_streams": claims}); + if serde_json::to_vec(&app) + .map_err(|_| EngineError::response())? + .len() + > 3072 + { + return Err(EngineError::new(413, "ENGINE_CLAIMS_TOO_LARGE")); + } + let body = serde_json::to_vec(&json!({"audience":"durable-streams", "purpose":kind.purpose(), "duration":MAX_LIFETIME, "claims":app})).map_err(|_| EngineError::response())?; + let request = self.authenticate( + self.inner + .http + .post(self.url("/auth/application-tokens")) + .timeout(Duration::from_secs(10)) + .header("content-type", "application/json") + .body(body), + )?; + let issued: IssuedToken = self.json(request).await?; + // Verify Engine output before persisting it; this also primes the public-key + // cache for the usual callback path, without trusting returned token claims. + let verified = self + .verify( + &issued.token, + namespace, + kind, + kind.operations()[0], + super::now_ms(), + ) + .await + .map_err(|error| { + // A newly issued token can race emergency key publication or + // expire in transit. Retry the Engine response; it is not a + // rejected caller credential that should pause delivery. + if matches!(error.code, "TOKEN_INVALID" | "TOKEN_EXPIRED") { + EngineError::response() + } else { + error + } + })?; + if verified.claims != *claims + || verified.expires_ts != issued.expires_ts + || issued.issued_ts < 0 + || issued.expires_ts <= issued.issued_ts + || issued.expires_ts - issued.issued_ts > MAX_LIFETIME as i64 * 1000 + { + return Err(EngineError::response()); + } + Ok(issued) + } + + /// Network/cache work happens before the actor takes its state lock. The + /// caller must subsequently validate every returned scope against live state. + pub async fn verify( + &self, + token: &str, + namespace: &str, + kind: CapabilityKind, + operation: &str, + now_ms: i64, + ) -> Result { + if token.len() > MAX_TOKEN || !kind.operations().contains(&operation) { + return Err(EngineError::invalid()); + } + let header = strict_header(token)?; + let key = self.jwt_key(&header.kid).await?; + verify_with_key( + token, + &key, + &self.inner.issuer, + namespace, + kind, + operation, + now_ms, + ) + } + + async fn jwt_key(&self, kid: &str) -> Result { + // Known fresh keys never wait for network work triggered by an attacker + // sending unknown kids. A separate gate coalesces discovery requests. + { + let cache = self.inner.jwt_keys.read().await; + if cache + .expires + .is_some_and(|expires| Instant::now() < expires) + && let Some(key) = cache.keys.get(kid) + { + return Ok(key.clone()); + } + } + let _refresh = self.inner.jwt_refresh.lock().await; + let mut cache = self.inner.jwt_keys.write().await; + let now = Instant::now(); + if cache.expires.is_some_and(|expires| now < expires) + && let Some(key) = cache.keys.get(kid) + { + return Ok(key.clone()); + } + if cache + .last_attempt + .is_some_and(|last| now.duration_since(last) < KEY_REFRESH_INTERVAL) + && (!cache.keys.contains_key(kid) || cache.last_error.is_some()) + { + return Err(cache.last_error.clone().unwrap_or_else(|| { + if cache.keys.contains_key(kid) { + EngineError::unavailable() + } else { + EngineError::invalid() + } + })); + } + cache.last_attempt = Some(now); + drop(cache); + let fetched = self.fetch_jwt_keys().await; + let mut cache = self.inner.jwt_keys.write().await; + match fetched { + Ok((keys, ttl)) => { + cache.keys = keys; + cache.expires = Some(Instant::now() + ttl); + cache.last_error = None; + cache + .keys + .get(kid) + .cloned() + .ok_or_else(EngineError::invalid) + } + Err(error) => { + cache.last_error = Some(error.clone()); + Err(error) + } + } + } + + async fn fetch_jwt_keys(&self) -> Result<(HashMap, Duration)> { + let reply = response( + self.inner + .http + .get(self.url("/.well-known/jwks.json")) + .header(CACHE_CONTROL, "no-cache") + .timeout(Duration::from_secs(5)), + ) + .await?; + let jwks: JwkSet = decode(&reply.body)?; + validate_jwks(&jwks, false)?; + let mut keys = HashMap::with_capacity(jwks.keys.len()); + for key in jwks.keys { + let decoding = + DecodingKey::from_ed_components(&key.x).map_err(|_| EngineError::response())?; + keys.insert(key.kid, decoding); + } + let age = reply.age(); + if reply + .max_age + .is_some_and(|max_age| max_age > 0 && age >= Duration::from_secs(max_age)) + { + return Err(EngineError::unavailable()); + } + let ttl = Duration::from_secs(reply.max_age.unwrap_or(0).min(60)).saturating_sub(age); + Ok((keys, ttl)) + } + + async fn json(&self, request: RequestBuilder) -> Result { + decode(&response(request).await?.body) + } +} + +fn trusted_url(raw: &str) -> Result { + let url = url::Url::parse(raw).map_err(|_| EngineError::new(503, "ENGINE_CONFIGURATION"))?; + if !matches!(url.scheme(), "http" | "https") + || url.host_str().is_none() + || !url.username().is_empty() + || url.password().is_some() + || url.query().is_some() + || url.fragment().is_some() + { + return Err(EngineError::new(503, "ENGINE_CONFIGURATION")); + } + Ok(url) +} +fn identifier(value: &str) -> bool { + !value.is_empty() + && value.len() <= 128 + && value + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"_-".contains(&b)) +} +fn namespace_path(namespace: &str) -> Result { + if !identifier(namespace) { + return Err(EngineError::new(503, "ENGINE_NAMESPACE_INVALID")); + } + Ok(format!("/namespaces/{namespace}/auth/webhooks")) +} +fn webhook_kid(kid: &str) -> bool { + kid.strip_prefix("ds_") + .is_some_and(|v| URL_SAFE_NO_PAD.decode(v).is_ok_and(|b| b.len() == 32)) +} +fn valid_capability(claims: &CapabilityClaims, kind: CapabilityKind) -> bool { + !claims.root.is_empty() + && !claims.subscription_id.is_empty() + && !claims.wake_id.is_empty() + && claims.generation > 0 + && claims.operations.len() == kind.operations().len() + && kind + .operations() + .iter() + .all(|op| claims.operations.iter().filter(|v| *v == op).count() == 1) +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Header { + alg: String, + typ: String, + kid: String, +} +fn strict_header(token: &str) -> Result

{ + let encoded = token.split('.').next().ok_or_else(EngineError::invalid)?; + let bytes = URL_SAFE_NO_PAD + .decode(encoded) + .map_err(|_| EngineError::invalid())?; + let header: Header = serde_json::from_slice(&bytes).map_err(|_| EngineError::invalid())?; + if header.alg != "EdDSA" || header.typ != "rivet-app+jwt" || !identifier(&header.kid) { + return Err(EngineError::invalid()); + } + Ok(header) +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ApplicationClaims { + rivet_ver: u16, + rivet_purpose: String, + iss: String, + aud: String, + #[serde(rename = "sub")] + _subject: Option, + iat: u64, + exp: u64, + jti: String, + rivet_ns: String, + app: ApplicationData, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ApplicationData { + durable_streams: CapabilityClaims, +} +fn verify_with_key( + token: &str, + key: &DecodingKey, + issuer: &str, + namespace: &str, + kind: CapabilityKind, + operation: &str, + now_ms: i64, +) -> Result { + strict_header(token)?; + let mut validation = Validation::new(Algorithm::EdDSA); + validation.set_issuer(&[issuer]); + validation.set_audience(&["durable-streams"]); + // Validate time below against the actor's supplied clock, without the JWT + // library's implicit expiry leeway. Signature verification stays mandatory. + validation.validate_exp = false; + let claims = jsonwebtoken::decode::(token, key, &validation) + .map_err(|_| EngineError::invalid())? + .claims; + let now = u64::try_from(now_ms / 1000).map_err(|_| EngineError::invalid())?; + if claims.rivet_ver != 1 + || claims.rivet_purpose != kind.purpose() + || claims.iss != issuer + || claims.aud != "durable-streams" + || claims.rivet_ns != namespace + || claims.jti.is_empty() + || claims.exp <= claims.iat + || claims.exp - claims.iat > MAX_LIFETIME + || claims.iat > now.saturating_add(30) + || !valid_capability(&claims.app.durable_streams, kind) + || !claims + .app + .durable_streams + .operations + .iter() + .any(|op| op == operation) + { + return Err(EngineError::invalid()); + } + if claims.exp <= now { + return Err(EngineError::new(401, "TOKEN_EXPIRED")); + } + let expires_ts = i64::try_from(claims.exp) + .ok() + .and_then(|exp| exp.checked_mul(1000)) + .ok_or_else(EngineError::invalid)?; + Ok(VerifiedCapability { + claims: claims.app.durable_streams, + expires_ts, + }) +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct JwkSet { + keys: Vec, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Jwk { + kty: String, + crv: String, + kid: String, + #[serde(rename = "use")] + use_: String, + alg: String, + x: String, +} +fn validate_jwks(jwks: &JwkSet, webhook: bool) -> Result<()> { + let mut seen = std::collections::HashSet::new(); + if jwks.keys.len() > 256 { + return Err(EngineError::response()); + } + for key in &jwks.keys { + if key.kty != "OKP" + || key.crv != "Ed25519" + || key.use_ != "sig" + || key.alg != "EdDSA" + || !identifier(&key.kid) + || !seen.insert(&key.kid) + || URL_SAFE_NO_PAD + .decode(&key.x) + .map_or(true, |v| v.len() != 32) + { + return Err(EngineError::response()); + } + if webhook { + let canonical = format!(r#"{{"crv":"Ed25519","kty":"OKP","x":"{}"}}"#, key.x); + let kid = format!( + "ds_{}", + URL_SAFE_NO_PAD.encode(Sha256::digest(canonical.as_bytes())) + ); + if key.kid != kid { + return Err(EngineError::response()); + } + } + } + Ok(()) +} +struct HttpResponse { + body: Vec, + age: Duration, + received_at: Instant, + max_age: Option, +} +impl HttpResponse { + fn age(&self) -> Duration { + self.age.saturating_add(self.received_at.elapsed()) + } + fn age_secs(&self) -> u64 { + let age = self.age(); + age.as_secs() + .saturating_add(u64::from(age.subsec_nanos() > 0)) + } +} +async fn response(request: RequestBuilder) -> Result { + let start = Instant::now(); + let response = request + .send() + .await + .map_err(|_| EngineError::unavailable())?; + let status = response.status().as_u16(); + if !response.status().is_success() { + let code = match status { + 400 => "ENGINE_INVALID_REQUEST", + 401 => "ENGINE_UNAUTHORIZED", + 403 => "ENGINE_FORBIDDEN", + 404 => "ENGINE_NOT_FOUND", + 409 => "ENGINE_CONFLICT", + 413 => "ENGINE_PAYLOAD_TOO_LARGE", + 429 => "ENGINE_RATE_LIMITED", + _ => "ENGINE_UNAVAILABLE", + }; + let retry_after = response + .headers() + .get("retry-after") + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse::().ok()); + return Err(EngineError { + status: if matches!(status, 400 | 401 | 403 | 404 | 409 | 413 | 429) { + status + } else { + 503 + }, + code, + retry_after, + }); + } + if response + .content_length() + .is_some_and(|v| v > MAX_RESPONSE as u64) + { + return Err(EngineError::response()); + } + let age = response + .headers() + .get(AGE) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse::().ok()) + .unwrap_or(0); + let cache_control = response + .headers() + .get(CACHE_CONTROL) + .and_then(|v| v.to_str().ok()) + .unwrap_or(""); + let no_cache = cache_control + .split(',') + .any(|v| matches!(v.trim(), "no-store" | "no-cache")); + let max_age = (!no_cache) + .then(|| { + cache_control.split(',').find_map(|v| { + v.trim() + .strip_prefix("max-age=") + .and_then(|v| v.parse().ok()) + }) + }) + .flatten(); + let mut body = Vec::new(); + let mut chunks = response.bytes_stream(); + while let Some(chunk) = chunks.next().await { + let chunk = chunk.map_err(|_| EngineError::unavailable())?; + if body.len() + chunk.len() > MAX_RESPONSE { + return Err(EngineError::response()); + } + body.extend_from_slice(&chunk); + } + // Preserve precise age for short JWT cache lifetimes, rounding up only + // when exposing the whole-second Age header to downstream webhook clients. + let received_at = Instant::now(); + let age = Duration::from_secs(age).saturating_add(received_at.duration_since(start)); + Ok(HttpResponse { + body, + age, + received_at, + max_age, + }) +} +fn decode(body: &[u8]) -> Result { + serde_json::from_slice(body).map_err(|_| EngineError::response()) +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::{ + Router, + extract::{Request, State}, + response::{IntoResponse, Response}, + routing::any, + }; + use ring::signature::{ED25519, Ed25519KeyPair, KeyPair, UnparsedPublicKey}; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + + fn key() -> Ed25519KeyPair { + Ed25519KeyPair::from_seed_unchecked(&[7; 32]).unwrap() + } + fn capability(kind: CapabilityKind) -> CapabilityClaims { + CapabilityClaims { + root: "https://streams.gameinc.io/v1/stream".into(), + tenant_scope: "tenant-a".into(), + subscription_id: "sub-a".into(), + wake_id: "wake-a".into(), + generation: 7, + operations: kind.operations().iter().map(|v| v.to_string()).collect(), + } + } + fn claims(issuer: &str, kind: CapabilityKind) -> Value { + let now = super::super::now_ms() / 1000; + json!({"iss":issuer,"aud":"durable-streams","rivet_ver":1,"rivet_purpose":kind.purpose(),"rivet_ns":"namespace-a","iat":now,"exp":now+1800,"jti":"token-a","app":{"durable_streams":capability(kind)}}) + } + fn token(claims: &Value, header: &Value) -> String { + let message = format!( + "{}.{}", + URL_SAFE_NO_PAD.encode(serde_json::to_vec(header).unwrap()), + URL_SAFE_NO_PAD.encode(serde_json::to_vec(claims).unwrap()) + ); + format!( + "{message}.{}", + URL_SAFE_NO_PAD.encode(key().sign(message.as_bytes()).as_ref()) + ) + } + fn header() -> Value { + json!({"typ":"rivet-app+jwt","alg":"EdDSA","kid":"token-key"}) + } + fn public_key(webhook: bool) -> Value { + public_key_from(&key(), webhook) + } + fn public_key_from(key: &Ed25519KeyPair, webhook: bool) -> Value { + let x = URL_SAFE_NO_PAD.encode(key.public_key().as_ref()); + let canonical = format!(r#"{{"crv":"Ed25519","kty":"OKP","x":"{x}"}}"#); + let kid = if webhook { + format!( + "ds_{}", + URL_SAFE_NO_PAD.encode(Sha256::digest(canonical.as_bytes())) + ) + } else { + "token-key".into() + }; + json!({"kty":"OKP","crv":"Ed25519","kid":kid,"alg":"EdDSA","use":"sig","x":x}) + } + + #[test] + fn strict_application_profile_and_lifecycle() { + let issuer = "https://engine.gameinc.io"; + let claims = claims(issuer, CapabilityKind::Callback); + let key = + DecodingKey::from_ed_components(public_key(false)["x"].as_str().unwrap()).unwrap(); + let check = |claims: &Value, header: &Value, namespace: &str, kind, operation| { + verify_with_key( + &token(claims, header), + &key, + issuer, + namespace, + kind, + operation, + super::super::now_ms(), + ) + }; + assert!( + check( + &claims, + &header(), + "namespace-a", + CapabilityKind::Callback, + "callback" + ) + .is_ok() + ); + for (field, value) in [ + ("aud", json!("rivet-api")), + ("iss", json!("https://untrusted.gameinc.io")), + ("rivet_ns", json!("namespace-b")), + ("rivet_ver", json!(2)), + ("rivet_purpose", json!("durable-streams.claim")), + ("jti", json!("")), + ("exp", claims["iat"].clone()), + ("iat", json!(claims["exp"].as_u64().unwrap() + 60)), + ("exp", json!(claims["iat"].as_u64().unwrap() + 86401)), + ] { + let mut invalid = claims.clone(); + invalid[field] = value; + assert!( + check( + &invalid, + &header(), + "namespace-a", + CapabilityKind::Callback, + "callback" + ) + .is_err(), + "{field}" + ); + } + for invalid_header in [ + json!({"typ":"rivet-auth+jwt","alg":"EdDSA","kid":"token-key"}), + json!({"typ":"rivet-app+jwt","alg":"HS256","kid":"token-key"}), + json!({"typ":"rivet-app+jwt","alg":"EdDSA","kid":"token-key","jku":"https://untrusted.gameinc.io"}), + ] { + assert!( + check( + &claims, + &invalid_header, + "namespace-a", + CapabilityKind::Callback, + "callback" + ) + .is_err() + ); + } + assert!( + check( + &claims, + &header(), + "namespace-b", + CapabilityKind::Callback, + "callback" + ) + .is_err() + ); + assert!( + check( + &claims, + &header(), + "namespace-a", + CapabilityKind::Claim, + "ack" + ) + .is_err() + ); + assert!( + check( + &claims, + &header(), + "namespace-a", + CapabilityKind::Callback, + "release" + ) + .is_err() + ); + let mut grants = claims.clone(); + grants["rivet_grants"] = json!([]); + assert!( + check( + &grants, + &header(), + "namespace-a", + CapabilityKind::Callback, + "callback" + ) + .is_err() + ); + let mut expired = claims.clone(); + expired["iat"] = json!(1); + expired["exp"] = json!(2); + assert_eq!( + check( + &expired, + &header(), + "namespace-a", + CapabilityKind::Callback, + "callback" + ) + .err() + .unwrap() + .code, + "TOKEN_EXPIRED" + ); + for operations in [ + json!(["callback", "ack"]), + json!(["callback", "callback"]), + json!([]), + ] { + let mut invalid = claims.clone(); + invalid["app"]["durable_streams"]["operations"] = operations; + assert!( + check( + &invalid, + &header(), + "namespace-a", + CapabilityKind::Callback, + "callback" + ) + .is_err() + ); + } + let claim = claims_for_claim(issuer); + assert!( + check( + &claim, + &header(), + "namespace-a", + CapabilityKind::Claim, + "ack" + ) + .is_ok() + ); + assert!( + check( + &claim, + &header(), + "namespace-a", + CapabilityKind::Claim, + "release" + ) + .is_ok() + ); + } + fn claims_for_claim(issuer: &str) -> Value { + claims(issuer, CapabilityKind::Claim) + } + + #[test] + fn exact_public_jwk_shape_and_thumbprint_are_enforced() { + let valid = json!({"keys":[public_key(true)]}); + validate_jwks(&serde_json::from_value(valid.clone()).unwrap(), true).unwrap(); + let mut bad = valid.clone(); + bad["keys"][0]["kid"] = json!("ds_wrong"); + assert!(validate_jwks(&serde_json::from_value(bad).unwrap(), true).is_err()); + let mut bad = valid.clone(); + bad["keys"][0]["d"] = json!("private"); + assert!(serde_json::from_value::(bad).is_err()); + let duplicate = json!({"keys":[public_key(true),public_key(true)]}); + assert!(validate_jwks(&serde_json::from_value(duplicate).unwrap(), true).is_err()); + assert!(trusted_url("https://secret@engine.gameinc.io").is_err()); + assert!(trusted_url("https://engine.gameinc.io?token=secret").is_err()); + assert!(namespace_path("../../auth").is_err()); + } + + struct TestState { + issuer: String, + jwt_fetches: AtomicUsize, + webhook_fetches: AtomicUsize, + sign_calls: AtomicUsize, + expiry_seconds: AtomicUsize, + jwt_max_age: AtomicUsize, + issue_status: AtomicUsize, + rotated_jwt: AtomicBool, + jwt_status: AtomicUsize, + sign_status: AtomicUsize, + webhook_age: AtomicUsize, + wrong_signing_namespace: AtomicBool, + rotated_webhook: AtomicBool, + provisioned: AtomicBool, + ensures: AtomicUsize, + response_delay_ms: AtomicUsize, + last_body: Mutex>, + } + struct Server { + api: CatalogApi, + state: Arc, + task: tokio::task::JoinHandle<()>, + } + impl Drop for Server { + fn drop(&mut self) { + self.task.abort(); + } + } + impl Server { + async fn start() -> Self { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let state = Arc::new(TestState { + issuer: endpoint.clone(), + jwt_fetches: AtomicUsize::new(0), + webhook_fetches: AtomicUsize::new(0), + sign_calls: AtomicUsize::new(0), + expiry_seconds: AtomicUsize::new(29), + jwt_max_age: AtomicUsize::new(60), + issue_status: AtomicUsize::new(200), + rotated_jwt: AtomicBool::new(false), + jwt_status: AtomicUsize::new(200), + sign_status: AtomicUsize::new(200), + webhook_age: AtomicUsize::new(0), + wrong_signing_namespace: AtomicBool::new(false), + rotated_webhook: AtomicBool::new(false), + provisioned: AtomicBool::new(true), + ensures: AtomicUsize::new(0), + response_delay_ms: AtomicUsize::new(0), + last_body: Mutex::new(Vec::new()), + }); + let app = Router::new().fallback(any(serve)).with_state(state.clone()); + let task = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + Self { + api: CatalogApi { + endpoint, + namespace: "test".into(), + token: Some("acl-secret".into()), + }, + state, + task, + } + } + fn client(&self) -> EngineClient { + EngineClient::new(&self.api, None).unwrap() + } + } + async fn serve(State(state): State>, request: Request) -> Response { + let path = request.uri().path().to_owned(); + tokio::time::sleep(Duration::from_millis( + state.response_delay_ms.load(Ordering::SeqCst) as u64, + )) + .await; + if path.starts_with("/auth/") || path == "/namespaces" { + assert_eq!( + request.headers().get("authorization").unwrap(), + "Bearer acl-secret" + ); + } + let body = axum::body::to_bytes(request.into_body(), MAX_BODY_LIMIT) + .await + .unwrap(); + let webhook_key = Ed25519KeyPair::from_seed_unchecked( + &[if state.rotated_webhook.load(Ordering::SeqCst) { + 8 + } else { + 7 + }; 32], + ) + .unwrap(); + let webhook_public = public_key_from(&webhook_key, true); + let metadata = || json!({"namespace_id":"namespace-a","alg":"ed25519","kid":webhook_public["kid"],"jwks_url":format!("{}/namespaces/namespace-a/auth/webhooks/jwks.json",state.issuer),"generation":1}); + let reply = match path.as_str() { + "/namespaces" => json!({"namespaces":[{"namespace_id":"namespace-a","name":"test"}]}), + "/auth/webhooks/ensure" => { + state.ensures.fetch_add(1, Ordering::SeqCst); + state.provisioned.store(true, Ordering::SeqCst); + let mut ensured = metadata(); + if state.wrong_signing_namespace.load(Ordering::SeqCst) { + ensured["namespace_id"] = json!("namespace-b"); + } + ensured + } + "/namespaces/namespace-a/auth/webhooks" => { + if !state.provisioned.load(Ordering::SeqCst) { + return http::StatusCode::NOT_FOUND.into_response(); + } + metadata() + } + "/.well-known/jwks.json" => { + state.jwt_fetches.fetch_add(1, Ordering::SeqCst); + let status = state.jwt_status.load(Ordering::SeqCst); + if status != 200 { + return ( + http::StatusCode::from_u16(status as u16).unwrap(), + "SECRET UPSTREAM ERROR", + ) + .into_response(); + } + let mut jwk = public_key(false); + if state.rotated_jwt.load(Ordering::SeqCst) { + jwk["kid"] = json!("rotated-key"); + } + return ( + [( + "cache-control", + format!( + "public, max-age={}", + state.jwt_max_age.load(Ordering::SeqCst) + ), + )], + axum::Json(json!({"keys":[jwk]})), + ) + .into_response(); + } + "/namespaces/namespace-a/auth/webhooks/jwks.json" => { + state.webhook_fetches.fetch_add(1, Ordering::SeqCst); + if !state.provisioned.load(Ordering::SeqCst) { + return http::StatusCode::BAD_REQUEST.into_response(); + } + return ( + [ + ("cache-control", "no-store".into()), + ("age", state.webhook_age.load(Ordering::SeqCst).to_string()), + ], + axum::Json(json!({"keys":[webhook_public]})), + ) + .into_response(); + } + "/auth/application-tokens" => { + let status = state.issue_status.load(Ordering::SeqCst); + if status != 200 { + return http::StatusCode::from_u16(status as u16) + .unwrap() + .into_response(); + } + let request: Value = serde_json::from_slice(&body).unwrap(); + assert!(request.get("namespace").is_none()); + assert_eq!(request["duration"], 86400); + let mut claims = claims(&state.issuer, CapabilityKind::Callback); + claims["aud"] = request["audience"].clone(); + claims["rivet_purpose"] = request["purpose"].clone(); + claims["app"] = request["claims"].clone(); + claims["exp"] = json!( + claims["iat"].as_u64().unwrap() + + state.expiry_seconds.load(Ordering::SeqCst) as u64 + ); + let mut jwt_header = header(); + if state.rotated_jwt.load(Ordering::SeqCst) { + jwt_header["kid"] = json!("rotated-key"); + } + json!({"token":token(&claims,&jwt_header),"issued_ts":claims["iat"].as_u64().unwrap()*1000,"expires_ts":claims["exp"].as_u64().unwrap()*1000}) + } + "/auth/webhooks/sign" => { + state.sign_calls.fetch_add(1, Ordering::SeqCst); + *state.last_body.lock().await = body.to_vec(); + let status = state.sign_status.load(Ordering::SeqCst); + if status != 200 { + return ( + http::StatusCode::from_u16(status as u16).unwrap(), + [("retry-after", "17")], + "SECRET UPSTREAM ERROR", + ) + .into_response(); + } + let timestamp = (super::super::now_ms() / 1000).to_string(); + let mut message = format!("{timestamp}.").into_bytes(); + message.extend_from_slice(&body); + let signer = if state.wrong_signing_namespace.load(Ordering::SeqCst) { + Ed25519KeyPair::from_seed_unchecked(&[9; 32]).unwrap() + } else { + webhook_key + }; + json!({"timestamp":timestamp,"kid":public_key_from(&signer,true)["kid"],"signature":URL_SAFE_NO_PAD.encode(signer.sign(&message).as_ref())}) + } + _ => return http::StatusCode::NOT_FOUND.into_response(), + }; + axum::Json(reply).into_response() + } + + #[tokio::test] + async fn exact_bytes_and_fresh_discovery() { + let server = Server::start().await; + let client = server.client(); + assert_eq!(client.namespace_id().await.unwrap(), "namespace-a"); + let metadata = client.ensure().await.unwrap(); + assert_eq!(metadata.kid, public_key(true)["kid"]); + assert_eq!( + client.metadata("namespace-a").await.unwrap().kid, + metadata.kid + ); + for body in [ + Bytes::new(), + Bytes::from_static(b"\x00\xff\r\n { \"a\": 1 }\n"), + ] { + let signed = client.sign(body.clone()).await.unwrap(); + assert_eq!(*server.state.last_body.lock().await, body); + let mut message = format!("{}.", signed.timestamp).into_bytes(); + message.extend_from_slice(&body); + UnparsedPublicKey::new(&ED25519, key().public_key()) + .verify( + &message, + &URL_SAFE_NO_PAD.decode(&signed.signature).unwrap(), + ) + .unwrap(); + assert_eq!( + signed.header(), + format!( + "t={},kid={},ed25519={}", + signed.timestamp, metadata.kid, signed.signature + ) + ); + } + for _ in 0..2 { + let jwks = client.webhook_jwks("namespace-a").await.unwrap(); + assert_eq!(jwks.value, json!({"keys":[public_key(true)]})); + assert!(jwks.age_secs >= 1); + } + assert_eq!(server.state.webhook_fetches.load(Ordering::SeqCst), 2); + server.state.webhook_age.store(300, Ordering::SeqCst); + assert_eq!( + client + .webhook_jwks("namespace-a") + .await + .err() + .unwrap() + .status, + 503 + ); + } + + #[tokio::test] + async fn short_expiry_and_bounded_unknown_key_refresh() { + let server = Server::start().await; + let client = server.client(); + let issued = client + .issue( + &capability(CapabilityKind::Claim), + CapabilityKind::Claim, + "namespace-a", + ) + .await + .unwrap(); + assert_eq!(issued.expires_ts - issued.issued_ts, 29_000); + let verified = client + .verify( + &issued.token, + "namespace-a", + CapabilityKind::Claim, + "release", + super::super::now_ms(), + ) + .await + .unwrap(); + assert_eq!(verified.expires_ts, issued.expires_ts); + assert_eq!(server.state.jwt_fetches.load(Ordering::SeqCst), 1); + // A malicious unknown-kid lookup must not hold up a cached, valid token. + { + let _refresh = client.inner.jwt_refresh.lock().await; + tokio::time::timeout( + Duration::from_millis(100), + client.verify( + &issued.token, + "namespace-a", + CapabilityKind::Claim, + "ack", + super::super::now_ms(), + ), + ) + .await + .unwrap() + .unwrap(); + } + let mut unknown_header = header(); + unknown_header["kid"] = json!("unknown-key"); + let unknown = token( + &claims(&server.api.endpoint, CapabilityKind::Callback), + &unknown_header, + ); + client.inner.jwt_keys.write().await.last_attempt = + Some(Instant::now() - Duration::from_secs(6)); + for _ in 0..10 { + assert_eq!( + client + .verify( + &unknown, + "namespace-a", + CapabilityKind::Callback, + "callback", + super::super::now_ms() + ) + .await + .err() + .unwrap() + .code, + "TOKEN_INVALID" + ); + } + assert_eq!(server.state.jwt_fetches.load(Ordering::SeqCst), 2); + { + let mut cache = client.inner.jwt_keys.write().await; + cache.expires = Some(Instant::now() - Duration::from_secs(1)); + cache.last_attempt = Some(Instant::now() - Duration::from_secs(6)); + } + server.state.jwt_status.store(503, Ordering::SeqCst); + let error = client + .verify( + &issued.token, + "namespace-a", + CapabilityKind::Claim, + "ack", + super::super::now_ms(), + ) + .await + .err() + .unwrap(); + assert_eq!(error.status, 503); + assert!(!format!("{error:?}").contains("SECRET")); + } + + #[tokio::test] + async fn one_second_key_cache_uses_precise_age_and_refreshes_after_expiry() { + let server = Server::start().await; + server.state.jwt_max_age.store(1, Ordering::SeqCst); + let client = server.client(); + let issued = client + .issue( + &capability(CapabilityKind::Claim), + CapabilityKind::Claim, + "namespace-a", + ) + .await + .unwrap(); + client + .verify( + &issued.token, + "namespace-a", + CapabilityKind::Claim, + "ack", + super::super::now_ms(), + ) + .await + .unwrap(); + assert_eq!(server.state.jwt_fetches.load(Ordering::SeqCst), 1); + let cache = client.inner.jwt_keys.read().await; + let remaining = cache + .expires + .unwrap() + .saturating_duration_since(Instant::now()); + assert!(!remaining.is_zero() && remaining < Duration::from_secs(1)); + drop(cache); + + client.inner.jwt_keys.write().await.expires = Some(Instant::now()); + client + .verify( + &issued.token, + "namespace-a", + CapabilityKind::Claim, + "ack", + super::super::now_ms(), + ) + .await + .unwrap(); + assert_eq!(server.state.jwt_fetches.load(Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn known_keys_refresh_without_reusing_zero_lifetime_snapshots() { + let server = Server::start().await; + server.state.jwt_max_age.store(0, Ordering::SeqCst); + let client = server.client(); + let issued = client + .issue( + &capability(CapabilityKind::Claim), + CapabilityKind::Claim, + "namespace-a", + ) + .await + .unwrap(); + for _ in 0..2 { + client + .verify( + &issued.token, + "namespace-a", + CapabilityKind::Claim, + "ack", + super::super::now_ms(), + ) + .await + .unwrap(); + } + assert_eq!(server.state.jwt_fetches.load(Ordering::SeqCst), 3); + + let mut unknown_header = header(); + unknown_header["kid"] = json!("unknown-key"); + let unknown = token( + &claims(&server.api.endpoint, CapabilityKind::Claim), + &unknown_header, + ); + for _ in 0..2 { + assert_eq!( + client + .verify( + &unknown, + "namespace-a", + CapabilityKind::Claim, + "ack", + super::super::now_ms() + ) + .await + .err() + .unwrap() + .status, + 401 + ); + } + assert_eq!(server.state.jwt_fetches.load(Ordering::SeqCst), 3); + + server.state.jwt_status.store(503, Ordering::SeqCst); + for _ in 0..2 { + assert_eq!( + client + .verify( + &issued.token, + "namespace-a", + CapabilityKind::Claim, + "ack", + super::super::now_ms() + ) + .await + .err() + .unwrap() + .status, + 503 + ); + } + assert_eq!(server.state.jwt_fetches.load(Ordering::SeqCst), 4); + } + + #[tokio::test] + async fn newly_issued_rotated_key_retries_without_hiding_credential_failures() { + let server = Server::start().await; + let client = server.client(); + let claims = capability(CapabilityKind::Callback); + client + .issue(&claims, CapabilityKind::Callback, "namespace-a") + .await + .unwrap(); + server.state.rotated_jwt.store(true, Ordering::SeqCst); + + let error = client + .issue(&claims, CapabilityKind::Callback, "namespace-a") + .await + .err() + .unwrap(); + assert_eq!(error.status, 503); + assert_eq!(error.code, "ENGINE_INVALID_RESPONSE"); + assert_eq!(server.state.jwt_fetches.load(Ordering::SeqCst), 1); + + client.inner.jwt_keys.write().await.last_attempt = + Some(Instant::now() - Duration::from_secs(6)); + let issued = client + .issue(&claims, CapabilityKind::Callback, "namespace-a") + .await + .unwrap(); + assert_eq!(strict_header(&issued.token).unwrap().kid, "rotated-key"); + assert_eq!(server.state.jwt_fetches.load(Ordering::SeqCst), 2); + + server.state.issue_status.store(401, Ordering::SeqCst); + let error = client + .issue(&claims, CapabilityKind::Callback, "namespace-a") + .await + .err() + .unwrap(); + assert_eq!(error.status, 401); + assert_eq!(error.code, "ENGINE_UNAUTHORIZED"); + } + + #[tokio::test] + async fn signing_errors_preserve_status_and_retry_without_leaking_body() { + let server = Server::start().await; + let client = server.client(); + for status in [400, 401, 403, 404, 409, 413, 429, 503] { + server.state.sign_status.store(status, Ordering::SeqCst); + let error = client + .sign(Bytes::from_static(b"secret payload")) + .await + .err() + .unwrap(); + assert_eq!(usize::from(error.status), status); + assert_eq!(error.retry_after, Some(17)); + assert!(!format!("{error:?}").contains("SECRET")); + assert!(!format!("{error:?}").contains("secret payload")); + } + let before = server.state.sign_calls.load(Ordering::SeqCst); + let error = client + .sign_with_limit(Bytes::from(vec![0; 1024 * 1024 + 1]), 1024 * 1024) + .await + .err() + .unwrap(); + assert_eq!(error.status, 413); + assert_eq!(server.state.sign_calls.load(Ordering::SeqCst), before); + let mut oversized = capability(CapabilityKind::Callback); + oversized.root = "x".repeat(3072); + assert_eq!( + client + .issue(&oversized, CapabilityKind::Callback, "namespace-a") + .await + .err() + .unwrap() + .status, + 413 + ); + } + + #[tokio::test] + async fn existing_metadata_and_cached_capabilities_cannot_skip_namespace_binding() { + let server = Server::start().await; + server + .state + .wrong_signing_namespace + .store(true, Ordering::SeqCst); + let client = server.client(); + assert!(client.metadata_read("namespace-a").await.is_ok()); + assert_eq!( + client.metadata("namespace-a").await.err().unwrap().status, + 503 + ); + assert_eq!( + client + .sign(Bytes::from_static(b"persisted callback capability")) + .await + .err() + .unwrap() + .status, + 503 + ); + assert_eq!(server.state.sign_calls.load(Ordering::SeqCst), 0); + + // Pull-only application operations do not need webhook permission. + let issued = client + .issue( + &capability(CapabilityKind::Claim), + CapabilityKind::Claim, + "namespace-a", + ) + .await + .unwrap(); + client + .verify( + &issued.token, + "namespace-a", + CapabilityKind::Claim, + "ack", + super::super::now_ms(), + ) + .await + .unwrap(); + } + + #[tokio::test] + async fn mutable_signing_scope_cannot_return_another_namespaces_artifact() { + let server = Server::start().await; + let client = server.client(); + client.metadata("namespace-a").await.unwrap(); + client.sign(Bytes::from_static(b"first")).await.unwrap(); + client.sign(Bytes::from_static(b"second")).await.unwrap(); + assert_eq!(server.state.ensures.load(Ordering::SeqCst), 1); + assert_eq!(server.state.webhook_fetches.load(Ordering::SeqCst), 0); + + server + .state + .wrong_signing_namespace + .store(true, Ordering::SeqCst); + assert_eq!( + client + .sign(Bytes::from_static(b"must not deliver")) + .await + .err() + .unwrap() + .status, + 503 + ); + assert_eq!(server.state.webhook_fetches.load(Ordering::SeqCst), 1); + assert_eq!(server.state.ensures.load(Ordering::SeqCst), 1); + + // A fresh namespace snapshot replaces prior bindings instead of growing + // a historical key cache, and stable current keys remain inexpensive. + assert_eq!(client.inner.webhook_kids.read().await.len(), 1); + server + .state + .wrong_signing_namespace + .store(false, Ordering::SeqCst); + client.sign(Bytes::from_static(b"corrected")).await.unwrap(); + assert_eq!(server.state.webhook_fetches.load(Ordering::SeqCst), 1); + + server.state.rotated_webhook.store(true, Ordering::SeqCst); + let rotated = client.sign(Bytes::from_static(b"rotated")).await.unwrap(); + assert_ne!(rotated.kid, public_key(true)["kid"]); + client + .sign(Bytes::from_static(b"same rotated key")) + .await + .unwrap(); + assert_eq!(server.state.webhook_fetches.load(Ordering::SeqCst), 2); + assert_eq!(server.state.ensures.load(Ordering::SeqCst), 1); + let known = client.inner.webhook_kids.read().await; + assert_eq!(known.len(), 1); + assert!(known.contains(&rotated.kid)); + } + + #[tokio::test] + async fn concurrent_metadata_only_ensures_once_and_timeouts_are_safe() { + let server = Server::start().await; + let client = server.client(); + server.state.provisioned.store(false, Ordering::SeqCst); + let calls = (0..12).map(|_| client.metadata("namespace-a")); + for result in futures_util::future::join_all(calls).await { + result.unwrap(); + } + assert_eq!(server.state.ensures.load(Ordering::SeqCst), 1); + let server = Server::start().await; + let client = server.client(); + server.state.provisioned.store(false, Ordering::SeqCst); + assert_eq!( + client.webhook_jwks("namespace-a").await.unwrap().value, + json!({"keys":[public_key(true)]}) + ); + assert_eq!(server.state.ensures.load(Ordering::SeqCst), 1); + assert_eq!(server.state.webhook_fetches.load(Ordering::SeqCst), 1); + server.state.response_delay_ms.store(100, Ordering::SeqCst); + let error = response( + client + .inner + .http + .get(client.url("/.well-known/jwks.json")) + .timeout(Duration::from_millis(5)), + ) + .await + .err() + .unwrap(); + assert_eq!(error.status, 503); + assert_eq!(error.code, "ENGINE_UNAVAILABLE"); + } + + #[tokio::test] + async fn large_4096_stream_body_remains_byte_exact() { + let server = Server::start().await; + let client = server.client(); + // A canonicalized 2048-byte Unicode path expands to over 6000 bytes. + let path = "%E2%82%AC".repeat(682); + let streams = (0..4096).map(|i|json!({"path":format!("{path}/{i}"),"acked_offset":"00000000_0000000000000000","tail_offset":"00000000_0000000000000001","has_pending":true})).collect::>(); + let body = Bytes::from( + serde_json::to_vec(&json!({"streams":streams,"callback":{"token":"x".repeat(6144)}})) + .unwrap(), + ); + assert!(body.len() > 25_000_000 && body.len() < 32 * 1024 * 1024); + let signature = client.sign(body.clone()).await.unwrap(); + assert_eq!(*server.state.last_body.lock().await, body); + let mut signed = format!("{}.", signature.timestamp).into_bytes(); + signed.extend_from_slice(&body); + UnparsedPublicKey::new(&ED25519, key().public_key()) + .verify( + &signed, + &URL_SAFE_NO_PAD.decode(signature.signature).unwrap(), + ) + .unwrap(); + } +} diff --git a/src/subscriptions/facade.rs b/src/subscriptions/facade.rs new file mode 100644 index 0000000..232d00c --- /dev/null +++ b/src/subscriptions/facade.rs @@ -0,0 +1,321 @@ +use std::sync::Arc; + +use axum::{ + body::Body, + http::{HeaderMap, Method, StatusCode}, + response::Response, +}; +use serde_json::{Value, json}; +use tokio::sync::OnceCell; + +use super::{Control, Reply, catalog_import::CatalogApi, model::ApiError}; + +/// Subscription management is closed by default. Embedders may explicitly use +/// their existing authenticated boundary instead of a separate bearer secret. +#[derive(Clone, Default)] +pub struct SubscriptionConfig { + pub management_token: Option, + pub allow_unauthenticated: bool, + pub allow_local_http: bool, + pub catalog_api: Option, +} +impl std::fmt::Debug for SubscriptionConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SubscriptionConfig") + .field( + "management_token_configured", + &self.management_token.is_some(), + ) + .field("allow_unauthenticated", &self.allow_unauthenticated) + .field("allow_local_http", &self.allow_local_http) + .field("catalog_api_configured", &self.catalog_api.is_some()) + .finish() + } +} + +#[allow(clippy::too_many_arguments)] +pub async fn route( + client: &rivetkit_client::Client, + config: &crate::facade::DurableStreamsConfig, + initialized: &Arc>, + original_path: &str, + routed_path: &str, + method: Method, + headers: &HeaderMap, + body: &[u8], +) -> Response { + let started = std::time::Instant::now(); + match dispatch( + client, + config, + initialized, + original_path, + routed_path, + method, + headers, + body, + ) + .await + { + Ok((mut reply, jwks)) => { + if jwks && let Some(age) = reply.age.as_mut() { + // Include the entire actor round trip conservatively, so waiting + // for legacy storage or serialization never resets cache age. + let elapsed = started.elapsed(); + *age = age + .saturating_add(elapsed.as_secs()) + .saturating_add(u64::from(elapsed.subsec_nanos() > 0)); + if *age >= 300 { + return response( + ApiError::new( + 503, + "SIGNING_DISCOVERY_STALE", + "Engine discovery exceeded its freshness budget", + ) + .into(), + ); + } + } + let mut r = response(reply); + if jwks && r.status().is_success() { + r.headers_mut().insert( + http::header::CONTENT_TYPE, + http::HeaderValue::from_static("application/jwk-set+json"), + ); + r.headers_mut().insert( + http::header::CACHE_CONTROL, + http::HeaderValue::from_static("public, max-age=300"), + ); + } + r + } + Err(e) => response(e.into()), + } +} +#[allow(clippy::too_many_arguments)] +async fn dispatch( + client: &rivetkit_client::Client, + config: &crate::facade::DurableStreamsConfig, + initialized: &Arc>, + original_path: &str, + routed_path: &str, + method: Method, + headers: &HeaderMap, + body: &[u8], +) -> Result<(Reply, bool), ApiError> { + let decoded = percent_encoding::percent_decode_str(routed_path) + .decode_utf8() + .map_err(|_| ApiError::invalid("Invalid path encoding"))?; + let path = decoded.trim_start_matches('/'); + let mut parts = path.split('/'); + if parts.next() != Some("__ds") { + return Err(ApiError::new(404, "NOT_FOUND", "Unknown control route")); + } + let kind = parts.next(); + let (id, operation, member) = if kind == Some("jwks.json") && parts.next().is_none() { + if method != Method::GET { + return Err(ApiError::new(405, "METHOD_NOT_ALLOWED", "Expected GET")); + } + (None, "jwks", None) + } else if kind == Some("subscriptions") { + let id = parts + .next() + .filter(|s| { + !s.is_empty() + && s.len() <= 256 + && *s != "." + && *s != ".." + && !s.chars().any(char::is_control) + }) + .ok_or_else(|| ApiError::invalid("Invalid subscription id"))?; + let remainder = parts.collect::>(); + let (op, member) = match remainder.as_slice() { + [] => match method { + Method::PUT => ("put", None), + Method::GET => ("get", None), + Method::DELETE => ("delete", None), + _ => { + return Err(ApiError::new( + 405, + "METHOD_NOT_ALLOWED", + "Expected PUT, GET or DELETE", + )); + } + }, + ["streams"] if method == Method::POST => ("add", None), + ["streams", rest @ ..] if !rest.is_empty() && method == Method::DELETE => { + ("remove", Some(rest.join("/"))) + } + [op @ ("claim" | "ack" | "release" | "callback")] if method == Method::POST => { + (*op, None) + } + ["streams" | "claim" | "ack" | "release" | "callback"] => { + return Err(ApiError::new( + 405, + "METHOD_NOT_ALLOWED", + "Unsupported control method", + )); + } + _ => return Err(ApiError::new(404, "NOT_FOUND", "Unknown control route")), + }; + (Some(id.to_owned()), op, member) + } else { + return Err(ApiError::new(404, "NOT_FOUND", "Unknown control route")); + }; + let bearer = headers + .get(http::header::AUTHORIZATION) + .and_then(|h| h.to_str().ok()) + .and_then(|s| s.strip_prefix("Bearer ")); + if !matches!(operation, "jwks" | "callback" | "ack" | "release") + && !config.subscriptions.allow_unauthenticated + { + let valid = config + .subscriptions + .management_token + .as_deref() + .zip(bearer) + .is_some_and(|(expected, got)| { + super::security::constant_time_equal(expected.as_bytes(), got.as_bytes()) + }); + if !valid { + return Err(ApiError::new( + 401, + "UNAUTHORIZED", + "Subscription management requires authorization", + )); + } + } + let body = if body.is_empty() { + json!({}) + } else { + serde_json::from_slice::(body).map_err(|_| ApiError::invalid("Invalid JSON body"))? + }; + // Derive the mount from Axum's original/routed paths, never from forwarded headers. + let root = original_path + .strip_suffix(routed_path) + .unwrap_or("") + .trim_end_matches('/') + .to_owned(); + let origin = config.public_origin.as_deref().ok_or_else(|| { + ApiError::new( + 503, + "NOT_CONFIGURED", + "PUBLIC_ORIGIN is required for subscription URLs", + ) + })?; + let parsed = url::Url::parse(origin) + .map_err(|_| ApiError::new(503, "NOT_CONFIGURED", "Invalid public origin"))?; + if !matches!(parsed.scheme(), "http" | "https") + || parsed.host_str().is_none() + || !parsed.username().is_empty() + || parsed.password().is_some() + || parsed.query().is_some() + || parsed.fragment().is_some() + || parsed.path() != "/" + { + return Err(ApiError::new( + 503, + "NOT_CONFIGURED", + "Public origin must contain only scheme and authority", + )); + } + // Only pattern backfill needs namespace enumeration. Explicit memberships + // refresh their addressed streams directly, including on later additions. + let import_catalog = operation == "put" + && body + .get("pattern") + .is_some_and(|pattern| !pattern.is_null()); + let request = Control { + root: root.clone(), + public_base: format!("{}{root}", origin.trim_end_matches('/')), + id, + operation: operation.to_owned(), + body, + token: bearer.map(str::to_owned), + path: member, + allow_local: config.subscriptions.allow_local_http, + deadline: None, + }; + let actor = super::handle(client, &config.tenant_scope).map_err(gateway)?; + if import_catalog + && initialized.get().is_none() + && let Some(api) = &config.subscriptions.catalog_api + { + // Validate/reconfirm atomically before migration. A conflicting PUT + // must return 409 even when an unrelated stream runner is unavailable. + let mut confirm = request.clone(); + confirm.operation = "confirm".to_owned(); + let reply = control(&actor, confirm, config.actor_timeout).await?; + if reply.status != 404 { + return Ok((reply, false)); + } + initialized + .get_or_try_init(|| async { + super::catalog_import::import(client, api, &config.tenant_scope).await + }) + .await + .map_err(|error| { + tracing::warn!(?error, "Subscription catalog import failed"); + ApiError::new( + 503, + "CATALOG_UNAVAILABLE", + "Existing-stream catalog is not ready", + ) + })?; + } + let reply = control(&actor, request, config.actor_timeout).await?; + Ok((reply, operation == "jwks")) +} + +async fn control( + actor: &rivetkit_client::handle::ActorHandle, + mut request: Control, + timeout: std::time::Duration, +) -> Result { + let budget_ms = timeout.as_millis().min(i64::MAX as u128) as i64; + request.deadline = Some( + super::now_ms() + .saturating_add(budget_ms) + .saturating_sub((budget_ms / 10).min(1_000)), + ); + let value = tokio::time::timeout( + timeout, + actor.action( + super::CONTROL_NAME, + vec![serde_json::to_value(request).map_err(|e| gateway(e.into()))?], + ), + ) + .await + .map_err(|_| ApiError::new(504, "TIMEOUT", "Subscription request timed out"))? + .map_err(gateway)?; + serde_json::from_value(value).map_err(|e| gateway(e.into())) +} +fn gateway(error: anyhow::Error) -> ApiError { + tracing::warn!(?error, "Subscription actor request failed"); + ApiError::new(502, "UNAVAILABLE", "Subscription actor unavailable") +} +fn response(reply: Reply) -> Response { + let status = StatusCode::from_u16(reply.status).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + let body = if status == StatusCode::NO_CONTENT { + Body::empty() + } else { + Body::from(serde_json::to_vec(&reply.body).unwrap()) + }; + let mut r = crate::facade::cors_response(status, body); + r.headers_mut().insert( + http::header::CONTENT_TYPE, + http::HeaderValue::from_static("application/json"), + ); + r.headers_mut().insert( + http::header::CACHE_CONTROL, + http::HeaderValue::from_static("no-store"), + ); + if let Some(seconds) = reply.retry_after { + r.headers_mut() + .insert(http::header::RETRY_AFTER, seconds.into()); + } + if let Some(age) = reply.age { + r.headers_mut().insert(http::header::AGE, age.into()); + } + r +} diff --git a/src/subscriptions/fault.rs b/src/subscriptions/fault.rs new file mode 100644 index 0000000..7bf6291 --- /dev/null +++ b/src/subscriptions/fault.rs @@ -0,0 +1,29 @@ +//! Explicit opt-in crash points used by the real-Engine recovery tests. +#[inline] +pub(crate) fn checkpoint(_name: &str) { + #[cfg(feature = "subscription-fault-injection")] + if std::env::var("DS_SUBSCRIPTION_FAULT").as_deref() == Ok(_name) { + if let Ok(marker) = std::env::var("DS_SUBSCRIPTION_FAULT_MARKER") { + std::fs::write(marker, _name).expect("write recovery test checkpoint"); + } + std::process::abort(); + } +} + +/// A bounded barrier for races while an Engine request is outside the actor lock. +#[inline] +pub(crate) async fn pause(_name: &str) { + #[cfg(feature = "subscription-fault-injection")] + if std::env::var("DS_SUBSCRIPTION_PAUSE_POINT").as_deref() == Ok(_name) + && let Ok(marker) = std::env::var("DS_SUBSCRIPTION_PAUSE_MARKER") + { + std::fs::write(&marker, _name).expect("write recovery test barrier"); + let release = format!("{marker}.release"); + let _ = tokio::time::timeout(std::time::Duration::from_secs(10), async { + while !std::path::Path::new(&release).exists() { + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await; + } +} diff --git a/src/subscriptions/legacy.rs b/src/subscriptions/legacy.rs new file mode 100644 index 0000000..61cc217 --- /dev/null +++ b/src/subscriptions/legacy.rs @@ -0,0 +1,277 @@ +//! Verification-only compatibility for artifacts issued before Engine cutover. +use anyhow::{Result, anyhow, ensure}; +use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; +use ring::{ + hmac, + signature::{Ed25519KeyPair, KeyPair}, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; + +use super::model::{ApiError, ApiResult, Subscription}; + +pub const TOKEN_RETENTION_MS: i64 = 86_400_000; +const PUBLIC_RETENTION_MS: i64 = 365_000; + +#[derive(Clone, Serialize, Deserialize)] +pub struct LegacyKeys { + pub cutover: i64, + pub token_expires: i64, + pub public_expires: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + token_key: Option>, + public_keys: Vec, +} + +impl LegacyKeys { + pub fn migrate(value: &str, cutover: i64, latest_expiry: i64) -> Result { + #[derive(Deserialize)] + struct OldKeys { + private_key: Vec, + token_key: Vec, + old_public_keys: Vec, + } + let old: OldKeys = serde_json::from_str(value)?; + ensure!(old.token_key.len() == 32, "Invalid legacy capability key"); + let signer = Ed25519KeyPair::from_pkcs8(&old.private_key) + .map_err(|_| anyhow!("Invalid legacy webhook key"))?; + let active = public_jwk(signer.public_key().as_ref()); + let mut public_keys = Vec::with_capacity(old.old_public_keys.len() + 1); + for key in old.old_public_keys { + ensure!( + key["kty"] == "OKP" && key["crv"] == "Ed25519", + "Invalid legacy public key" + ); + let x = URL_SAFE_NO_PAD.decode( + key["x"] + .as_str() + .ok_or_else(|| anyhow!("Missing legacy public key"))?, + )?; + ensure!(x.len() == 32, "Invalid legacy public key size"); + let public = public_jwk(&x); + ensure!(key["kid"] == public["kid"], "Invalid legacy public key id"); + public_keys.push(public); + } + public_keys.push(active); + let token_expires = + latest_expiry.clamp(cutover, cutover.saturating_add(TOKEN_RETENTION_MS)); + Ok(Self { + cutover, + token_expires, + public_expires: cutover.saturating_add(PUBLIC_RETENTION_MS), + token_key: (token_expires > cutover).then_some(old.token_key), + public_keys, + }) + } + + pub fn public_jwks(&self, now: i64) -> Vec { + if now < self.public_expires { + self.public_keys.clone() + } else { + Vec::new() + } + } + + pub fn next_deadline(&self) -> Option { + self.token_key + .as_ref() + .map(|_| self.token_expires) + .into_iter() + .chain((!self.public_keys.is_empty()).then_some(self.public_expires)) + .min() + } + + pub fn cleanup(&mut self, now: i64) { + if now >= self.token_expires { + self.token_key = None; + } + if now >= self.public_expires { + self.public_keys.clear(); + } + } + + pub fn verify(&self, token: &str, sub: &Subscription, now: i64) -> ApiResult<()> { + let invalid = || ApiError::new(401, "TOKEN_INVALID", "Invalid capability token"); + if token.len() > 6144 || now >= self.token_expires { + return Err(invalid()); + } + let key = self.token_key.as_ref().ok_or_else(invalid)?; + let (input, tag) = token.rsplit_once('.').ok_or_else(invalid)?; + let tag = URL_SAFE_NO_PAD.decode(tag).map_err(|_| invalid())?; + hmac::verify( + &hmac::Key::new(hmac::HMAC_SHA256, key), + input.as_bytes(), + &tag, + ) + .map_err(|_| invalid())?; + let (header, payload) = input.split_once('.').ok_or_else(invalid)?; + if header != URL_SAFE_NO_PAD.encode(br#"{"alg":"HS256","typ":"JWT"}"#) { + return Err(invalid()); + } + let claims: Claims = + serde_json::from_slice(&URL_SAFE_NO_PAD.decode(payload).map_err(|_| invalid())?) + .map_err(|_| invalid())?; + if claims.sub != sub.id || claims.root != sub.root { + return Err(invalid()); + } + let wake = sub.wake.as_ref().ok_or_else(ApiError::fenced)?; + if sub.deleted + || wake.issued_at >= self.cutover + || claims.generation != wake.generation + || claims.wake_id != wake.id + { + return Err(ApiError::fenced()); + } + if claims.exp <= now / 1000 { + return Err(ApiError::new( + 401, + "TOKEN_EXPIRED", + "Capability token expired", + )); + } + if claims.exp > self.token_expires / 1000 + || claims.exp > wake.issued_at.saturating_add(TOKEN_RETENTION_MS) / 1000 + || claims.kind + != if sub.config.webhook.is_some() { + "callback" + } else { + "claim" + } + { + return Err(invalid()); + } + Ok(()) + } +} + +fn public_jwk(bytes: &[u8]) -> Value { + let x = URL_SAFE_NO_PAD.encode(bytes); + let canonical = format!(r#"{{"crv":"Ed25519","kty":"OKP","x":"{x}"}}"#); + let kid = format!( + "ds_{}", + URL_SAFE_NO_PAD.encode(Sha256::digest(canonical.as_bytes())) + ); + json!({"kty":"OKP","crv":"Ed25519","kid":kid,"use":"sig","alg":"EdDSA","x":x}) +} + +#[derive(Serialize, Deserialize)] +struct Claims { + sub: String, + root: String, + generation: i64, + wake_id: String, + exp: i64, + kind: String, +} + +#[cfg(test)] +mod tests { + use super::super::model::{CatalogEntry, Config}; + use super::*; + + fn old_keys() -> Value { + json!({"private_key":Ed25519KeyPair::generate_pkcs8(&ring::rand::SystemRandom::new()).unwrap().as_ref(),"token_key":vec![7;32],"old_public_keys":[]}) + } + + fn fixture() -> (LegacyKeys, Subscription, String) { + let old = old_keys(); + let config = serde_json::from_value::( + json!({"type":"pull-wake","pattern":"**","wake_stream":"wake"}), + ) + .unwrap() + .normalize() + .unwrap(); + let mut sub = Subscription::new( + "sub".into(), + "/root".into(), + "https://streams.gameinc.io/root".into(), + config, + 0, + 1000, + ); + sub.observe(&CatalogEntry { + path: "/root/source".into(), + version: 1, + incarnation: "first".into(), + tail: "0000000000000000_0000000000000001".into(), + start: crate::protocol::ZERO_OFFSET.into(), + exists: true, + wakes: vec![], + }); + sub.ensure_wake(1000); + let wake = sub.wake.as_ref().unwrap(); + let claims = Claims { + sub: sub.id.clone(), + root: sub.root.clone(), + generation: wake.generation, + wake_id: wake.id.clone(), + exp: wake.token_expires / 1000, + kind: "claim".into(), + }; + let input = format!( + "{}.{}", + URL_SAFE_NO_PAD.encode(br#"{"alg":"HS256","typ":"JWT"}"#), + URL_SAFE_NO_PAD.encode(serde_json::to_vec(&claims).unwrap()) + ); + let tag = hmac::sign( + &hmac::Key::new(hmac::HMAC_SHA256, &[7; 32]), + input.as_bytes(), + ); + let token = format!("{input}.{}", URL_SAFE_NO_PAD.encode(tag.as_ref())); + ( + LegacyKeys::migrate(&old.to_string(), 2000, wake.token_expires).unwrap(), + sub, + token, + ) + } + + #[test] + fn cutover_preserves_old_capabilities_without_retaining_private_webhook_keys() { + let (keys, mut sub, token) = fixture(); + assert!(keys.verify(&token, &sub, 3000).is_ok()); + let serialized = serde_json::to_string(&keys).unwrap(); + assert!(!serialized.contains("private_key")); + let restored: LegacyKeys = serde_json::from_str(&serialized).unwrap(); + assert_eq!(restored.token_expires, keys.token_expires); + assert_eq!(restored.public_expires, keys.public_expires); + assert!(restored.verify(&token, &sub, 3000).is_ok()); + sub.root = "/other".into(); + assert!(keys.verify(&token, &sub, 3000).is_err()); + sub.root = "/root".into(); + sub.wake.as_mut().unwrap().generation += 1; + assert_eq!(keys.verify(&token, &sub, 3000).unwrap_err().status, 409); + sub.wake.as_mut().unwrap().generation -= 1; + sub.wake.as_mut().unwrap().issued_at = keys.cutover; + assert_eq!(keys.verify(&token, &sub, 3000).unwrap_err().status, 409); + } + + #[test] + fn migration_bounds_capability_retention_and_drops_unused_hmac_keys() { + let old = old_keys().to_string(); + let capped = LegacyKeys::migrate(&old, 2000, i64::MAX).unwrap(); + assert_eq!(capped.token_expires, 2000 + TOKEN_RETENTION_MS); + let unused = LegacyKeys::migrate(&old, 2000, 1000).unwrap(); + assert_eq!(unused.token_expires, 2000); + assert!( + !serde_json::to_string(&unused) + .unwrap() + .contains("token_key") + ); + assert_eq!(unused.next_deadline(), Some(2000 + PUBLIC_RETENTION_MS)); + } + + #[test] + fn legacy_material_is_removed_at_fixed_deadlines() { + let (mut keys, sub, token) = fixture(); + assert_eq!(keys.next_deadline(), Some(keys.public_expires)); + assert_eq!(keys.public_jwks(keys.public_expires - 1).len(), 1); + assert!(keys.public_jwks(keys.public_expires).is_empty()); + keys.cleanup(keys.public_expires); + assert_eq!(keys.next_deadline(), Some(keys.token_expires)); + assert!(keys.verify(&token, &sub, keys.token_expires).is_err()); + keys.cleanup(keys.token_expires); + assert_eq!(keys.next_deadline(), None); + assert!(!serde_json::to_string(&keys).unwrap().contains("token_key")); + } +} diff --git a/src/subscriptions/mod.rs b/src/subscriptions/mod.rs new file mode 100644 index 0000000..94061ba --- /dev/null +++ b/src/subscriptions/mod.rs @@ -0,0 +1,1141 @@ +//! Durable subscription coordinator. One actor owns a tenant's catalog and cursors; +//! stream actors publish a transactional outbox into this owner. +pub(crate) mod catalog_import; +mod db; +mod engine; +pub(crate) mod facade; +pub(crate) mod fault; +mod legacy; +pub(crate) mod model; +mod security; +mod signing; +use signing::*; + +use anyhow::{Result, anyhow}; +use async_trait::async_trait; +use engine::{CapabilityClaims, CapabilityKind, EngineClient, EngineError, VerifiedCapability}; +use futures_util::{StreamExt, stream}; +use rivetkit::{Action, Handles, action, prelude::*}; +use rivetkit_client::{Client, GetOptions, GetOrCreateOptions, handle::ActorHandle}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::{ + future::Future, + pin::Pin, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; +use tokio::sync::{Mutex, OnceCell}; + +use db::text; +use model::*; + +pub const ACTOR_NAME: &str = "durableStreamSubscriptions"; +const TICK_NAME: &str = "__durableStreamSubscriptionTick"; +const NOTIFY_NAME: &str = "__durableStreamCatalogChanged"; +const CONTROL_NAME: &str = "__durableStreamSubscriptionControl"; +const DISPATCH_TIMEOUT: i64 = 60_000; +type BoxFuture = Pin> + Send>>; + +#[derive(Clone, Default, Serialize, Deserialize)] +pub struct Input { + pub tenant_scope: String, +} + +pub struct SubscriptionActor { + client: Client, + engine: EngineClient, + namespace: OnceCell, + lock: Mutex<()>, + pumping: AtomicBool, + alarm: Mutex>, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct CatalogChanged(pub CatalogEntry); +impl Action for CatalogChanged { + type Output = (); + const NAME: &'static str = NOTIFY_NAME; +} +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct CatalogReady { + pub complete: bool, +} +impl Action for CatalogReady { + type Output = bool; + const NAME: &'static str = "__durableStreamCatalogReady"; +} +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Tick; +impl Action for Tick { + type Output = (); + const NAME: &'static str = TICK_NAME; +} +#[derive(Clone, Serialize, Deserialize)] +pub struct Control { + root: String, + public_base: String, + id: Option, + operation: String, + body: Value, + token: Option, + path: Option, + allow_local: bool, + #[serde(default)] + deadline: Option, +} +#[derive(Clone, Serialize, Deserialize)] +pub struct Reply { + pub status: u16, + pub body: Value, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry_after: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub age: Option, +} +impl Reply { + fn ok(status: u16, body: Value) -> Self { + Self { + status, + body, + retry_after: None, + age: None, + } + } +} +impl From for Reply { + fn from(e: ApiError) -> Self { + Self { + status: e.status, + body: e.body, + retry_after: e.retry_after, + age: None, + } + } +} +impl Action for Control { + type Output = Reply; + const NAME: &'static str = CONTROL_NAME; +} + +pub fn handle(client: &Client, tenant: &str) -> Result { + client.get_or_create( + ACTOR_NAME, + vec![crate::facade::stream_actor_key( + tenant, + "__ds/subscriptions", + )], + GetOrCreateOptions { + create_with_input: Some(json!({"tenant_scope":tenant})), + ..Default::default() + }, + ) +} + +#[async_trait] +impl Actor for SubscriptionActor { + type State = Input; + type Input = Input; + type Actions = (CatalogChanged, Tick, Control, CatalogReady); + type Events = (); + type Queue = (); + type ConnParams = (); + type ConnState = (); + type Action = action::Raw; + const HAS_DATABASE: bool = true; + async fn create_state(_: &Ctx, input: Input) -> Result { + Ok(input) + } + async fn create(ctx: &Ctx) -> Result { + db::ensure(ctx.sql()).await?; + ctx.sql().exec("CREATE TABLE IF NOT EXISTS subscription_engine_identity (id INTEGER PRIMARY KEY CHECK(id=1), namespace TEXT NOT NULL)").await?; + let runtime = ctx.inner(); + let connection = catalog_import::CatalogApi { + endpoint: runtime + .client_endpoint() + .ok_or_else(|| anyhow!("Missing Engine endpoint"))? + .to_owned(), + namespace: runtime + .client_namespace() + .ok_or_else(|| anyhow!("Missing Engine namespace"))? + .to_owned(), + token: runtime.client_token().map(str::to_owned), + }; + let issuer = std::env::var("DS_ENGINE_ISSUER").ok(); + Ok(Self { + client: crate::client::actor_client(ctx)?, + engine: EngineClient::new(&connection, issuer.as_deref())?, + namespace: OnceCell::new(), + lock: Mutex::new(()), + pumping: AtomicBool::new(false), + alarm: Mutex::new(None), + }) + } + async fn on_start(self: Arc, ctx: Ctx) -> Result<()> { + self.kick(&ctx); + Ok(()) + } +} +impl Handles for SubscriptionActor { + type Future = BoxFuture<()>; + fn handle(self: Arc, ctx: Ctx, action: CatalogChanged) -> Self::Future { + Box::pin(async move { + self.observe(&ctx, action.0).await?; + self.kick(&ctx); + Ok(()) + }) + } +} +impl Handles for SubscriptionActor { + type Future = BoxFuture; + fn handle(self: Arc, ctx: Ctx, action: CatalogReady) -> Self::Future { + Box::pin(async move { + let _guard = self.lock.lock().await; + if action.complete { + ctx.sql() + .execute( + "UPDATE subscription_catalog_status SET ready=1 WHERE id=1", + None, + ) + .await?; + } + let rows = ctx + .sql() + .query( + "SELECT ready FROM subscription_catalog_status WHERE id=1", + None, + ) + .await?; + Ok(matches!( + rows.rows.first().and_then(|r| r.first()), + Some(rivetkit::ColumnValue::Integer(1)) + )) + }) + } +} +impl Handles for SubscriptionActor { + type Future = BoxFuture<()>; + fn handle(self: Arc, ctx: Ctx, _: Tick) -> Self::Future { + Box::pin(async move { + { + let mut alarm = self.alarm.lock().await; + if alarm.as_ref().is_some_and(|(time, _)| *time <= now_ms()) { + *alarm = None; + } + } + if let Err(error) = self.fanout_tick(&ctx).await { + self.arm(&ctx, now_ms() + 1_000).await?; + return Err(error); + } + self.kick(&ctx); + Ok(()) + }) + } +} +impl Handles for SubscriptionActor { + type Future = BoxFuture; + fn handle(self: Arc, ctx: Ctx, request: Control) -> Self::Future { + Box::pin(async move { + let reply = self.control(&ctx, request).await?; + self.kick(&ctx); + Ok(reply) + }) + } +} + +impl SubscriptionActor { + async fn arm(&self, ctx: &Ctx, when: i64) -> Result<()> { + let mut alarm = self.alarm.lock().await; + if alarm.as_ref().is_some_and(|(at, _)| *at <= when) { + return Ok(()); + } + let id = ctx.schedule().at(when, TICK_NAME, &[]).await?; + if let Some((_, old)) = alarm.replace((when, id)) { + ctx.schedule().cancel(&old).await?; + } + Ok(()) + } + fn kick(self: &Arc, ctx: &Ctx) { + let this = self.clone(); + let work_ctx = ctx.clone(); + ctx.wait_until(async move { + if let Err(error) = this.clone().pump(work_ctx.clone()).await { + tracing::warn!(?error, "Subscription delivery deferred to durable schedule"); + if let Err(error) = this.arm(&work_ctx, now_ms() + 1_000).await { + tracing::error!(?error, "Subscription recovery schedule failed"); + } + } + }); + } + async fn observe(&self, ctx: &Ctx, entry: CatalogEntry) -> Result<()> { + let _guard = self.lock.lock().await; + self.arm(ctx, now_ms() + 1_000).await?; + let tx = db::begin(ctx.sql()).await?; + db::save_catalog(&tx, &entry).await?; + // Stream responses wait for this durable catalog update. Fanout runs + // separately in bounded pump transactions. + tx.commit().await + } + + async fn fanout_batch( + &self, + tx: &rivetkit::SqliteTransaction, + path: Option<&str>, + ) -> Result<()> { + const BATCH: usize = 64; + let Some(job) = db::fanout(tx, path).await? else { + return Ok(()); + }; + let entry = &job.entry; + // Bind prefix lengths rather than every prefix: a deep path otherwise + // exceeds SQLite's compound-select limit and duplicates quadratic bytes. + // Each root still uses the (root,key) index and reads at most one batch. + let roots = entry + .path + .chars() + .enumerate() + .filter_map(|(i, c)| (job.discover && entry.exists && c == '/').then_some(i)) + .collect::>(); + let sql = format!( + "WITH roots AS (SELECT substr(?1,1,value) AS root FROM json_each(?3)), + candidates AS ( + SELECT key FROM (SELECT sub_key AS key FROM subscription_links WHERE path=?1 AND sub_key>?2 ORDER BY sub_key LIMIT {BATCH}) + UNION + SELECT candidate.value FROM roots JOIN json_each(( + SELECT json_group_array(key) FROM ( + SELECT key FROM subscriptions WHERE root=roots.root AND key>?2 AND pattern IS NOT NULL ORDER BY key LIMIT {BATCH} + ) + )) AS candidate + ) + SELECT s.key,s.root,s.pattern,EXISTS(SELECT 1 FROM subscription_links l WHERE l.path=?1 AND l.sub_key=s.key),octet_length(s.value) + FROM subscriptions s JOIN candidates c ON c.key=s.key ORDER BY s.key LIMIT {BATCH}" + ); + let rows = tx + .execute( + sql, + Some(vec![ + text(&entry.path), + text(&job.cursor), + text(serde_json::to_string(&roots)?), + ]), + ) + .await?; + let mut cursor = job.cursor.clone(); + let mut state_bytes = 0_i64; + let mut limited = false; + for row in &rows.rows { + let key = db::string(row, 0)?; + let root = db::string(row, 1)?; + let linked = matches!(row.get(3), Some(rivetkit::ColumnValue::Integer(1))); + let relative = entry + .path + .strip_prefix(root) + .and_then(|p| p.strip_prefix('/')); + if !linked && !relative.is_some_and(|path|matches!(row.get(2),Some(rivetkit::ColumnValue::Text(pattern)) if glob_matches(pattern,path))) { + cursor = key.to_owned(); + continue; + } + let Some(rivetkit::ColumnValue::Integer(bytes)) = row.get(4) else { + return Err(anyhow!("Missing subscription state size")); + }; + // Admit at least one state, then cap aggregate rewrites. Leave the + // cursor before the deferred row so the next transaction resumes it. + if state_bytes > 0 && state_bytes.saturating_add(*bytes) > 8 * 1024 * 1024 { + limited = true; + break; + } + cursor = key.to_owned(); + // Reads and JSON parsing also consume the transaction budget, even + // when reconciliation has already applied this catalog version. + state_bytes = state_bytes.saturating_add(*bytes); + let Some(mut sub) = db::load(tx, key).await? else { + continue; + }; + if sub.deleted + || relative + .and_then(|p| sub.links.get(p)) + .is_some_and(|l| l.version >= entry.version) + { + continue; + } + sub.observe(entry); + sub.expire(now_ms()); + sub.ensure_wake(now_ms()); + if relative.is_some_and(|p| sub.links.contains_key(p)) { + tx.execute( + "INSERT OR IGNORE INTO subscription_links(path,sub_key) VALUES(?,?)", + Some(vec![text(&entry.path), text(key)]), + ) + .await?; + } + db::save(tx, &sub).await?; + } + db::advance_fanout(tx, &job, &cursor, !limited && rows.rows.len() < BATCH).await + } + + async fn fanout_tick(&self, ctx: &Ctx) -> Result<()> { + let _guard = self.lock.lock().await; + if ctx + .sql() + .query("SELECT 1 FROM subscription_fanout LIMIT 1", None) + .await? + .rows + .is_empty() + { + return Ok(()); + } + // Fanout progresses while the delivery pump awaits slow destinations. + self.arm(ctx, now_ms() + 1_000).await?; + let tx = db::begin(ctx.sql()).await?; + self.fanout_batch(&tx, None).await?; + let more = !tx + .execute("SELECT 1 FROM subscription_fanout LIMIT 1", None) + .await? + .rows + .is_empty(); + tx.commit().await?; + if more { + self.arm(ctx, now_ms() + 10).await?; + } + Ok(()) + } + + async fn control(&self, ctx: &Ctx, request: Control) -> Result { + if request.operation == "jwks" { + return Ok(match self.public_keys(ctx).await { + Ok(r) => r, + Err(e) => e.into(), + }); + } + let id = request + .id + .as_deref() + .ok_or_else(|| anyhow!("Missing routed subscription ID"))?; + let config = if matches!(request.operation.as_str(), "put" | "confirm") { + let config = match serde_json::from_value::(request.body.clone()) { + Ok(c) => match c.normalize() { + Ok(c) => c, + Err(e) => return Ok(e.into()), + }, + Err(e) => return Ok(ApiError::invalid(&e.to_string()).into()), + }; + Some(config) + } else { + None + }; + if let Some(config) = &config { + let existing = { + let _guard = self.lock.lock().await; + self.arm(ctx, now_ms() + 1_000).await?; + let tx = db::begin(ctx.sql()).await?; + let mut existing = db::load(&tx, &db::key(&request.root, id)).await?; + if let Some(sub) = existing.as_mut().filter(|s| !s.deleted) { + let mut changed = sub.expire(now_ms()); + if sub.config_hash == config.hash() { + changed |= sub.resume_delivery(now_ms()); + } + if changed { + db::save(&tx, sub).await?; + } + tx.commit().await?; + } else { + tx.rollback().await?; + } + existing + }; + if let Some(sub) = existing.filter(|s| !s.deleted) { + return Ok(if sub.config_hash == config.hash() { + Reply::ok( + 200, + sub.public( + match self + .signing_metadata( + ctx, + &sub.public_base, + sub.config.webhook.is_some(), + ) + .await + { + Ok(v) => v, + Err(e) => return Ok(e.into()), + }, + ), + ) + } else { + ApiError::new( + 409, + "SUBSCRIPTION_ALREADY_EXISTS", + "Different subscription configuration", + ) + .into() + }); + } + if request.operation == "confirm" { + return Ok( + ApiError::new(404, "SUBSCRIPTION_NOT_FOUND", "Subscription not found").into(), + ); + } + if let Some(webhook) = &config.webhook + && let Err(error) = security::destination(&webhook.url, request.allow_local).await + { + return Ok(ApiError::new(400, "WEBHOOK_URL_REJECTED", &error.to_string()).into()); + } + } + if request.operation == "add" + && let Err(error) = added_streams(&request.body) + { + return Ok(error.into()); + } + // All external work happens before acquiring the mutation lock. + let webhook = if let Some(config) = &config { + config.webhook.is_some() + } else if request.operation == "get" { + let _guard = self.lock.lock().await; + let tx = db::begin(ctx.sql()).await?; + let webhook = db::load(&tx, &db::key(&request.root, id)) + .await? + .is_some_and(|s| !s.deleted && s.config.webhook.is_some()); + tx.rollback().await?; + webhook + } else { + false + }; + let signing = match self + .signing_metadata(ctx, &request.public_base, webhook) + .await + { + Ok(value) => value, + Err(e) => return Ok(e.into()), + }; + let authorization = match self.prepare_authorization(ctx, &request).await { + Ok(value) => value, + Err(e) => return Ok(e.into()), + }; + if matches!(request.operation.as_str(), "put" | "add") { + self.refresh_membership_catalog(ctx, &request, config.as_ref()) + .await?; + } + let _guard = self.lock.lock().await; + // Arm before the mutation; a surplus schedule is safe, missing recovery isn't. + self.arm(ctx, now_ms() + 1_000).await?; + let tx = db::begin(ctx.sql()).await?; + let key = db::key(&request.root, id); + let existing = db::load(&tx, &key).await?; + if let Some(config) = config { + if let Some(sub) = existing.as_ref().filter(|s| !s.deleted) { + let reply = if sub.config_hash == config.hash() { + Reply::ok(200, sub.public(signing.clone())) + } else { + ApiError::new( + 409, + "SUBSCRIPTION_ALREADY_EXISTS", + "Different subscription configuration", + ) + .into() + }; + tx.rollback().await?; + return Ok(reply); + } + let mut sub = Subscription::new( + id.to_owned(), + request.root.clone(), + request.public_base.clone(), + config, + existing.map_or(0, |s| s.generation + 1), + now_ms(), + ); + let paths = sub.config.streams.clone(); + self.add_links(&tx, &mut sub, &paths) + .await + .map_err(|e| anyhow!("add subscription links: {}", e.body))?; + if sub.config.pattern.is_some() { + let mut after = String::new(); + loop { + let rows=tx.execute("SELECT path,value FROM subscription_catalog WHERE path>? AND path>=? AND path r, + Err(e) => e.into(), + }); + } + let result = self + .apply_control(&mut sub, &request, authorization.as_ref(), signing, &tx) + .await; + let renew_lease = result.is_ok() + && matches!(request.operation.as_str(), "callback" | "ack") + && request.body.get("done").and_then(Value::as_bool) != Some(true); + let reply = match result { + Ok(r) => { + changed |= request.operation != "get"; + r + } + Err(e) => e.into(), + }; + // apply_control validates the entire batch before mutation. Expiry itself + // persists even when the triggering request is fenced. + if changed { + if renew_lease { + let persisted = db::save_with_lease(&tx, &mut sub) + .await + .map_err(internal) + .and_then(|_| signing::require_live_lease(&sub)); + if let Err(error) = persisted { + tx.rollback().await?; + return Ok(error.into()); + } + } else { + db::save(&tx, &sub).await?; + } + } + if matches!(request.operation.as_str(), "delete" | "add" | "remove") { + db::sync_links(&tx, &sub).await?; + } + tx.commit().await?; + Ok(reply) + } + + async fn reconcile_subscription( + &self, + tx: &rivetkit::SqliteTransaction, + sub: &mut Subscription, + ) -> Result { + let mut after = String::new(); + let key = db::key(&sub.root, &sub.id); + let mut changed = false; + loop { + let rows=tx.execute("SELECT c.path,c.value FROM subscription_links l JOIN subscription_catalog c ON c.path=l.path WHERE l.sub_key=? AND l.path>? ORDER BY l.path LIMIT 128",Some(vec![text(&key),text(&after)])).await?; + if rows.rows.is_empty() { + break; + } + for row in &rows.rows { + after = db::string(row, 0)?.to_owned(); + let entry: CatalogEntry = serde_json::from_str(db::string(row, 1)?)?; + changed |= sub.observe(&entry); + } + } + Ok(changed) + } + + async fn refresh_membership_catalog( + &self, + ctx: &Ctx, + request: &Control, + config: Option<&Config>, + ) -> Result<()> { + let mut paths = std::collections::BTreeSet::new(); + if let Some(config) = config { + paths.extend( + config + .streams + .iter() + .map(|p| format!("{}/{p}", request.root)), + ); + if let Some(pattern) = &config.pattern { + let mut after = String::new(); + loop { + let rows=ctx.sql().query("SELECT path FROM subscription_catalog WHERE path>? AND path>=? AND path(None); + } + let value = actor + .action("__durableStreamSubscriptionSnapshot", vec![]) + .await?; + let entry = value + .get("entry") + .filter(|v| !v.is_null()) + .cloned() + .map(serde_json::from_value::) + .transpose()?; + Ok(entry) + }) + .await? + } + }) + .buffer_unordered(8); + while let Some(entry) = snapshots.next().await { + if let Some(entry) = entry? { + self.observe(ctx, entry).await?; + } + } + Ok(()) + } + + async fn add_links( + &self, + tx: &rivetkit::SqliteTransaction, + sub: &mut Subscription, + paths: &[String], + ) -> ApiResult<()> { + for batch in paths.chunks(16) { + let placeholders = vec!["?"; batch.len()].join(","); + let rows = tx + .execute( + format!( + "SELECT path,value FROM subscription_catalog WHERE path IN ({placeholders})" + ), + Some( + batch + .iter() + .map(|p| text(format!("{}/{p}", sub.root))) + .collect(), + ), + ) + .await + .map_err(internal)?; + let mut catalog = std::collections::HashMap::new(); + for row in &rows.rows { + let entry: CatalogEntry = + serde_json::from_str(db::string(row, 1).map_err(internal)?) + .map_err(|e| internal(e.into()))?; + catalog.insert(entry.path.clone(), entry); + } + for path in batch { + sub.link( + path, + catalog.get(&format!("{}/{path}", sub.root)), + true, + true, + ); + } + } + Ok(()) + } + + async fn apply_control( + &self, + sub: &mut Subscription, + r: &Control, + authorization: Option<&Authorization>, + signing: Value, + tx: &rivetkit::SqliteTransaction, + ) -> ApiResult { + let now = now_ms(); + match r.operation.as_str() { + "get" => Ok(Reply::ok(200, sub.public(signing))), + "delete" => { + sub.deleted = true; + sub.wake = None; + sub.links.clear(); + sub.generation += 1; + Ok(Reply::ok(204, Value::Null)) + } + "add" => { + let paths = added_streams(&r.body)?; + self.add_links(tx, sub, &paths).await?; + sub.resume_delivery(now); + Ok(Reply::ok(204, Value::Null)) + } + "remove" => { + let path = normalize_path(r.path.as_deref().unwrap_or(""), false)?; + if let Some(link) = sub.links.get_mut(&path) { + link.explicit = false; + if !link.glob { + sub.links.remove(&path); + } + } + sub.resume_delivery(now); + Ok(Reply::ok(204, Value::Null)) + } + "callback" | "ack" | "release" => { + let progress: Progress = serde_json::from_value(r.body.clone()) + .map_err(|_| ApiError::invalid("Invalid wake progress"))?; + if (r.operation == "callback") != (sub.config.kind == DeliveryType::Webhook) { + return Err(ApiError::invalid("Wrong delivery type")); + } + if r.public_base != sub.public_base { + return Err(ApiError::new( + 401, + "TOKEN_INVALID", + "Capability is for a different public stream root", + )); + } + match authorization.ok_or_else(|| { + ApiError::new(401, "TOKEN_INVALID", "Invalid capability token") + })? { + Authorization::Engine(capability) => { + validate_capability(capability, sub, &r.operation, now)? + } + Authorization::Legacy(keys) => { + keys.verify(r.token.as_deref().unwrap_or(""), sub, now)? + } + } + let w = sub.wake.as_ref().ok_or_else(ApiError::fenced)?; + if w.id != progress.wake_id + || w.generation != progress.generation + || (sub.config.kind == DeliveryType::PullWake && w.holder.is_none()) + { + return Err(ApiError::fenced()); + } + if r.operation == "release" { + if !progress.acks.is_empty() || progress.done { + return Err(ApiError::invalid("release cannot acknowledge")); + } + sub.finish(now, false); + return Ok(Reply::ok(204, Value::Null)); + } + sub.apply_acks(&progress.acks)?; + let next = if progress.done { + sub.finish(now, false) + } else { + let w = sub.wake.as_mut().unwrap(); + w.deadline = Some((now + sub.config.lease_ttl_ms).min(w.token_expires)); + false + }; + Ok(Reply::ok(200, json!({"ok":true,"next_wake":next}))) + } + _ => Err(ApiError::new(404, "NOT_FOUND", "Unknown control route")), + } + } + + async fn pump(self: Arc, ctx: Ctx) -> Result<()> { + { + let _guard = self.lock.lock().await; + if let Some(deadline) = db::cleanup_legacy(ctx.sql(), now_ms()).await? { + self.arm(&ctx, deadline).await?; + } + } + if self.pumping.swap(true, Ordering::AcqRel) { + return Ok(()); + } + struct Reset<'a>(&'a AtomicBool); + impl Drop for Reset<'_> { + fn drop(&mut self) { + self.0.store(false, Ordering::Release); + } + } + let _reset = Reset(&self.pumping); + // Fanout and reservation can each rewrite a maximum-size snapshot. + // Commit separately and release the mutation lock between those steps. + self.fanout_tick(&ctx).await?; + // Always leave a recovery alarm before marking delivery in flight. + let jobs = { + let _guard = self.lock.lock().await; + let due = ctx + .sql() + .query( + "SELECT key FROM subscriptions WHERE due<=? LIMIT 1", + Some(vec![rivetkit::BindParam::Integer(now_ms())]), + ) + .await?; + if due.rows.is_empty() { + let rows = ctx + .sql() + .query( + "SELECT MIN(due) FROM subscriptions WHERE due IS NOT NULL", + None, + ) + .await?; + if let Some(rivetkit::ColumnValue::Integer(at)) = + rows.rows.first().and_then(|r| r.first()) + { + self.arm(&ctx, (*at).max(now_ms() + 10)).await?; + } + return Ok(()); + } + self.arm(&ctx, now_ms() + 1_000).await?; + let tx = db::begin(ctx.sql()).await?; + let rows = tx + .execute( + "SELECT key,octet_length(value) FROM subscriptions WHERE due<=? ORDER BY due LIMIT 8", + Some(vec![rivetkit::BindParam::Integer(now_ms())]), + ) + .await?; + let mut jobs = Vec::new(); + let mut state_bytes = 0_i64; + for row in &rows.rows { + let Some(rivetkit::ColumnValue::Integer(bytes)) = row.get(1) else { + return Err(anyhow!("Missing subscription state size")); + }; + // Bound aggregate rewrites within the transaction budget. + // Always process one row so a large subscription cannot starve. + if state_bytes > 0 && state_bytes.saturating_add(*bytes) > 8 * 1024 * 1024 { + break; + } + state_bytes = state_bytes.saturating_add(*bytes); + let mut sub = db::load(&tx, db::string(row, 0)?).await?.unwrap(); + sub.expire(now_ms()); + if let Some(w) = sub.wake.as_mut() + && w.next_attempt_at.is_some_and(|n| n <= now_ms()) + && w.dispatch_until.is_none_or(|d| d <= now_ms()) + { + w.dispatch_until = Some(now_ms() + DISPATCH_TIMEOUT); + jobs.push(sub.clone()); + } + db::save(&tx, &sub).await?; + } + tx.commit().await?; + jobs + }; + if !jobs.is_empty() { + fault::checkpoint("after_wake_persist"); + } + if !jobs.is_empty() { + stream::iter(jobs) + .for_each_concurrent(8, |sub| { + let ctx = &ctx; + let this = &self; + async move { + let result = tokio::time::timeout( + Duration::from_millis(DISPATCH_TIMEOUT as u64 - 1_000), + this.deliver(ctx, &sub), + ) + .await + .unwrap_or_else(|_| Err(anyhow!("Delivery attempt timed out"))); + if let Err(error) = this.delivered(ctx, &sub, result).await { + tracing::warn!(?error, "Failed to commit subscription delivery result"); + } + } + }) + .await; + } + let rows = ctx + .sql() + .query( + "SELECT MIN(due) FROM subscriptions WHERE due IS NOT NULL", + None, + ) + .await?; + if let Some(rivetkit::ColumnValue::Integer(due)) = rows.rows.first().and_then(|r| r.first()) + { + self.arm(&ctx, (*due).max(now_ms() + 10)).await?; + } + if !ctx + .sql() + .query("SELECT 1 FROM subscription_fanout LIMIT 1", None) + .await? + .rows + .is_empty() + { + self.arm(&ctx, now_ms() + 10).await?; + } + Ok(()) + } + + async fn deliver(&self, ctx: &Ctx, sub: &Subscription) -> Result { + let w = sub.wake.as_ref().unwrap(); + if let Some(webhook) = &sub.config.webhook { + let token = self + .webhook_capability(ctx, sub) + .await + .map_err(api_failure)?; + let body = webhook_body(sub, &token)?; + let signature = self.engine.sign(body.clone()).await?; + fault::pause("after_webhook_signing").await; + // Signing may outlive a lease or race deletion/recreation. Never deliver + // a result for a wake that is already fenced. + self.check_dispatch(ctx, sub).await?; + // Persisted subscriptions cannot grant the development exception: it + // is a deployment setting and must be consistent across actor runners. + let local = std::env::var("DS_SUBSCRIPTIONS_ALLOW_LOCAL_HTTP").is_ok_and(|s| s == "1"); + return security::deliver(&webhook.url, local, signature.header(), body).await; + } + let target = format!("{}/{}", sub.root, sub.config.wake_stream.as_ref().unwrap()); + let actor = self.client.get( + crate::actor::ACTOR_NAME, + vec![crate::facade::stream_actor_key( + &ctx.state().tenant_scope, + &target, + )], + GetOptions::default(), + )?; + let event = json!({"type":"wake","subscription_id":sub.id,"stream":w.snapshot.iter().find(|s|s.info.has_pending).map(|s|&s.info.path),"generation":w.generation,"ts":w.issued_at}); + let value=tokio::time::timeout(Duration::from_secs(10),actor.action("__durableStreamAppendWake",vec![json!({"wake_id":w.id,"event":event,"origin_sub_key":db::key(&sub.root,&sub.id)})])).await??; + let status = value + .as_u64() + .ok_or_else(|| anyhow!("Wake append omitted status"))?; + if !(200..300).contains(&status) { + return Err(anyhow!("Wake stream append HTTP {status}")); + } + Ok(false) + } + async fn delivered( + &self, + ctx: &Ctx, + sent: &Subscription, + result: Result, + ) -> Result<()> { + let _guard = self.lock.lock().await; + let tx = db::begin(ctx.sql()).await?; + let Some(mut sub) = db::load(&tx, &db::key(&sent.root, &sent.id)).await? else { + tx.rollback().await?; + return Ok(()); + }; + sub.expire(now_ms()); + let same = sub + .wake + .as_ref() + .zip(sent.wake.as_ref()) + .is_some_and(|(a, b)| a.id == b.id && a.generation == b.generation); + if !sub.deleted && same { + match result { + Ok(true) => { + sub.finish(now_ms(), true); + } + Ok(false) => { + let w = sub.wake.as_mut().unwrap(); + w.delivery_error = None; + w.published = true; + w.attempts = 0; + w.next_attempt_at = None; + w.dispatch_until = None; + } + Err(error) => { + let w = sub.wake.as_mut().unwrap(); + w.attempts = w.attempts.saturating_add(1); + let engine = error.downcast_ref::(); + let api = error.downcast_ref::(); + let status = engine.map(|e| e.status).or_else(|| api.map(|e| e.status)); + let code = engine + .map(|e| e.code) + .or_else(|| api.and_then(|e| e.body["error"]["code"].as_str())) + .unwrap_or("WEBHOOK_DELIVERY_FAILED"); + let retry_after = engine + .and_then(|e| e.retry_after) + .or_else(|| api.and_then(|e| e.retry_after)); + let permanent = matches!(status, Some(400 | 401 | 403 | 413)); + let delay = retry_delay(w.attempts) + .max(retry_after.unwrap_or(0).min((i64::MAX / 1000) as u64) as i64 * 1_000); + w.next_attempt_at = (!permanent).then(|| now_ms().saturating_add(delay)); + w.delivery_error = + Some(json!({"code":code,"status":status,"retry_at":w.next_attempt_at})); + if permanent { + w.deadline = None; + } + w.dispatch_until = None; + // Retries retain identity until lease expiry; expiry preserves + // next_attempt_at and attempts in the next fenced generation. + } + } + } + db::save(&tx, &sub).await?; + tx.commit().await + } +} +fn added_streams(body: &Value) -> ApiResult> { + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct Add { + streams: Vec, + } + let add: Add = serde_json::from_value(body.clone()) + .map_err(|_| ApiError::invalid("Expected streams array"))?; + if add.streams.len() > 4096 { + return Err(ApiError::invalid("Too many streams")); + } + add.streams + .iter() + .map(|path| normalize_path(path, false)) + .collect() +} + +fn retry_delay(attempt: u32) -> i64 { + let base = (1_000_i64.saturating_mul(1_i64 << attempt.saturating_sub(1).min(6))).min(60_000); + let random = uuid::Uuid::new_v4().as_u128() as u64; + (base * 80 / 100 + (random % (base as u64 * 40 / 100 + 1)) as i64).min(60_000) +} +fn internal(error: anyhow::Error) -> ApiError { + let error = match error.downcast::() { + Ok(error) => return error, + Err(error) => error, + }; + tracing::error!(?error, "Subscription storage failure"); + ApiError::new(500, "INTERNAL", "Subscription storage failure") +} +pub fn now_ms() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as i64 +} +fn url_component(s: &str) -> String { + url::form_urlencoded::byte_serialize(s.as_bytes()) + .collect::() + .replace('+', "%20") +} + +#[cfg(test)] +mod tests; + +#[cfg(test)] +mod signing_tests; diff --git a/src/subscriptions/model.rs b/src/subscriptions/model.rs new file mode 100644 index 0000000..2b0929e --- /dev/null +++ b/src/subscriptions/model.rs @@ -0,0 +1,588 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; + +use crate::protocol::ZERO_OFFSET; + +pub type ApiResult = Result; + +#[derive(Debug)] +pub struct ApiError { + pub status: u16, + pub body: Value, + pub retry_after: Option, +} + +impl ApiError { + pub fn new(status: u16, code: &str, message: &str) -> Self { + Self { + status, + retry_after: None, + body: json!({"error": {"code": code, "message": message}}), + } + } + pub fn invalid(message: &str) -> Self { + Self::new(400, "INVALID_REQUEST", message) + } + pub fn fenced() -> Self { + Self::new(409, "FENCED", "Wake is no longer current") + } +} + +impl std::fmt::Display for ApiError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{} (HTTP {})", + self.body["error"]["code"].as_str().unwrap_or("ERROR"), + self.status + ) + } +} +impl std::error::Error for ApiError {} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub enum DeliveryType { + Webhook, + PullWake, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct Webhook { + pub url: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct Config { + #[serde(rename = "type")] + pub kind: DeliveryType, + #[serde(default)] + pub pattern: Option, + #[serde(default)] + pub streams: Vec, + #[serde(default)] + pub webhook: Option, + #[serde(default)] + pub wake_stream: Option, + #[serde(default = "default_lease")] + pub lease_ttl_ms: i64, + #[serde(default)] + pub description: Option, +} +fn default_lease() -> i64 { + 30_000 +} + +impl Config { + pub fn normalize(mut self) -> ApiResult { + if !(1_000..=600_000).contains(&self.lease_ttl_ms) { + return Err(ApiError::invalid( + "lease_ttl_ms must be an integer between 1000 and 600000", + )); + } + if self.streams.len() > 4096 { + return Err(ApiError::invalid("Too many explicit streams")); + } + self.streams = self + .streams + .iter() + .map(|p| normalize_path(p, false)) + .collect::>>()?; + self.streams.sort(); + self.streams.dedup(); + self.pattern = self + .pattern + .as_deref() + .map(|p| normalize_path(p, true)) + .transpose()?; + if self.pattern.is_none() && self.streams.is_empty() { + return Err(ApiError::invalid( + "A pattern or explicit streams is required", + )); + } + self.wake_stream = self + .wake_stream + .as_deref() + .map(|p| normalize_path(p, false)) + .transpose()?; + match self.kind { + DeliveryType::Webhook if self.webhook.is_none() || self.wake_stream.is_some() => { + return Err(ApiError::invalid( + "webhook delivery requires webhook.url and no wake_stream", + )); + } + DeliveryType::PullWake if self.wake_stream.is_none() || self.webhook.is_some() => { + return Err(ApiError::invalid( + "pull-wake delivery requires wake_stream and no webhook", + )); + } + _ => {} + } + if self.description.as_ref().is_some_and(|d| d.len() > 4096) { + return Err(ApiError::invalid("Description too long")); + } + Ok(self) + } + pub fn hash(&self) -> String { + format!( + "{:x}", + Sha256::digest(serde_json::to_vec(self).expect("config is serializable")) + ) + } +} + +pub fn normalize_path(path: &str, pattern: bool) -> ApiResult { + let path = path.trim_matches('/'); + if path.is_empty() + || path.len() > 2048 + || path.contains(['?', '#', '\\']) + || path.chars().any(char::is_control) + { + return Err(ApiError::invalid("Invalid stream-root-relative path")); + } + for (index, segment) in path.split('/').enumerate() { + let decoded = percent_encoding::percent_decode_str(segment) + .decode_utf8() + .map_err(|_| ApiError::invalid("Invalid path encoding"))?; + let bytes = segment.as_bytes(); + if bytes.iter().enumerate().any(|(i, b)| { + *b == b'%' + && (i + 2 >= bytes.len() + || !bytes[i + 1].is_ascii_hexdigit() + || !bytes[i + 2].is_ascii_hexdigit()) + }) || segment.is_empty() + || decoded == "." + || decoded == ".." + || decoded.contains('\\') + || decoded.chars().any(char::is_control) + || (index == 0 && (decoded == "__ds" || decoded.starts_with("__ds/"))) + || (segment.contains('*') && (!pattern || (segment != "*" && segment != "**"))) + { + return Err(ApiError::invalid("Invalid stream-root-relative path")); + } + } + let base = url::Url::parse("https://durable-stream.invalid/").unwrap(); + let canonical = base + .join(&format!("/{path}")) + .map_err(|_| ApiError::invalid("Invalid stream path"))?; + Ok(canonical.path().trim_start_matches('/').to_owned()) +} + +// Dynamic programming avoids exponential recursion for patterns with repeated **. +pub fn glob_matches(pattern: &str, path: &str) -> bool { + let parts: Vec<_> = path.split('/').collect(); + let mut matched = vec![false; parts.len() + 1]; + matched[0] = true; + for segment in pattern.split('/') { + if segment == "**" { + for i in 1..matched.len() { + matched[i] |= matched[i - 1]; + } + } else { + for i in (1..matched.len()).rev() { + matched[i] = matched[i - 1] && (segment == "*" || segment == parts[i - 1]); + } + matched[0] = false; + } + } + matched[parts.len()] +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct WakeInterval { + pub before: String, + pub after: String, + pub incarnation: String, + #[serde(default)] + pub origin: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct CatalogEntry { + pub path: String, + pub version: i64, + pub incarnation: String, + pub tail: String, + /// Beginning of the new incarnation; inherited fork data is historical. + pub start: String, + pub exists: bool, + #[serde(default)] + pub wakes: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Link { + #[serde(default)] + pub version: i64, + pub explicit: bool, + pub glob: bool, + pub incarnation: String, + pub acked: String, + pub tail: String, + #[serde(default)] + pub synthetic: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct StreamInfo { + pub path: String, + pub link_type: String, + pub acked_offset: String, + pub tail_offset: String, + pub has_pending: bool, + #[serde(skip)] + pub incarnation: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Snapshot { + pub info: StreamInfo, + pub incarnation: String, +} + +#[derive(Clone, Serialize, Deserialize)] +pub struct Wake { + pub id: String, + pub generation: i64, + pub snapshot: Vec, + pub holder: Option, + pub deadline: Option, + pub issued_at: i64, + pub token_expires: i64, + /// Engine-issued capability, committed before it is exposed to a worker. + #[serde(default)] + pub capability: Option, + #[serde(default)] + pub delivery_error: Option, + pub published: bool, + pub attempts: u32, + pub next_attempt_at: Option, + /// Durable dispatch reservation prevents overlapping sends after a crash. + pub dispatch_until: Option, +} + +#[derive(Clone, Serialize, Deserialize)] +pub struct Subscription { + pub id: String, + pub root: String, + pub public_base: String, + pub config: Config, + pub config_hash: String, + pub created_at: String, + pub generation: i64, + pub deleted: bool, + pub links: BTreeMap, + pub wake: Option, +} + +impl Subscription { + pub fn new( + id: String, + root: String, + public_base: String, + config: Config, + generation: i64, + now: i64, + ) -> Self { + Self { + id, + root, + public_base, + config_hash: config.hash(), + config, + created_at: chrono::DateTime::from_timestamp_millis(now) + .unwrap() + .to_rfc3339_opts(chrono::SecondsFormat::Millis, true), + generation, + deleted: false, + links: BTreeMap::new(), + wake: None, + } + } + pub fn relative<'a>(&self, path: &'a str) -> Option<&'a str> { + path.strip_prefix(&self.root)?.strip_prefix('/') + } + pub fn matches(&self, path: &str) -> bool { + self.config + .pattern + .as_ref() + .is_some_and(|p| glob_matches(p, path)) + } + pub fn link( + &mut self, + path: &str, + entry: Option<&CatalogEntry>, + explicit: bool, + historical: bool, + ) { + let glob = self.matches(path); + let tail = entry + .filter(|e| e.exists) + .map_or(ZERO_OFFSET, |e| e.tail.as_str()); + let ack = if historical { + tail + } else { + entry.map_or(ZERO_OFFSET, |e| e.start.as_str()) + }; + let incarnation = entry.map_or("", |e| e.incarnation.as_str()); + let link = self.links.entry(path.to_owned()).or_insert_with(|| Link { + version: entry.map_or(0, |e| e.version), + synthetic: Vec::new(), + explicit: false, + glob, + incarnation: incarnation.to_owned(), + acked: ack.to_owned(), + tail: tail.to_owned(), + }); + link.explicit |= explicit; + link.glob |= glob; + } + pub fn observe(&mut self, entry: &CatalogEntry) -> bool { + let Some(path) = self.relative(&entry.path).map(str::to_owned) else { + return false; + }; + let newly_linked = !self.links.contains_key(&path); + if newly_linked && self.matches(&path) && entry.exists { + self.link(&path, Some(entry), false, false); + } + let origin = format!("{}/{}", self.root, self.id); + if let Some(link) = self.links.get_mut(&path) { + if !newly_linked && entry.version <= link.version { + return false; + } + link.version = entry.version; + if link.incarnation != entry.incarnation || !entry.exists { + // An old worker must never acknowledge an offset in another + // incarnation, even when the opaque strings happen to overlap. + self.wake = None; + } + if link.incarnation != entry.incarnation { + link.incarnation = entry.incarnation.clone(); + link.acked = entry.start.clone(); + link.synthetic.clear(); + } + for interval in &entry.wakes { + if interval.origin == origin + && interval.incarnation == entry.incarnation + && interval.after > link.acked + && !link.synthetic.iter().any(|old| old.after == interval.after) + { + link.synthetic.push(interval.clone()); + } + } + link.skip_synthetic(); + link.tail = if entry.exists { + entry.tail.clone() + } else { + link.acked.clone() + }; + return true; + } + false + } + pub fn infos(&self) -> Vec { + self.links + .iter() + .map(|(path, link)| StreamInfo { + path: path.clone(), + link_type: if link.explicit { "explicit" } else { "glob" }.to_owned(), + acked_offset: link.acked.clone(), + tail_offset: link.tail.clone(), + has_pending: link.tail > link.acked, + incarnation: link.incarnation.clone(), + }) + .collect() + } + pub fn pending(&self) -> bool { + self.links.values().any(|l| l.tail > l.acked) + } + pub fn ensure_wake(&mut self, now: i64) -> bool { + if self.deleted || self.wake.is_some() || !self.pending() { + return false; + } + self.generation += 1; + self.wake = Some(Wake { + id: format!("w_{}", uuid::Uuid::new_v4().simple()), + generation: self.generation, + snapshot: self + .infos() + .into_iter() + .map(|info| Snapshot { + incarnation: info.incarnation.clone(), + info, + }) + .collect(), + holder: None, + // The processing lease starts immediately before delivery, after Engine I/O. + deadline: None, + issued_at: now, + token_expires: now + 86_400_000, + capability: None, + delivery_error: None, + published: false, + attempts: 0, + next_attempt_at: Some(now), + dispatch_until: None, + }); + true + } + /// Explicit management changes retry operator-correctable signing failures. + pub fn resume_delivery(&mut self, now: i64) -> bool { + if self + .wake + .as_ref() + .is_some_and(|w| w.delivery_error.is_some() && w.next_attempt_at.is_none()) + { + // Rebuild the snapshot so membership corrections can reduce its size. + self.wake = None; + self.ensure_wake(now); + return true; + } + false + } + pub fn expire(&mut self, now: i64) -> bool { + if self + .wake + .as_ref() + .is_some_and(|w| w.deadline.is_some_and(|d| d <= now) || w.token_expires <= now) + { + // Failed delivery retains its persisted retry schedule across generations. + let delivery_error = self.wake.as_ref().and_then(|w| w.delivery_error.clone()); + let retry = self.wake.as_ref().map(|w| { + if self.config.kind == DeliveryType::Webhook && w.dispatch_until.is_some() { + // An expired in-flight attempt has an unknown result. Fence + // it, but charge its retry before the old response arrives. + let attempts = w.attempts.saturating_add(1); + (attempts, Some(now + super::retry_delay(attempts))) + } else { + (w.attempts, w.next_attempt_at) + } + }); + self.wake = None; + self.ensure_wake(now); + if let (Some(w), Some((attempts, next))) = (&mut self.wake, retry) + && attempts > 0 + { + w.delivery_error = delivery_error; + w.attempts = attempts; + w.next_attempt_at = next; + // A failed delivery does not consume its next processing lease while waiting. + w.deadline = None; + } + return true; + } + false + } + pub fn finish(&mut self, now: i64, auto_ack: bool) -> bool { + if auto_ack && let Some(wake) = &self.wake { + for snap in &wake.snapshot { + if let Some(link) = self.links.get_mut(&snap.info.path) + && link.incarnation == snap.incarnation + && snap.info.tail_offset > link.acked + { + link.acked = snap.info.tail_offset.clone(); + } + } + } + for link in self.links.values_mut() { + link.skip_synthetic(); + } + self.wake = None; + self.ensure_wake(now) + } + pub fn apply_acks(&mut self, acks: &[Ack]) -> ApiResult<()> { + let mut offsets = BTreeMap::new(); + for ack in acks { + let path = normalize_path(&ack.stream, false)?; + let link = self.links.get(&path).ok_or_else(|| { + ApiError::new(409, "INVALID_OFFSET", "Unknown subscription stream") + })?; + let prior = offsets.get(&path).copied().unwrap_or(&link.acked); + if !valid_offset(&ack.offset) || ack.offset < *prior || ack.offset > link.tail { + return Err(ApiError::new( + 409, + "INVALID_OFFSET", + "Invalid, regressing, or beyond-tail acknowledgment", + )); + } + offsets.insert(path, &ack.offset); + } + for (path, offset) in offsets { + let link = self.links.get_mut(&path).unwrap(); + link.acked = offset.clone(); + link.skip_synthetic(); + } + Ok(()) + } + pub fn due(&self) -> Option { + let w = self.wake.as_ref()?; + [ + w.deadline, + Some(w.token_expires), + w.next_attempt_at + .map(|n| n.max(w.dispatch_until.unwrap_or(0))), + ] + .into_iter() + .flatten() + .min() + } + pub fn public(&self, signing: Value) -> Value { + json!({"id": self.id, "subscription_id": self.id, "type": self.config.kind, + "pattern": self.config.pattern, "streams": self.infos().into_iter().map(|s| json!({"path":s.path,"link_type":s.link_type,"acked_offset":s.acked_offset})).collect::>(), + "webhook": self.config.webhook.as_ref().map(|w| json!({"url":w.url,"signing":signing})), + "wake_stream": self.config.wake_stream, "lease_ttl_ms":self.config.lease_ttl_ms, + "created_at":self.created_at, "status": if self.wake.as_ref().is_some_and(|w| w.attempts > 0) {"failed"} else {"active"}, + "description":self.config.description, + "delivery_error":self.wake.as_ref().and_then(|w| w.delivery_error.as_ref())}) + } + pub fn paths(&self) -> BTreeSet { + self.links + .keys() + .map(|p| format!("{}/{p}", self.root)) + .collect() + } +} + +impl Link { + fn skip_synthetic(&mut self) { + self.synthetic.sort_by(|a, b| a.before.cmp(&b.before)); + for span in &self.synthetic { + if span.before <= self.acked && span.after > self.acked { + self.acked = span.after.clone(); + } + } + self.synthetic.retain(|span| span.after > self.acked); + } +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Ack { + #[serde(alias = "path")] + pub stream: String, + pub offset: String, +} +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Progress { + pub wake_id: String, + pub generation: i64, + #[serde(default)] + pub acks: Vec, + #[serde(default)] + pub done: bool, +} + +pub fn valid_offset(offset: &str) -> bool { + offset.len() == ZERO_OFFSET.len() + && offset.as_bytes()[16] == b'_' + && offset + .bytes() + .enumerate() + .all(|(i, c)| i == 16 || c.is_ascii_digit()) +} diff --git a/src/subscriptions/security.rs b/src/subscriptions/security.rs new file mode 100644 index 0000000..28b52c2 --- /dev/null +++ b/src/subscriptions/security.rs @@ -0,0 +1,121 @@ +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::time::Duration; + +use anyhow::{Context, Result, bail}; +use futures_util::StreamExt; +use ring::hmac; +use serde_json::Value; + +pub fn constant_time_equal(a: &[u8], b: &[u8]) -> bool { + let key = hmac::Key::new(hmac::HMAC_SHA256, b"durable-streams-management-comparison"); + hmac::verify(&key, b, hmac::sign(&key, a).as_ref()).is_ok() +} + +fn public_ipv4(ip: Ipv4Addr) -> bool { + let [a, b, c, _] = ip.octets(); + !(ip.is_private() + || ip.is_loopback() + || ip.is_link_local() + || ip.is_unspecified() + || ip.is_multicast() + || ip.is_broadcast() + || a == 0 + || a >= 240 + || (a == 100 && (64..=127).contains(&b)) + || (a == 192 && b == 0 && (c == 0 || c == 2)) + || (a == 192 && b == 88 && c == 99) + || (a == 198 && (b == 18 || b == 19 || (b == 51 && c == 100))) + || (a == 203 && b == 0 && c == 113)) +} +pub fn public_ip(ip: IpAddr) -> bool { + match ip { + IpAddr::V4(v4) => public_ipv4(v4), + IpAddr::V6(v6) => { + if let Some(v4) = v6.to_ipv4_mapped() { + return public_ipv4(v4); + } + let s = v6.segments(); + // Only global unicast; exclude transition, benchmarking and documentation ranges. + (s[0] & 0xe000) == 0x2000 + && s[0] != 0x2002 + && !(s[0] == 0x2001 && (s[1] < 0x200 || s[1] == 0xdb8)) + && !(s[0] == 0x3fff && s[1] < 0x1000) + } + } +} + +pub async fn destination(raw: &str, local: bool) -> Result<(url::Url, Vec)> { + let url = url::Url::parse(raw)?; + if !url.username().is_empty() || url.password().is_some() || url.fragment().is_some() { + bail!("Webhook credentials and fragments are not allowed"); + } + let host = url + .host_str() + .context("Webhook host required")? + .trim_matches(['[', ']']); + let dev_host = local + && (host == "localhost" + || host.parse::().is_ok_and(|ip| match ip { + IpAddr::V4(ip) => ip.octets()[..3] == [127, 0, 0], + IpAddr::V6(ip) => ip.is_loopback(), + })); + if url.scheme() != "https" && !(url.scheme() == "http" && dev_host) { + bail!("Webhook requires HTTPS (localhost HTTP requires development mode)"); + } + let port = url.port_or_known_default().context("Missing port")?; + let addresses: Vec<_> = tokio::time::timeout( + Duration::from_secs(5), + tokio::net::lookup_host((host, port)), + ) + .await?? + .collect(); + if addresses.is_empty() + || addresses + .iter() + .any(|a| !(public_ip(a.ip()) || dev_host && a.ip().is_loopback())) + { + bail!("Webhook resolves to a non-public address"); + } + Ok((url, addresses)) +} + +pub async fn deliver( + url: &str, + local: bool, + signature: String, + body: bytes::Bytes, +) -> Result { + // Resolve on every attempt and pin vetted addresses to this connection. Disable + // proxies and redirects: neither may bypass address validation or leak tokens. + let (url, addresses) = destination(url, local).await?; + let host = url.host_str().unwrap().to_owned(); + let client = reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(10)) + .connect_timeout(Duration::from_secs(5)) + .resolve_to_addrs(&host, &addresses) + .build()?; + let response = client + .post(url) + .header("content-type", "application/json") + .header("webhook-signature", signature) + .body(body) + .send() + .await?; + if !response.status().is_success() { + bail!("Webhook HTTP {}", response.status()); + } + let mut bytes = Vec::new(); + let mut stream = response.bytes_stream(); + while let Some(chunk) = stream.next().await { + let chunk = chunk?; + if bytes.len() + chunk.len() > 64 * 1024 { + bail!("Webhook response too large"); + } + bytes.extend_from_slice(&chunk); + } + Ok(serde_json::from_slice::(&bytes) + .ok() + .is_some_and(|v| v.get("done") == Some(&Value::Bool(true)))) +} diff --git a/src/subscriptions/signing.rs b/src/subscriptions/signing.rs new file mode 100644 index 0000000..14c08f9 --- /dev/null +++ b/src/subscriptions/signing.rs @@ -0,0 +1,501 @@ +//! Engine I/O is deliberately outside the coordinator's mutation lock. +use super::*; + +pub(super) enum Authorization { + Engine(VerifiedCapability), + Legacy(legacy::LegacyKeys), +} + +impl SubscriptionActor { + pub(super) async fn namespace_id(&self, ctx: &Ctx) -> ApiResult { + self.namespace + .get_or_try_init(|| async { + let namespace = self.engine.namespace_id().await.map_err(engine_failure)?; + let _guard = self.lock.lock().await; + let rows = ctx + .sql() + .query( + "SELECT namespace FROM subscription_engine_identity WHERE id=1", + None, + ) + .await + .map_err(internal)?; + if let Some(row) = rows.rows.first() { + if db::string(row, 0).map_err(internal)? != namespace { + return Err(ApiError::new( + 503, + "ENGINE_NAMESPACE_CHANGED", + "Configured Engine namespace does not match subscription storage", + )); + } + } else { + ctx.sql() + .execute( + "INSERT INTO subscription_engine_identity(id,namespace) VALUES(1,?)", + Some(vec![text(&namespace)]), + ) + .await + .map_err(internal)?; + } + Ok(namespace) + }) + .await + .cloned() + } + + pub(super) async fn signing_metadata( + &self, + ctx: &Ctx, + base: &str, + webhook: bool, + ) -> ApiResult { + if !webhook { + return Ok(Value::Null); + } + let namespace = self.namespace_id(ctx).await?; + let metadata = self + .engine + .metadata(&namespace) + .await + .map_err(engine_failure)?; + Ok( + json!({"alg":metadata.alg,"kid":metadata.kid,"jwks_url":format!("{base}/__ds/jwks.json")}), + ) + } + + pub(super) async fn public_keys(&self, ctx: &Ctx) -> ApiResult { + let namespace = self.namespace_id(ctx).await?; + let mut fresh = self + .engine + .webhook_jwks(&namespace) + .await + .map_err(engine_failure)?; + if fresh.age_secs >= 300 { + return Err(ApiError::new( + 503, + "SIGNING_DISCOVERY_STALE", + "Engine signing discovery exceeded its freshness budget", + )); + } + let old = { + let _guard = self.lock.lock().await; + db::legacy(ctx.sql()).await.map_err(internal)? + }; + if let Some(old) = old { + let keys = fresh + .value + .get_mut("keys") + .and_then(Value::as_array_mut) + .ok_or_else(|| { + ApiError::new( + 503, + "SIGNING_DISCOVERY_INVALID", + "Invalid Engine key discovery", + ) + })?; + for key in old.public_jwks(now_ms()) { + if !keys.iter().any(|k| k["kid"] == key["kid"]) { + keys.push(key); + } + } + } + let mut reply = Reply::ok(200, fresh.value); + reply.age = Some(fresh.age_secs); + Ok(reply) + } + + pub(super) async fn prepare_authorization( + &self, + ctx: &Ctx, + request: &Control, + ) -> ApiResult> { + if !matches!(request.operation.as_str(), "callback" | "ack" | "release") { + return Ok(None); + } + let token = request.token.as_deref().unwrap_or(""); + // This exact header was the only format emitted by the old implementation. + // Never fall back to legacy verification after an Engine verification failure. + if token.starts_with("eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.") { + let _guard = self.lock.lock().await; + return db::legacy(ctx.sql()) + .await + .map_err(internal)? + .map(|keys| Some(Authorization::Legacy(keys))) + .ok_or_else(|| { + ApiError::new( + 401, + "TOKEN_INVALID", + "Legacy capability verification has ended", + ) + }); + } + let namespace = self.namespace_id(ctx).await?; + let kind = if request.operation == "callback" { + CapabilityKind::Callback + } else { + CapabilityKind::Claim + }; + let verified = self + .engine + .verify(token, &namespace, kind, &request.operation, now_ms()) + .await + .map_err(engine_failure)?; + if verified.claims.tenant_scope != ctx.state().tenant_scope + || verified.claims.root != request.public_base + { + return Err(ApiError::new( + 401, + "TOKEN_INVALID", + "Capability is for a different tenant or stream root", + )); + } + Ok(Some(Authorization::Engine(verified))) + } + + pub(super) async fn claim( + &self, + ctx: &Ctx, + snapshot: Subscription, + request: &Control, + ) -> ApiResult { + if snapshot.config.kind != DeliveryType::PullWake { + return Err(ApiError::invalid("Claim requires pull-wake subscription")); + } + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct Claim { + worker: String, + } + let claim: Claim = serde_json::from_value(request.body.clone()) + .map_err(|_| ApiError::invalid("worker is required"))?; + if claim.worker.is_empty() || claim.worker.len() > 256 { + return Err(ApiError::invalid("Invalid worker")); + } + let wake = snapshot + .wake + .as_ref() + .ok_or_else(|| ApiError::new(409, "NO_PENDING_WORK", "No pending work"))?; + if let Some(holder) = &wake.holder { + return Ok(already_claimed(holder, wake.generation)); + } + let deadline = request.deadline.unwrap_or_else(|| now_ms() + 14_000); + let issued = tokio::time::timeout(claim_budget(deadline)?, async { + let namespace = self.namespace_id(ctx).await?; + let claims = capability_claims(&snapshot, &ctx.state().tenant_scope)?; + self.engine + .issue(&claims, CapabilityKind::Claim, &namespace) + .await + .map_err(engine_failure) + }) + .await + .map_err(|_| { + ApiError::new( + 503, + "ENGINE_UNAVAILABLE", + "Engine capability issuance timed out", + ) + })??; + fault::pause("after_application_issuance").await; + let _guard = self.lock.lock().await; + self.arm(ctx, now_ms() + 1_000).await.map_err(internal)?; + // A slow issuer or lock wait must not acquire a lease after the caller + // has already timed out. Bound the commit phase by the same request budget. + let tx = ctx + .sql() + .begin_transaction(Some(claim_budget(deadline)?)) + .await + .map_err(internal)?; + let mut sub = db::load(&tx, &db::key(&snapshot.root, &snapshot.id)) + .await + .map_err(internal)? + .ok_or_else(ApiError::fenced)?; + let mut changed = self + .reconcile_subscription(&tx, &mut sub) + .await + .map_err(internal)?; + changed |= sub.expire(now_ms()); + changed |= sub.ensure_wake(now_ms()); + let mut acquired = false; + let result = if sub.deleted || !same_wake(&sub, &snapshot) { + Err(ApiError::fenced()) + } else if let Some(holder) = &sub.wake.as_ref().unwrap().holder { + Ok(already_claimed( + holder, + sub.wake.as_ref().unwrap().generation, + )) + } else if issued.expires_ts <= now_ms() { + Err(ApiError::new( + 401, + "TOKEN_EXPIRED", + "Engine capability expired before the claim completed", + )) + } else { + let w = sub.wake.as_mut().unwrap(); + w.holder = Some(claim.worker); + w.token_expires = issued.expires_ts; + w.capability = Some(issued.token.clone()); + w.deadline = Some((now_ms() + sub.config.lease_ttl_ms).min(issued.expires_ts)); + changed = true; + acquired = true; + Ok(Reply::ok( + 200, + json!({"wake_id":w.id,"generation":w.generation,"token":issued.token,"streams":sub.infos(),"lease_ttl_ms":sub.config.lease_ttl_ms}), + )) + }; + if changed { + if acquired { + db::save_with_lease(&tx, &mut sub).await.map_err(internal)?; + } else { + db::save(&tx, &sub).await.map_err(internal)?; + } + } + if acquired { + claim_budget(deadline)?; + require_live_lease(&sub)?; + } + tx.commit().await.map_err(internal)?; + result + } + + pub(super) async fn webhook_capability( + &self, + ctx: &Ctx, + snapshot: &Subscription, + ) -> ApiResult { + let wake = snapshot.wake.as_ref().ok_or_else(ApiError::fenced)?; + if let Some(token) = &wake.capability { + return Ok(token.clone()); + } + let namespace = self.namespace_id(ctx).await?; + let claims = capability_claims(snapshot, &ctx.state().tenant_scope)?; + let issued = self + .engine + .issue(&claims, CapabilityKind::Callback, &namespace) + .await + .map_err(engine_failure)?; + let _guard = self.lock.lock().await; + let tx = db::begin(ctx.sql()).await.map_err(internal)?; + let mut sub = db::load(&tx, &db::key(&snapshot.root, &snapshot.id)) + .await + .map_err(internal)? + .ok_or_else(ApiError::fenced)?; + let mut changed = sub.expire(now_ms()); + changed |= sub.ensure_wake(now_ms()); + let result = if sub.deleted || !same_wake(&sub, snapshot) { + Err(ApiError::fenced()) + } else if issued.expires_ts <= now_ms() { + Err(ApiError::new( + 401, + "TOKEN_EXPIRED", + "Engine capability expired before delivery", + )) + } else { + let w = sub.wake.as_mut().unwrap(); + if let Some(token) = &w.capability { + Ok(token.clone()) + } else { + w.token_expires = issued.expires_ts; + w.capability = Some(issued.token.clone()); + w.deadline = w.deadline.map(|at| at.min(issued.expires_ts)); + changed = true; + Ok(issued.token) + } + }; + if changed { + db::save(&tx, &sub).await.map_err(internal)?; + } + tx.commit().await.map_err(internal)?; + result + } + + pub(super) async fn check_dispatch( + &self, + ctx: &Ctx, + snapshot: &Subscription, + ) -> Result<()> { + let _guard = self.lock.lock().await; + let tx = db::begin(ctx.sql()).await?; + let mut sub = db::load(&tx, &db::key(&snapshot.root, &snapshot.id)) + .await? + .ok_or_else(|| anyhow!("Wake fenced"))?; + let mut changed = self.reconcile_subscription(&tx, &mut sub).await?; + changed |= sub.expire(now_ms()); + changed |= sub.ensure_wake(now_ms()); + let valid = !sub.deleted && same_wake(&sub, snapshot); + let mut start_lease = false; + if valid { + let w = sub.wake.as_mut().unwrap(); + if w.deadline.is_none() { + w.deadline = Some((now_ms() + sub.config.lease_ttl_ms).min(w.token_expires)); + changed = true; + start_lease = true; + } + } + if changed { + if start_lease { + db::save_with_lease(&tx, &mut sub).await?; + } else { + db::save(&tx, &sub).await?; + } + } + if valid { + require_live_lease(&sub)?; + } + tx.commit().await?; + if !valid { + anyhow::bail!("Wake fenced"); + } + Ok(()) + } +} + +pub(super) fn require_live_lease(sub: &Subscription) -> ApiResult<()> { + let now = now_ms(); + if sub + .wake + .as_ref() + .is_some_and(|wake| wake.token_expires <= now) + { + return Err(ApiError::new( + 401, + "TOKEN_EXPIRED", + "Capability expired before the operation completed", + )); + } + if sub + .wake + .as_ref() + .is_none_or(|wake| wake.deadline.is_none_or(|at| at <= now)) + { + return Err(ApiError::new( + 409, + "LEASE_EXPIRED", + "Processing lease expired before the operation completed", + )); + } + Ok(()) +} + +fn claim_budget(deadline: i64) -> ApiResult { + let remaining = deadline.saturating_sub(now_ms()); + if remaining <= 0 { + return Err(ApiError::new( + 503, + "CLAIM_DEADLINE_EXCEEDED", + "Claim request expired before ownership was acquired", + )); + } + Ok(Duration::from_millis(remaining.min(10_000) as u64)) +} + +fn already_claimed(holder: &str, generation: i64) -> Reply { + Reply::ok( + 409, + json!({"error":{"code":"ALREADY_CLAIMED","current_holder":holder,"generation":generation}}), + ) +} +fn capability_claims(sub: &Subscription, tenant: &str) -> ApiResult { + let w = sub.wake.as_ref().ok_or_else(ApiError::fenced)?; + Ok(CapabilityClaims { + root: sub.public_base.clone(), + tenant_scope: tenant.to_owned(), + subscription_id: sub.id.clone(), + wake_id: w.id.clone(), + generation: w.generation, + operations: if sub.config.kind == DeliveryType::Webhook { + vec!["callback".into()] + } else { + vec!["ack".into(), "release".into()] + }, + }) +} +pub(super) fn same_wake(a: &Subscription, b: &Subscription) -> bool { + a.wake + .as_ref() + .zip(b.wake.as_ref()) + .is_some_and(|(a, b)| a.id == b.id && a.generation == b.generation) +} +pub(super) fn validate_capability( + capability: &VerifiedCapability, + sub: &Subscription, + operation: &str, + now: i64, +) -> ApiResult<()> { + let c = &capability.claims; + if capability.expires_ts <= now { + return Err(ApiError::new( + 401, + "TOKEN_EXPIRED", + "Capability token expired", + )); + } + if c.root != sub.public_base + || c.subscription_id != sub.id + || !c.operations.iter().any(|v| v == operation) + { + return Err(ApiError::new( + 401, + "TOKEN_INVALID", + "Capability is for a different subscription or operation", + )); + } + let w = sub.wake.as_ref().ok_or_else(ApiError::fenced)?; + if sub.deleted || c.wake_id != w.id || c.generation != w.generation { + return Err(ApiError::fenced()); + } + Ok(()) +} +pub(super) fn engine_failure(error: EngineError) -> ApiError { + let mut e = ApiError::new(error.status, error.code, "Engine signing request failed"); + e.retry_after = error.retry_after; + e +} +pub(super) fn api_failure(error: ApiError) -> anyhow::Error { + anyhow::Error::new(error) +} + +/// Bound serialization itself so unbounded glob/add memberships cannot allocate an +/// unbounded webhook. Engine enforces its configured (normally 32 MiB) lower cap. +pub(super) fn webhook_body(sub: &Subscription, token: &str) -> Result { + struct Limited(Vec); + impl std::io::Write for Limited { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + if self.0.len().saturating_add(bytes.len()) > 128 * 1024 * 1024 { + return Err(std::io::Error::other("Webhook snapshot exceeds 128 MiB")); + } + self.0.extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + #[derive(Serialize)] + struct Body<'a> { + subscription_id: &'a str, + wake_id: &'a str, + generation: i64, + streams: Vec<&'a StreamInfo>, + callback_url: String, + callback_token: &'a str, + } + let wake = sub + .wake + .as_ref() + .ok_or_else(|| anyhow!("No delivery wake"))?; + let body = Body { + subscription_id: &sub.id, + wake_id: &wake.id, + generation: wake.generation, + streams: wake.snapshot.iter().map(|s| &s.info).collect(), + callback_url: format!( + "{}/__ds/subscriptions/{}/callback", + sub.public_base, + url_component(&sub.id) + ), + callback_token: token, + }; + let mut output = Limited(Vec::new()); + serde_json::to_writer(&mut output, &body).map_err(|_| api_failure(ApiError::new(413, "WEBHOOK_PAYLOAD_TOO_LARGE", "Webhook snapshot exceeds the signing payload limit; reduce membership or increase the Engine limit")))?; + Ok(output.0.into()) +} diff --git a/src/subscriptions/signing_tests.rs b/src/subscriptions/signing_tests.rs new file mode 100644 index 0000000..9687839 --- /dev/null +++ b/src/subscriptions/signing_tests.rs @@ -0,0 +1,374 @@ +use super::{model::*, signing::webhook_body}; +use serde_json::json; + +fn maximum_subscription() -> Subscription { + let config = serde_json::from_value::(json!({ + "type":"webhook", "pattern":"events/**", + "webhook":{"url":"https://receiver.gameinc.io/hook"} + })) + .unwrap() + .normalize() + .unwrap(); + let mut sub = Subscription::new( + "quoted\"subscription".into(), + "/v1/stream".into(), + "https://streams.gameinc.io/v1/stream".into(), + config, + 0, + 1000, + ); + for i in 0..4096 { + let prefix = format!("events/{i:04}/"); + let remaining = 2048 - prefix.len(); + let raw = format!( + "{prefix}{}{}", + "\u{a0}".repeat(remaining / 2), + "x".repeat(remaining % 2) + ); + assert_eq!(raw.len(), 2048); + let path = normalize_path(&raw, false).unwrap(); + assert!(path.len() > 6000); + sub.links.insert( + path, + Link { + version: 1, + explicit: false, + glob: true, + incarnation: "incarnation".into(), + acked: crate::protocol::ZERO_OFFSET.into(), + tail: "0000000000000000_0000000000000001".into(), + synthetic: vec![], + }, + ); + } + assert!(sub.ensure_wake(1000)); + sub +} + +#[test] +fn maximum_4096_stream_webhook_snapshot_preserves_every_expanded_path() { + let sub = maximum_subscription(); + let token = "x".repeat(6144); + let started = std::time::Instant::now(); + let body = webhook_body(&sub, &token).unwrap(); + let elapsed = started.elapsed(); + assert!(body.len() > 24 * 1024 * 1024); + assert!(body.len() < 32 * 1024 * 1024); + let decoded: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(decoded["subscription_id"], sub.id); + assert_eq!(decoded["callback_token"], token); + let streams = decoded["streams"].as_array().unwrap(); + assert_eq!(streams.len(), 4096); + for (stream, expected) in streams.iter().zip(sub.links.keys()) { + assert_eq!(stream["path"], expected.as_str()); + assert_eq!(stream["tail_offset"], "0000000000000000_0000000000000001"); + } + let state_bytes = serde_json::to_vec(&sub).unwrap().len(); + eprintln!( + "4096 maximum paths: webhook={} bytes, state={} bytes, serialization={elapsed:?}", + body.len(), + state_bytes + ); +} + +mod storage { + use super::*; + use anyhow::{Context, Result}; + use async_trait::async_trait; + use rivetkit::{Action, Handles, prelude::*}; + use serde::{Deserialize, Serialize}; + use sha2::{Digest, Sha256}; + use std::{ + sync::Arc, + time::{Duration, Instant}, + }; + + const NAME: &str = "subscriptionLargeStateTest"; + struct Fixture; + #[derive(Serialize, Deserialize)] + struct Ready; + impl Action for Ready { + type Output = bool; + const NAME: &'static str = "ready"; + } + #[derive(Serialize, Deserialize)] + struct Persist; + impl Action for Persist { + type Output = serde_json::Value; + const NAME: &'static str = "persist"; + } + #[async_trait] + impl Actor for Fixture { + type State = (); + type Input = (); + type Actions = (Ready, Persist); + type Events = (); + type Queue = (); + type ConnParams = (); + type ConnState = (); + type Action = rivetkit::action::Raw; + const HAS_DATABASE: bool = true; + async fn create_state(_: &Ctx, _: ()) -> Result<()> { + Ok(()) + } + async fn create(ctx: &Ctx) -> Result { + super::super::db::ensure(ctx.sql()).await?; + Ok(Self) + } + } + impl Handles for Fixture { + type Future = super::super::BoxFuture; + fn handle(self: Arc, _: Ctx, _: Ready) -> Self::Future { + Box::pin(async { Ok(true) }) + } + } + impl Handles for Fixture { + type Future = super::super::BoxFuture; + fn handle(self: Arc, ctx: Ctx, _: Persist) -> Self::Future { + Box::pin(async move { + let mut sub = maximum_subscription(); + let key = super::super::db::key(&sub.root, &sub.id); + let mut times = Vec::new(); + let mut load_times = Vec::new(); + let mut lease_remaining = Vec::new(); + let mut lease_at_commit = Vec::new(); + // A 4,096-stream snapshot can take over a second to commit on CI. + sub.config.lease_ttl_ms = 10_000; + for round in 0..2 { + let old_deadline = super::super::now_ms() + 1_000; + let wake = sub.wake.as_mut().unwrap(); + wake.deadline = Some(old_deadline); + wake.token_expires = old_deadline + 60_000; + wake.next_attempt_at = None; + let started = Instant::now(); + let tx = super::super::db::begin(ctx.sql()).await?; + super::super::db::save_with_lease(&tx, &mut sub) + .await + .with_context(|| format!("save round {round}"))?; + tx.commit().await.context("commit snapshot")?; + times.push(started.elapsed().as_millis()); + eprintln!("maximum state save round {round}: {:?}", started.elapsed()); + let deadline = sub.wake.as_ref().unwrap().deadline.unwrap(); + lease_at_commit.push(deadline - super::super::now_ms()); + assert!(old_deadline <= super::super::now_ms()); + let persisted = ctx.sql().execute( + "SELECT due,json_extract(value,'$.wake.deadline') FROM subscriptions WHERE key=?", + Some(vec![super::super::db::text(&key)]), + ).await?; + assert!( + matches!(persisted.rows[0][0], rivetkit::ColumnValue::Integer(n) if n == deadline) + ); + assert!( + matches!(persisted.rows[0][1], rivetkit::ColumnValue::Integer(n) if n == deadline) + ); + let remaining = deadline - super::super::now_ms(); + assert!( + remaining > 0, + "fresh minimum lease expired after commit: {remaining}ms" + ); + lease_remaining.push(remaining); + let before = hash(&sub)?; + let started = Instant::now(); + let tx = super::super::db::begin(ctx.sql()).await?; + let loaded = super::super::db::load(&tx, &key) + .await? + .context("Missing saved state")?; + let parts = tx + .execute("SELECT count(*) FROM subscription_value_parts", None) + .await?; + assert!(matches!( + parts.rows[0][0], + rivetkit::ColumnValue::Integer(0) + )); + tx.rollback().await?; + load_times.push(started.elapsed().as_millis()); + eprintln!("maximum state load round {round}: {:?}", started.elapsed()); + assert_eq!(before, hash(&loaded)?); + if round == 0 { + sub.links.first_entry().unwrap().get_mut().acked = + "0000000000000000_0000000000000001".into(); + } + } + // Expired capabilities fail without inserting an owned wake. + let mut expired = sub.clone(); + expired.id = "expired capability".into(); + let wake = expired.wake.as_mut().unwrap(); + wake.token_expires = super::super::now_ms() + 1_000; + let tx = super::super::db::begin(ctx.sql()).await?; + let failure = super::super::db::save_with_lease(&tx, &mut expired) + .await + .unwrap_err(); + let failure = failure.downcast_ref::().context("typed expiry")?; + assert_eq!(failure.status, 401); + assert_eq!(failure.body["error"]["code"], "TOKEN_EXPIRED"); + tx.rollback().await?; + let parts = ctx + .sql() + .execute("SELECT count(*) FROM temp.subscription_value_parts", None) + .await?; + assert!(matches!( + parts.rows[0][0], + rivetkit::ColumnValue::Integer(0) + )); + let rows = ctx + .sql() + .execute( + "SELECT 1 FROM subscriptions WHERE key=?", + Some(vec![super::super::db::text(super::super::db::key( + &expired.root, + &expired.id, + ))]), + ) + .await?; + assert!(rows.rows.is_empty()); + let first_key = key; + sub.id = "second maximum subscription".into(); + let relative = sub.links.first_key_value().unwrap().0.clone(); + let entry = CatalogEntry { + path: format!("{}/{}", sub.root, relative), + version: 2, + incarnation: "incarnation".into(), + tail: "0000000000000000_0000000000000002".into(), + start: crate::protocol::ZERO_OFFSET.into(), + exists: true, + wakes: vec![], + }; + let tx = super::super::db::begin(ctx.sql()).await?; + super::super::db::save(&tx, &sub).await?; + super::super::db::save_catalog(&tx, &entry).await?; + tx.commit().await?; + // This fixture only runs the coordinator's SQL fanout path; + // neither the placeholder Engine client nor delivery is used. + let coordinator = super::super::SubscriptionActor { + client: crate::client::actor_client(&ctx)?, + engine: super::super::EngineClient::new( + &super::super::catalog_import::CatalogApi { + endpoint: "http://127.0.0.1:1".into(), + namespace: "default".into(), + token: None, + }, + None, + )?, + namespace: tokio::sync::OnceCell::new(), + lock: tokio::sync::Mutex::new(()), + pumping: std::sync::atomic::AtomicBool::new(false), + alarm: tokio::sync::Mutex::new(None), + }; + let mut fanout_times = Vec::new(); + for pass in 0..4 { + let round = pass % 2; + if pass == 2 { + // Replay an already-reconciled catalog entry. Reading + // those large states must still consume the batch cap. + ctx.sql().execute( + "INSERT INTO subscription_fanout(path,version,cursor,discover,value) SELECT path,version,'',1,value FROM subscription_catalog WHERE path=?", + Some(vec![super::super::db::text(&entry.path)]), + ).await?; + } + let started = Instant::now(); + let tx = super::super::db::begin(ctx.sql()).await?; + coordinator.fanout_batch(&tx, Some(&entry.path)).await?; + tx.commit().await?; + fanout_times.push(started.elapsed().as_millis()); + let tx = super::super::db::begin(ctx.sql()).await?; + let job = super::super::db::fanout(&tx, Some(&entry.path)).await?; + if round == 0 { + assert_eq!(job.unwrap().cursor, first_key); + } else { + assert!(job.is_none()); + } + let updated = tx + .execute( + "SELECT count(*) FROM subscriptions WHERE json_extract(value,?)=2", + Some(vec![super::super::db::text(format!( + "$.links.\"{relative}\".version" + ))]), + ) + .await?; + let expected = if pass < 2 { round + 1 } else { 2 }; + assert!( + matches!(updated.rows[0][0], rivetkit::ColumnValue::Integer(n) if n == expected) + ); + tx.rollback().await?; + } + Ok( + json!({"streams":sub.links.len(),"save_ms":times,"load_ms":load_times,"fanout_ms":fanout_times,"lease_remaining_ms":lease_remaining,"lease_at_commit_ms":lease_at_commit}), + ) + }) + } + } + fn hash(sub: &Subscription) -> Result> { + struct Writer(Sha256); + impl std::io::Write for Writer { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.update(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let mut writer = Writer(Sha256::new()); + serde_json::to_writer(&mut writer, sub)?; + Ok(writer.0.finalize().to_vec()) + } + + #[tokio::test] + #[ignore = "requires RIVET_TEST_ENDPOINT and matching Engine credentials"] + async fn maximum_snapshot_persists_through_real_engine() -> Result<()> { + let endpoint = + std::env::var("RIVET_TEST_ENDPOINT").context("RIVET_TEST_ENDPOINT required")?; + let token = std::env::var("RIVET_TOKEN").context("RIVET_TOKEN required")?; + let namespace = std::env::var("RIVET_NAMESPACE").unwrap_or_else(|_| "default".into()); + let pool = format!("subscription-state-{}", uuid::Uuid::new_v4()); + let mut registry = rivetkit::Registry::new(); + registry.register_actor_with::( + NAME, + rivetkit::ActorConfig { + has_database: true, + has_state: true, + action_timeout: Duration::from_secs(180), + ..Default::default() + }, + ); + let shutdown = tokio_util::sync::CancellationToken::new(); + let _guard = shutdown.clone().drop_guard(); + let server = tokio::spawn(registry.serve_with_config( + rivetkit::ServeConfig { + endpoint: endpoint.clone(), + token: Some(token.clone()), + namespace: namespace.clone(), + pool_name: pool.clone(), + ..Default::default() + }, + shutdown.clone(), + )); + let client = rivetkit_client::Client::new( + rivetkit_client::ClientConfig::new(endpoint) + .token(&token) + .header("authorization", format!("Bearer {token}")) + .namespace(namespace) + .pool_name(pool), + ); + let actor = client.get_or_create( + NAME, + vec![uuid::Uuid::new_v4().to_string()], + Default::default(), + )?; + tokio::time::timeout(Duration::from_secs(30), async { + loop { + if actor.action(Ready::NAME, vec![]).await.is_ok() { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + }) + .await?; + let result = actor.action(Persist::NAME, vec![]).await?; + assert_eq!(result["streams"], 4096); + eprintln!("Real Engine maximum subscription persistence: {result}"); + shutdown.cancel(); + server.await??; + Ok(()) + } +} diff --git a/src/subscriptions/tests.rs b/src/subscriptions/tests.rs new file mode 100644 index 0000000..93423a3 --- /dev/null +++ b/src/subscriptions/tests.rs @@ -0,0 +1,356 @@ +use super::*; + +fn offset(n: usize) -> String { + format!("0000000000000000_{n:016}") +} +fn config() -> Config { + serde_json::from_value::(json!({"type":"pull-wake","pattern":"events/**","wake_stream":"wake/pool","lease_ttl_ms":1000})).unwrap().normalize().unwrap() +} +fn sub() -> Subscription { + Subscription::new( + "sub".into(), + "/v1/stream".into(), + "https://gameinc.io/v1/stream".into(), + config(), + 0, + 1000, + ) +} +fn entry(path: &str, n: usize) -> CatalogEntry { + CatalogEntry { + path: format!("/v1/stream/{path}"), + version: n as i64 + 1, + incarnation: "inc-1".into(), + tail: offset(n), + start: ZERO_OFFSET.into(), + exists: true, + wakes: vec![], + } +} +use crate::protocol::ZERO_OFFSET; + +#[test] +fn segment_globs_and_configuration_canonicalization() { + for (pattern, path, expected) in [ + ("events/*", "events/a", true), + ("events/*", "events/a/b", false), + ("events/**", "events", true), + ("events/**", "events/a/b", true), + ("**/a/**/b", "a/b", true), + ("**/a/**/b", "x/a/y/b", true), + ("**/a/**/b", "x/ab/b", false), + ] { + assert_eq!(glob_matches(pattern, path), expected, "{pattern} {path}"); + } + let mut a = config(); + a.streams = vec!["/events/b/".into(), "events/a".into(), "events/a".into()]; + let mut b = config(); + b.streams = vec!["events/a".into(), "events/b".into()]; + assert_eq!(a.normalize().unwrap().hash(), b.normalize().unwrap().hash()); + for ttl in [0, 999, 600001] { + let mut c = config(); + c.lease_ttl_ms = ttl; + assert!(c.normalize().is_err()); + } +} +#[test] +fn backfill_new_put_duplicate_observation_and_explicit_precedence() { + let mut s = sub(); + let first = entry("events/a", 5); + s.link("events/a", Some(&first), false, true); + assert!(!s.pending()); + s.link("events/a", Some(&entry("events/a", 8)), true, true); + assert_eq!(s.links["events/a"].acked, offset(5)); + assert_eq!(s.infos()[0].link_type, "explicit"); + s.links.get_mut("events/a").unwrap().explicit = false; + assert_eq!(s.infos()[0].link_type, "glob"); + assert!(s.observe(&entry("events/b", 3))); + assert!(s.pending()); + assert_eq!(s.links["events/b"].acked, ZERO_OFFSET); + assert!(s.ensure_wake(1000)); + let wake = s.wake.clone().unwrap(); + assert!(!s.observe(&entry("events/b", 3))); + assert!(!s.observe(&entry("unmatched/b", 3))); + assert!(!s.ensure_wake(1000)); + assert_eq!(s.wake.as_ref().unwrap().id, wake.id); +} +#[test] +fn snapshot_completion_never_acks_later_appends_or_recreated_streams() { + let mut s = sub(); + s.observe(&entry("events/a", 3)); + s.ensure_wake(1000); + s.observe(&entry("events/a", 7)); + assert!(s.finish(1001, true)); + assert_eq!(s.links["events/a"].acked, offset(3)); + assert_eq!(s.generation, 2); + let mut recreated = entry("events/a", 2); + recreated.incarnation = "inc-2".into(); + recreated.version = 99; + s.observe(&recreated); + assert!(s.finish(1002, true)); + assert_eq!(s.links["events/a"].acked, ZERO_OFFSET); +} +#[test] +fn acknowledgement_batch_is_atomic_and_duplicate_entries_cannot_regress() { + let mut s = sub(); + s.observe(&entry("events/a", 10)); + s.observe(&entry("events/b", 10)); + for acks in [ + vec![("events/a", offset(4)), ("unknown", offset(1))], + vec![("events/a", offset(4)), ("events/b", offset(11))], + vec![("events/a", offset(4)), ("events/a", offset(3))], + vec![("events/a", offset(4)), ("events/b", "now".into())], + ] { + let acks: Vec<_> = acks + .into_iter() + .map(|(stream, offset)| Ack { + stream: stream.into(), + offset, + }) + .collect(); + assert!(s.apply_acks(&acks).is_err()); + assert_eq!(s.links["events/a"].acked, ZERO_OFFSET); + } + s.apply_acks(&[Ack { + stream: "events/a".into(), + offset: offset(4), + }]) + .unwrap(); + assert!( + s.apply_acks(&[Ack { + stream: "events/a".into(), + offset: offset(2) + }]) + .is_err() + ); +} +#[test] +fn persisted_leases_retry_deadlines_and_fencing_survive_roundtrip() { + let mut s = sub(); + s.config.kind = DeliveryType::Webhook; + s.config.webhook = Some(Webhook { + url: "https://gameinc.io/hook".into(), + }); + s.config.wake_stream = None; + s.observe(&entry("events/a", 1)); + s.ensure_wake(1000); + let w = s.wake.as_mut().unwrap(); + w.deadline = Some(2000); + w.attempts = 5; + w.next_attempt_at = Some(40_000); + let old = w.id.clone(); + let serialized = serde_json::to_vec(&s).unwrap(); + let mut restored: Subscription = serde_json::from_slice(&serialized).unwrap(); + assert!(!restored.expire(1999)); + assert_eq!(restored.wake.as_ref().unwrap().id, old); + assert!(restored.expire(2000)); + let restored: Subscription = + serde_json::from_slice(&serde_json::to_vec(&restored).unwrap()).unwrap(); + let w = restored.wake.as_ref().unwrap(); + assert_ne!(w.id, old); + assert_eq!(w.attempts, 5); + assert_eq!(w.next_attempt_at, Some(40_000)); + assert_eq!(w.deadline, None); + assert_eq!(restored.due(), Some(40_000)); +} +#[test] +fn synthetic_wakes_cannot_recurse_or_skip_application_data() { + let mut s = sub(); + s.config.pattern = Some("**".into()); + let mut wake = entry("wake/pool", 2); + wake.wakes = vec![WakeInterval { + before: ZERO_OFFSET.into(), + after: offset(2), + incarnation: wake.incarnation.clone(), + origin: "/v1/stream/sub".into(), + }]; + s.observe(&wake); + assert!(!s.pending()); + assert_eq!(s.links["wake/pool"].acked, offset(2)); + s.observe(&entry("wake/pool", 5)); + assert!(s.pending()); + wake.tail = offset(8); + wake.version = 9; + wake.wakes = vec![WakeInterval { + before: offset(5), + after: offset(8), + incarnation: wake.incarnation.clone(), + origin: "/v1/stream/sub".into(), + }]; + s.observe(&wake); + assert_eq!(s.links["wake/pool"].acked, offset(2)); + s.apply_acks(&[Ack { + stream: "wake/pool".into(), + offset: offset(5), + }]) + .unwrap(); + assert_eq!(s.links["wake/pool"].acked, offset(8)); + assert!(!s.pending()); +} +#[tokio::test] +async fn destination_policy_covers_ipv4_ipv6_dns_and_local_exception() { + for ip in [ + "127.0.0.1", + "10.1.2.3", + "169.254.169.254", + "100.64.1.1", + "0.0.0.0", + "192.0.0.1", + "::", + "::1", + "::ffff:127.0.0.1", + "fc00::1", + "fe80::1", + "2001:db8::1", + "2002:a00:1::1", + ] { + assert!(!security::public_ip(ip.parse().unwrap()), "{ip}"); + } + for ip in ["1.1.1.1", "2606:4700:4700::1111"] { + assert!(security::public_ip(ip.parse().unwrap())); + } + assert!( + security::destination("http://localhost:9911/hook", true) + .await + .is_ok() + ); + assert!( + security::destination("https://localhost/hook", false) + .await + .is_err() + ); + assert!( + security::destination("http://10.0.0.1/hook", true) + .await + .is_err() + ); + assert!( + security::destination("http://[::1]:9911/hook", true) + .await + .is_ok() + ); + assert!( + security::destination("https://user:pass@gameinc.io/", false) + .await + .is_err() + ); +} +#[test] +fn retry_jitter_is_bounded_and_exponential() { + for (attempt, base) in [(1, 1000), (2, 2000), (6, 32000), (7, 60000), (100, 60000)] { + for _ in 0..100 { + let delay = retry_delay(attempt); + assert!((base * 80 / 100..=base * 120 / 100).contains(&delay)); + } + } +} + +#[test] +fn expired_slow_dispatch_preserves_failure_pacing_in_new_generation() { + let mut s = sub(); + s.config.kind = DeliveryType::Webhook; + s.config.webhook = Some(Webhook { + url: "https://gameinc.io/hook".into(), + }); + s.observe(&entry("events/a", 1)); + s.ensure_wake(1000); + s.wake.as_mut().unwrap().deadline = Some(2000); + s.wake.as_mut().unwrap().dispatch_until = Some(16000); + let prior = s.wake.as_ref().unwrap().id.clone(); + s.expire(2500); + let w = s.wake.as_ref().unwrap(); + assert_ne!(w.id, prior); + assert_eq!(w.attempts, 1); + assert!((3300..=3700).contains(&w.next_attempt_at.unwrap())); + assert_eq!(w.deadline, None); +} + +#[test] +fn current_catalog_reconciliation_retires_and_replaces_old_incarnation_wake() { + let mut s = sub(); + s.observe(&entry("events/a", 4)); + s.ensure_wake(1000); + let prior = s.wake.as_ref().unwrap().id.clone(); + let mut replacement = entry("events/a", 4); + replacement.version = 9; + replacement.incarnation = "new".into(); + assert!(s.observe(&replacement)); + assert!(!s.expire(1100)); + assert!(s.ensure_wake(1100)); + let mut s: Subscription = serde_json::from_slice(&serde_json::to_vec(&s).unwrap()).unwrap(); + assert_ne!(s.wake.as_ref().unwrap().id, prior); + assert_eq!(s.links["events/a"].acked, ZERO_OFFSET); + // An older queued fanout must not overwrite the reconciled version. + assert!(!s.observe(&entry("events/a", 2))); + assert_eq!(s.links["events/a"].incarnation, "new"); +} + +#[test] +fn expiry_and_catalog_deletion_remain_dirty_without_replacement_work() { + for deletion in [false, true] { + let mut s = sub(); + s.observe(&entry("events/a", 4)); + s.ensure_wake(1000); + s.wake.as_mut().unwrap().deadline = Some(2000); + if !deletion { + s.apply_acks(&[Ack { + stream: "events/a".into(), + offset: offset(4), + }]) + .unwrap(); + } + let mut persisted = serde_json::to_vec(&s).unwrap(); + let mut read: Subscription = serde_json::from_slice(&persisted).unwrap(); + let mut changed = if deletion { + let mut deleted = entry("events/a", 5); + deleted.exists = false; + read.observe(&deleted) + } else { + read.expire(2000) + }; + changed |= read.ensure_wake(2000); + // Read/claim preflight saves only changed state, including retirement + // when all work was acknowledged or its stream disappeared. + if changed { + persisted = serde_json::to_vec(&read).unwrap(); + } + let restored: Subscription = serde_json::from_slice(&persisted).unwrap(); + assert!(restored.wake.is_none(), "deletion={deletion}"); + assert!(!restored.pending()); + } +} + +#[test] +fn permanent_signing_failure_preserves_pending_data_until_management_retry() { + let mut s = sub(); + s.config.kind = DeliveryType::Webhook; + s.config.webhook = Some(Webhook { + url: "https://worker.gameinc.io/hook".into(), + }); + s.observe(&entry("events/a", 1)); + s.ensure_wake(1000); + let w = s.wake.as_mut().unwrap(); + w.attempts = 1; + w.next_attempt_at = None; + w.deadline = None; + w.token_expires = 2000; + w.delivery_error = + Some(json!({"code":"ENGINE_PAYLOAD_TOO_LARGE","status":413,"retry_at":null})); + let mut restored: Subscription = + serde_json::from_str(&serde_json::to_string(&s).unwrap()).unwrap(); + assert!(restored.expire(2001)); + assert!(restored.infos().iter().any(|s| s.has_pending)); + assert_eq!(restored.wake.as_ref().unwrap().next_attempt_at, None); + assert_eq!( + restored.public(Value::Null)["delivery_error"]["status"], + 413 + ); + let old_generation = restored.generation; + assert!(restored.resume_delivery(2002)); + assert!(!restored.resume_delivery(2002)); + let wake = restored.wake.as_ref().unwrap(); + assert!(wake.generation > old_generation); + assert_eq!(wake.next_attempt_at, Some(2002)); + assert!(wake.delivery_error.is_none()); + assert!(restored.infos().iter().any(|s| s.has_pending)); +} diff --git a/tests/conformance/.gitignore b/tests/conformance/.gitignore new file mode 100644 index 0000000..c2658d7 --- /dev/null +++ b/tests/conformance/.gitignore @@ -0,0 +1 @@ +node_modules/ diff --git a/tests/conformance/caller-auth.ts b/tests/conformance/caller-auth.ts new file mode 100644 index 0000000..b268ad7 --- /dev/null +++ b/tests/conformance/caller-auth.ts @@ -0,0 +1,15 @@ +// Exercise the public data boundary with a caller credential. Subscription +// management/callback tests keep their own credentials and negative cases. +const origin = new URL(process.env.CONFORMANCE_TEST_URL!).origin; +const caller = process.env.RIVET_TOKEN; +if (!caller) throw new Error("RIVET_TOKEN is required for authenticated stream conformance"); +const fetchWithoutCredentials = globalThis.fetch; +globalThis.fetch = (input, init) => { + const request = new Request(input, init); + const url = new URL(request.url); + if (url.origin === origin && url.pathname.startsWith("/v1/stream/") && + !url.pathname.startsWith("/v1/stream/__ds/") && !request.headers.has("authorization")) { + request.headers.set("authorization", `Bearer ${caller}`); + } + return fetchWithoutCredentials(request); +}; diff --git a/tests/conformance/conformance.test.ts b/tests/conformance/conformance.test.ts new file mode 100644 index 0000000..461d4b7 --- /dev/null +++ b/tests/conformance/conformance.test.ts @@ -0,0 +1,6 @@ +import { runConformanceTests } from "@durable-streams/server-conformance-tests"; + +const baseUrl = process.env.CONFORMANCE_TEST_URL; +if (!baseUrl) throw new Error("CONFORMANCE_TEST_URL is required"); + +runConformanceTests({ baseUrl, subscriptions: true }); diff --git a/tests/conformance/package-lock.json b/tests/conformance/package-lock.json new file mode 100644 index 0000000..bc32ad8 --- /dev/null +++ b/tests/conformance/package-lock.json @@ -0,0 +1,1570 @@ +{ + "name": "rivet-durable-streams-conformance", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "rivet-durable-streams-conformance", + "devDependencies": { + "@durable-streams/server-conformance-tests": "0.3.7", + "vitest": "4.0.18" + } + }, + "node_modules/@durable-streams/client": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/@durable-streams/client/-/client-0.2.7.tgz", + "integrity": "sha512-oxWbPEo8NO1Ruv9UgUVu3OswdE2z5ERhjccaujlpwe4tWuVd+Y/kdx+HtrkOnwN+bnRHTlxKYUNY8JTyrnhKew==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@microsoft/fetch-event-source": "^2.0.1" + }, + "bin": { + "intent": "bin/intent.js" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@durable-streams/server-conformance-tests": { + "version": "0.3.7", + "resolved": "https://registry.npmjs.org/@durable-streams/server-conformance-tests/-/server-conformance-tests-0.3.7.tgz", + "integrity": "sha512-1hf14EuaTgLEgIxa5LBLl7623N87RV5zccLFpfCQhCioI3pYRX07k9XyIzAUFIT42U693HD4FUF+Lz9W+GQXVA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@durable-streams/client": "0.2.7", + "fast-check": "^4.4.0", + "vitest": "^4.0.0" + }, + "bin": { + "durable-streams-server-conformance": "dist/cli.js", + "durable-streams-server-conformance-dev": "bin/conformance-dev.mjs", + "server-conformance-tests": "dist/cli.js" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", + "integrity": "sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.2.tgz", + "integrity": "sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.2.tgz", + "integrity": "sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.2.tgz", + "integrity": "sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.2.tgz", + "integrity": "sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.2.tgz", + "integrity": "sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.2.tgz", + "integrity": "sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.2.tgz", + "integrity": "sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.2.tgz", + "integrity": "sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.2.tgz", + "integrity": "sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.2.tgz", + "integrity": "sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.2.tgz", + "integrity": "sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.2.tgz", + "integrity": "sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.2.tgz", + "integrity": "sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.2.tgz", + "integrity": "sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.2.tgz", + "integrity": "sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.2.tgz", + "integrity": "sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.2.tgz", + "integrity": "sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.2.tgz", + "integrity": "sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.2.tgz", + "integrity": "sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.2.tgz", + "integrity": "sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.2.tgz", + "integrity": "sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.2.tgz", + "integrity": "sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.2.tgz", + "integrity": "sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.2.tgz", + "integrity": "sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.2.tgz", + "integrity": "sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@microsoft/fetch-event-source": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@microsoft/fetch-event-source/-/fetch-event-source-2.0.1.tgz", + "integrity": "sha512-W6CLUJ2eBMw3Rec70qrsEW0jOm/3twwJv21mrmj2yORiaVmVYGS4sSS5yUwvQc1ZlDLYGPnClVWmUUMagKNsfA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@napi-rs/lzma-linux-x64-gnu": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@napi-rs/lzma-linux-x64-gnu/-/lzma-linux-x64-gnu-1.5.1.tgz", + "integrity": "sha512-oTXEIha4SsuXdTA4Iyskj0kpdx2yVXdhd75c2v3xGrHFfVMsbhTPZU/nMPL4sWKo4pBHm3aucLaqGlF696dTyQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^22.20 || ^24.12 || >=25" + } + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.63.5", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.63.5.tgz", + "integrity": "sha512-J25QJU+B78T4FhhBsNpLJyVWOi31mwtpcMwywHmOKH65Q9IWGA81gPj+dnwlhU8wktVriYE+tFAaQgrnJRzAZg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.63.5", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.63.5.tgz", + "integrity": "sha512-LDopB3zuZM5Ux9TT2luNEBJW/tYbGU2g1d+VpKk6I+gSKDb+/7sYE6M225gRQt4RbMX6MSwMsVR/phdjVUgRLg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.63.5", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.63.5.tgz", + "integrity": "sha512-wlJEERGfeuHeBavCL2qVnNacOK43NDoZM4sjkeRPymd04OAE9T1zBqDJgmZ+CIsPTYKwdzpUC8vmOw84dwY4Tg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.63.5", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.63.5.tgz", + "integrity": "sha512-4nJJGg5jbo2wwPP4JP+LfEBA3bvP8rU9CLuhp7jWvq9sxEyhjQFTFdrqi+/dHEin/pd8jpT0vcehIpnZtmEdcQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.63.5", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.63.5.tgz", + "integrity": "sha512-DrZbyCDF1hneuO6jRbvZ2D7+PIBM6yIwYnJpg2vIk58T+wuFpiaGZrfUr59lDWw45bg+IrpTGLPiNi/Fk4w3Cg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.63.5", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.63.5.tgz", + "integrity": "sha512-gqfUVMJMB3mehqywxp6hTBFfgtMQykZY19+cfiaYP0toIJLb/1DZRJHVkQQGP13W4TAwfZDWeg1qBcheTRioXQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.63.5", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.63.5.tgz", + "integrity": "sha512-CFmhpvAwzSaWMlN3VN7UtmoTihlZNzoP0juQib5TQRnYUyDV8dXeWOp29sobWAT6gXl/hQgAClLlEiYozQG3OQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.63.5", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.63.5.tgz", + "integrity": "sha512-Uc9H8eXCOayV6JLTH5bXKMId6qbhNHa818/BgYjm4jrlq3vZquC9cqyvHBw17xy5Mnj5f+I3gFK5JcEf3hSqrw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.63.5", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.63.5.tgz", + "integrity": "sha512-VcPr/szv/1BFw112Kt//fxulXt/JPqzzidU84iW68L2DdjnOO8QFUv2zTSYBEPHD6movBD4z+bbr5y60GYM7Jw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.63.5", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.63.5.tgz", + "integrity": "sha512-BnxtJ5/91BrIHYIkGrmjz/lbMhqEHt1dPFqIxIFR+jPn0xVc/oUSCtIT089zfp5ufwGDlYz2UC+Fe1SRBpYFbQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.63.5", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.63.5.tgz", + "integrity": "sha512-LrYcHZwF+fAMNKHYTOQ5osWM4AZF7YF6D+XtsjDyEvljtt11twc+zHVXBLNEjxVSUnKYsOhvVz4Z213eW02COQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.63.5", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.63.5.tgz", + "integrity": "sha512-nj7QKQePAAUpCpJHtg0pR0W/b92A9NO17JS3BAQmHDn/yhmkir2p8llrKY9TOhleKIaSzy1JhxS3T9FVld6coA==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.63.5", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.63.5.tgz", + "integrity": "sha512-5ylkX6dWMeBKge9nTU+Rxfb+ZfaCIJ9lRqIFaK0eAMcWp7OJbYnLveLgXmm0VrvuLKb8qIK+mHyH0qu88RM+iA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.63.5", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.63.5.tgz", + "integrity": "sha512-oHK4ZHYFDKjZviK34I+NwgfbGxgI7ztrNxj2hPTSSNFgeq1a/lEd7dHV2fdGAuTH4Iym3RHJg+vAbWaWG4B7Zg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.63.5", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.63.5.tgz", + "integrity": "sha512-UcetmHZ6XOXuUByiKZyQmb55ZPr0LABr3Ec/HB9wKZn6CEAFWZkE+hsJErJ9hbPBC7nI0dKuELx7CoV6IM7TMg==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.63.5", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.63.5.tgz", + "integrity": "sha512-C5CmDPQBtvjVo8cgQsBs+w6WB0JLkiixhgi6hVLV11hERWdn/p0XcPU2OUcZzac9BPOFq7SbaHFa8r3SWEysCQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.63.5", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.63.5.tgz", + "integrity": "sha512-lHVQHJFKsuuxLMi3MQO9XVL8Tje3JR82CzB+QDKC5NWBcsIWuwsn9uIM5e3lBhI+fF1/s63qnyYqsg65+8rV/w==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.63.5", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.63.5.tgz", + "integrity": "sha512-3W9bTFcQNJn71cSJVM9RKIiZOy8DO/XLDii8Uv/Pm6WKqDRj7JV3ZfuXIEfyuy5LXpIzAbB/1M4Ukp9GKNa7nA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.63.5", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.63.5.tgz", + "integrity": "sha512-VDC7rRJlee/scpki96GZ27Omf6yU87s1YXwVTpjE5841faVlDYYT565rgfmoR1U0sqL7z5ivQSDjcsF6VRXyBA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.63.5", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.63.5.tgz", + "integrity": "sha512-z86Ok2p4pTdv5xqCKZsTooO7yBEiaJR/HzU3Wx8RmWsPoLppnMKROhJusQob8B3IE1ghC343kUW9rC2r+Wf3ig==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.63.5", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.63.5.tgz", + "integrity": "sha512-IzQmj+xXwQFGhMAMKMQVXkMwMZN3TqkJgAE0nSsqvVwWWciP4AIPMmWRqOQ2GfX7TUDZr+xqGFcBS36CRPGw0g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.63.5", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.63.5.tgz", + "integrity": "sha512-F6qpTaPc9bwBH85kjy0/BLmLSW1uv7AoOXCoRIkg2arlgCYlWYcAbiMkvZuAcaWk9TpCRG//okznLAqLGshkMw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.63.5", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.63.5.tgz", + "integrity": "sha512-igoDsTFhhwECBeGbUuLeIk7t8Y1apa+cs6mDWpx2EZ0ch7oEQgzHbFUXN9euoHekCAQzXdXApAGkV6jznS7tWw==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.63.5", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.63.5.tgz", + "integrity": "sha512-U3teMeMbXFmaM5D+OTJpsOXd+wV/qftIeYF9kBKL4v73641qyJmoXFtA28DQLsnmlyayEsTe72xpLHrArq6vHw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.63.5", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.63.5.tgz", + "integrity": "sha512-ypfC34F3RKXvCXBglGqGMsUSMKlgwd1HX9AOAlx9RoZZ6GaI42YHVeKpzg3JG+wpBUJYTG+NNZhqbDWL8tBZkw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@vitest/expect": { + "version": "4.0.18", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.0.18.tgz", + "integrity": "sha512-8sCWUyckXXYvx4opfzVY03EOiYVxyNrHS5QxX3DAIi5dpJAAkyJezHCP77VMX4HKA2LDT/Jpfo8i2r5BE3GnQQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.0.0", + "@types/chai": "^5.2.2", + "@vitest/spy": "4.0.18", + "@vitest/utils": "4.0.18", + "chai": "^6.2.1", + "tinyrainbow": "^3.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "4.0.18", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.0.18.tgz", + "integrity": "sha512-HhVd0MDnzzsgevnOWCBj5Otnzobjy5wLBe4EdeeFGv8luMsGcYqDuFRMcttKWZA5vVO8RFjexVovXvAM4JoJDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "4.0.18", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.21" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^6.0.0 || ^7.0.0-0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "4.0.18", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.0.18.tgz", + "integrity": "sha512-P24GK3GulZWC5tz87ux0m8OADrQIUVDPIjjj65vBXYG17ZeU3qD7r+MNZ1RNv4l8CGU2vtTRqixrOi9fYk/yKw==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^3.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "4.0.18", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.0.18.tgz", + "integrity": "sha512-rpk9y12PGa22Jg6g5M3UVVnTS7+zycIGk9ZNGN+m6tZHKQb7jrP7/77WfZy13Y/EUDd52NDsLRQhYKtv7XfPQw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "4.0.18", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "4.0.18", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.0.18.tgz", + "integrity": "sha512-PCiV0rcl7jKQjbgYqjtakly6T1uwv/5BQ9SwBLekVg/EaYeQFPiXcgrC2Y7vDMA8dM1SUEAEV82kgSQIlXNMvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.0.18", + "magic-string": "^0.30.21", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "4.0.18", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.0.18.tgz", + "integrity": "sha512-cbQt3PTSD7P2OARdVW3qWER5EGq7PHlvE+QfzSC0lbwO+xnt7+XH06ZzFjFRgzUX//JmpxrCu92VdwvEPlWSNw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "4.0.18", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.0.18.tgz", + "integrity": "sha512-msMRKLMVLWygpK3u2Hybgi4MNjcYJvwTb0Ru09+fOyCXIgT5raYP041DRRdiJiI3k/2U6SEbAETB3YtBrUkCFA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.0.18", + "tinyrainbow": "^3.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/chai": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", + "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/es-module-lexer": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz", + "integrity": "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==", + "dev": true, + "license": "MIT" + }, + "node_modules/esbuild": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", + "integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.2", + "@esbuild/android-arm": "0.28.2", + "@esbuild/android-arm64": "0.28.2", + "@esbuild/android-x64": "0.28.2", + "@esbuild/darwin-arm64": "0.28.2", + "@esbuild/darwin-x64": "0.28.2", + "@esbuild/freebsd-arm64": "0.28.2", + "@esbuild/freebsd-x64": "0.28.2", + "@esbuild/linux-arm": "0.28.2", + "@esbuild/linux-arm64": "0.28.2", + "@esbuild/linux-ia32": "0.28.2", + "@esbuild/linux-loong64": "0.28.2", + "@esbuild/linux-mips64el": "0.28.2", + "@esbuild/linux-ppc64": "0.28.2", + "@esbuild/linux-riscv64": "0.28.2", + "@esbuild/linux-s390x": "0.28.2", + "@esbuild/linux-x64": "0.28.2", + "@esbuild/netbsd-arm64": "0.28.2", + "@esbuild/netbsd-x64": "0.28.2", + "@esbuild/openbsd-arm64": "0.28.2", + "@esbuild/openbsd-x64": "0.28.2", + "@esbuild/openharmony-arm64": "0.28.2", + "@esbuild/sunos-x64": "0.28.2", + "@esbuild/win32-arm64": "0.28.2", + "@esbuild/win32-ia32": "0.28.2", + "@esbuild/win32-x64": "0.28.2" + } + }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/expect-type": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", + "integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/fast-check": { + "version": "4.10.2", + "resolved": "https://registry.npmjs.org/fast-check/-/fast-check-4.10.2.tgz", + "integrity": "sha512-iK2f+YrcmoeGqk6fA0ea2bptcu/itMIm4NfEozq6N25+aG6h7s5HZbB/k1aV7b5w5sFLMCbbtRUsTVR+BgC3xw==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT", + "dependencies": { + "pure-rand": "^8.0.0" + }, + "engines": { + "node": ">=12.17.0" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/nanoid": { + "version": "3.3.19", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.19.tgz", + "integrity": "sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/obug": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.2.1.tgz", + "integrity": "sha512-XrsrhT5sybtKI6wakr2SPOlGZWWYbUXZ7a0jT8/QOeAPau+1X/bSegNe5YR75oJmEZQbKningirmGOEJCIk61Q==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT", + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "dev": true, + "license": "MIT" + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "dev": true, + "license": "MIT", + "peer": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.28", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.28.tgz", + "integrity": "sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.18", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/pure-rand": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-8.4.2.tgz", + "integrity": "sha512-vvuOGgcuPJAirlHvuQw1TrOiw7ptaIXXmIbNuiNOY6lNGJJH49PQ1Kj4nd783nPdQhQdicgOjVI2yI/9BD6/Ng==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT" + }, + "node_modules/rollup": { + "version": "4.63.5", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.63.5.tgz", + "integrity": "sha512-KRWwmNLlPw5M7HcdYfm15oBv9n9LPtjzpzCIxS/phwqvPyxHSoKX6Y2YU3pxSPfy0CLquVgsx/j/hBi6OvH1Nw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.9" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@napi-rs/lzma-linux-x64-gnu": "1.5.1", + "@rollup/rollup-android-arm-eabi": "4.63.5", + "@rollup/rollup-android-arm64": "4.63.5", + "@rollup/rollup-darwin-arm64": "4.63.5", + "@rollup/rollup-darwin-x64": "4.63.5", + "@rollup/rollup-freebsd-arm64": "4.63.5", + "@rollup/rollup-freebsd-x64": "4.63.5", + "@rollup/rollup-linux-arm-gnueabihf": "4.63.5", + "@rollup/rollup-linux-arm-musleabihf": "4.63.5", + "@rollup/rollup-linux-arm64-gnu": "4.63.5", + "@rollup/rollup-linux-arm64-musl": "4.63.5", + "@rollup/rollup-linux-loong64-gnu": "4.63.5", + "@rollup/rollup-linux-loong64-musl": "4.63.5", + "@rollup/rollup-linux-ppc64-gnu": "4.63.5", + "@rollup/rollup-linux-ppc64-musl": "4.63.5", + "@rollup/rollup-linux-riscv64-gnu": "4.63.5", + "@rollup/rollup-linux-riscv64-musl": "4.63.5", + "@rollup/rollup-linux-s390x-gnu": "4.63.5", + "@rollup/rollup-linux-x64-gnu": "4.63.5", + "@rollup/rollup-linux-x64-musl": "4.63.5", + "@rollup/rollup-openbsd-x64": "4.63.5", + "@rollup/rollup-openharmony-arm64": "4.63.5", + "@rollup/rollup-win32-arm64-msvc": "4.63.5", + "@rollup/rollup-win32-ia32-msvc": "4.63.5", + "@rollup/rollup-win32-x64-gnu": "4.63.5", + "@rollup/rollup-win32-x64-msvc": "4.63.5", + "fsevents": "~2.3.2" + } + }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/std-env": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.10.0.tgz", + "integrity": "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyexec": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.3.1.tgz", + "integrity": "sha512-GCvB3aoys96IuDFBMcTB46JOR6mdMtAToqwiW8JlWhsoh1mhHi/xn9ss/Dg7N555GiJyEt2qzoG/NHCwM6h1EA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinyrainbow": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.1.tgz", + "integrity": "sha512-yau8yJdTt989Mm0Bd/236QnzEiPf2xLLTqUZRUJOo/3CB078LSwzei343DgtJVmfJKJE3TMINY1u42SQsP6mXw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/vite": { + "version": "7.3.6", + "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.6.tgz", + "integrity": "sha512-4XP60spRGjSZFf1qYH+dJIkK2znL3zQfl9KkOV9MkkRR/3Dls0dxaBsQPTloEc5BLXWPL9vsOxopxyKoMmDueg==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "esbuild": "^0.27.0 || ^0.28.0", + "fdir": "^6.5.0", + "picomatch": "^4.0.3", + "postcss": "^8.5.6", + "rollup": "^4.43.0", + "tinyglobby": "^0.2.15" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "lightningcss": "^1.21.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vitest": { + "version": "4.0.18", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.0.18.tgz", + "integrity": "sha512-hOQuK7h0FGKgBAas7v0mSAsnvrIgAvWmRFjmzpJ7SwFHH3g1k2u37JtYwOwmEKhK6ZO3v9ggDBBm0La1LCK4uQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/expect": "4.0.18", + "@vitest/mocker": "4.0.18", + "@vitest/pretty-format": "4.0.18", + "@vitest/runner": "4.0.18", + "@vitest/snapshot": "4.0.18", + "@vitest/spy": "4.0.18", + "@vitest/utils": "4.0.18", + "es-module-lexer": "^1.7.0", + "expect-type": "^1.2.2", + "magic-string": "^0.30.21", + "obug": "^2.1.1", + "pathe": "^2.0.3", + "picomatch": "^4.0.3", + "std-env": "^3.10.0", + "tinybench": "^2.9.0", + "tinyexec": "^1.0.2", + "tinyglobby": "^0.2.15", + "tinyrainbow": "^3.0.3", + "vite": "^6.0.0 || ^7.0.0", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^20.0.0 || ^22.0.0 || >=24.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@opentelemetry/api": "^1.9.0", + "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "4.0.18", + "@vitest/browser-preview": "4.0.18", + "@vitest/browser-webdriverio": "4.0.18", + "@vitest/ui": "4.0.18", + "happy-dom": "*", + "jsdom": "*" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + } + } + }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + } + } +} diff --git a/tests/conformance/package.json b/tests/conformance/package.json new file mode 100644 index 0000000..47429a6 --- /dev/null +++ b/tests/conformance/package.json @@ -0,0 +1,12 @@ +{ + "name": "rivet-durable-streams-conformance", + "private": true, + "type": "module", + "scripts": { + "test": "vitest run --config vitest.config.ts --reporter=verbose" + }, + "devDependencies": { + "@durable-streams/server-conformance-tests": "0.3.7", + "vitest": "4.0.18" + } +} diff --git a/tests/conformance/provision_signing.py b/tests/conformance/provision_signing.py new file mode 100644 index 0000000..38bd4a5 --- /dev/null +++ b/tests/conformance/provision_signing.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Provision one isolated Engine namespace credential; stdout is the secret token.""" +import json +import os +import sys +import urllib.parse +import urllib.request + + +def provision(endpoint, namespace, admin, application_tokens=True): + def request(path, body=None): + request = urllib.request.Request( + endpoint.rstrip("/") + path, + data=None if body is None else json.dumps(body).encode(), + headers={"authorization": "Bearer " + admin, "content-type": "application/json"}, + ) + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response) + + namespaces = request("/namespaces?name=" + urllib.parse.quote(namespace))["namespaces"] + if namespaces: + namespace_id = namespaces[0]["namespace_id"] + else: + namespace_id = request("/namespaces", {"name": namespace, "display_name": "DS conformance"})["namespace"]["namespace_id"] + grants = [ + {"resource": "webhook_signing", "target": "any", "operations": ["create"]}, + {"resource": "namespace", "target": "any", "operations": ["read"]}, + {"resource": "datacenter", "target": "any", "operations": ["list"]}, + *[{"resource": resource, "target": "any", "operations": ["read", "list", "create", "update", "delete"]} + for resource in ("actor", "runner", "runner_config", "actor_gateway", "actor_kv")], + {"resource": "actor_stream", "target": "any", "operations": ["read", "list", "create", "append", "close", "delete"]}, + {"resource": "durable_stream", "target": "any", "operations": ["read", "list", "create", "update", "append", "close", "delete"]}, + ] + if application_tokens: + grants.append({"resource": "application_token", "target": "any", "operations": ["create"]}) + return request("/auth/tokens", {"namespace": namespace, "duration": 86400, "grants": grants})["token"] + + +if __name__ == "__main__": + try: + print(provision(os.environ["RIVET_ENDPOINT"], os.environ["RIVET_NAMESPACE"], os.environ["CONFORMANCE_ENGINE_ADMIN_TOKEN"])) + except Exception as error: + # HTTP exception text omits response bodies and credentials. + print(f"Engine signing credential provisioning failed: {error}", file=sys.stderr) + sys.exit(1) diff --git a/tests/conformance/subscriptions.test.ts b/tests/conformance/subscriptions.test.ts new file mode 100644 index 0000000..078bf07 --- /dev/null +++ b/tests/conformance/subscriptions.test.ts @@ -0,0 +1,765 @@ +import { createPublicKey, randomUUID, verify } from "node:crypto"; +import { createServer, request as httpRequest } from "node:http"; +import type { AddressInfo } from "node:net"; +import { afterEach, describe, expect, test } from "vitest"; + +const origin = process.env.CONFORMANCE_TEST_URL; +if (!origin) throw new Error("CONFORMANCE_TEST_URL is required"); +const root = `${origin}/v1/stream`; +const streams: string[] = []; +const subscriptions: string[] = []; +const receivers: ReturnType[] = []; +const prefix = () => `subscription-extra/${randomUUID()}`; +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +type Link = { path: string; acked_offset: string; link_type: string }; +type Info = Link & { tail_offset: string; has_pending: boolean }; +type Wake = { wake_id: string; generation: number; token: string; streams: Info[] }; +type Notification = Wake & { callback_token: string; callback_url: string }; +type Config = Record; + +function request(path: string, method = "GET", body?: unknown, token?: string) { + return fetch(`${root}/${path}`, { + method, + headers: { + ...(body === undefined ? {} : { "content-type": "application/json" }), + ...(token ? { authorization: `Bearer ${token}` } : {}), + }, + body: body === undefined ? undefined : JSON.stringify(body), + signal: AbortSignal.timeout(15_000), + }); +} + +async function json(response: Response, status = 200): Promise { + expect(response.status, `Unexpected response from ${response.url}`).toBe(status); + return response.json(); +} + +async function engine(path: string, method = "GET", body?: unknown, token = process.env.RIVET_TOKEN, status = 200) { + const endpoint = process.env.RIVET_ENDPOINT; + if (!endpoint || !token) throw new Error("Engine signing tests require RIVET_ENDPOINT and RIVET_TOKEN from the conformance harness"); + return json(await fetch(`${endpoint}${path}`, { + method, headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, + body: body === undefined ? undefined : JSON.stringify(body), signal: AbortSignal.timeout(15_000), + }), status); +} + +async function poll(read: () => Promise, timeout = 12_000): Promise { + const deadline = Date.now() + timeout; + do { + const result = await read(); + if (result !== undefined) return result; + await sleep(40); + } while (Date.now() < deadline); + throw new Error("Timed out waiting for subscription state"); +} + +async function concurrent(values: T[], run: (value: T) => Promise): Promise { + const output: R[] = new Array(values.length); + let next = 0; + const workers = await Promise.allSettled(Array.from({ length: Math.min(8, values.length) }, async () => { + for (let index = next++; index < values.length; index = next++) output[index] = await run(values[index]); + })); + for (const result of workers) if (result.status === "rejected") throw result.reason; + return output; +} + +async function createStream(path: string, records: unknown[] = []) { + streams.push(path); + expect((await request(path, "PUT", records)).status).toBe(201); + return tail(path); +} + +async function tail(path: string) { + const response = await request(path, "HEAD"); + expect(response.status).toBe(200); + const offset = response.headers.get("stream-next-offset"); + expect(offset).toBeTruthy(); + return offset!; +} + +async function append(path: string, value: unknown) { + expect((await request(path, "POST", value)).status).toBe(204); + return tail(path); +} + +async function subscribe(config: Config, id = `extra-${randomUUID()}`) { + subscriptions.push(id); + await json(await request(`__ds/subscriptions/${id}`, "PUT", config), 201); + return id; +} + +async function pull(paths: string[], extra: Config = {}) { + const wake = `${prefix()}/wake`; + await createStream(wake); + const config = { type: "pull-wake", streams: paths, wake_stream: wake, lease_ttl_ms: 10_000, ...extra }; + const id = await subscribe(config); + return { id, wake, config }; +} + +async function readSubscription(id: string): Promise<{ streams: Link[]; status: string }> { + return json(await request(`__ds/subscriptions/${id}`)); +} + +async function claim(id: string, worker = "worker"): Promise { + return poll(async () => { + const response = await request(`__ds/subscriptions/${id}/claim`, "POST", { worker }); + if (response.status === 409) { + const body = await response.json(); + expect(body.error.code).toBe("NO_PENDING_WORK"); + return undefined; + } + return json(response); + }); +} + +function progress(id: string, wake: Wake, acks: { stream: string; offset: string }[] = [], done = true) { + return request(`__ds/subscriptions/${id}/ack`, "POST", { + wake_id: wake.wake_id, generation: wake.generation, acks, done, + }, wake.token); +} + +async function receiver(reply: (index: number, notification: Notification) => unknown | Promise = () => ({})) { + const notifications: Notification[] = []; + const server = createServer(async (req, res) => { + try { + const chunks = []; + for await (const chunk of req) chunks.push(chunk); + const body = JSON.parse(Buffer.concat(chunks).toString()); + const index = notifications.push(body) - 1; + const response: any = await reply(index, body); + res.writeHead(response?.status ?? 200, { "content-type": "application/json", ...response?.headers }); + res.end(JSON.stringify(response?.body ?? response ?? {})); + } catch { + res.writeHead(500).end(); + } + }); + receivers.push(server); + await new Promise((resolve) => server.listen(0, "0.0.0.0", resolve)); + return { + url: `http://127.0.0.1:${(server.address() as AddressInfo).port}/hook`, + notifications, + next: (index = 0) => poll(async () => notifications[index]), + }; +} + +afterEach(async () => { + await concurrent(subscriptions.splice(0), (id) => request(`__ds/subscriptions/${id}`, "DELETE").catch(() => undefined)); + await Promise.all(receivers.splice(0).map((server) => new Promise((resolve) => { + server.closeAllConnections(); + server.close(() => resolve()); + }))); + await concurrent(streams.splice(0), (path) => request(path, "DELETE").catch(() => undefined)); +}, 90_000); + +describe("Subscription durability and protocol edges", () => { + test("backfills existing streams at tail, respects single segments, and wakes on initial PUT", async () => { + const base = prefix(); + const old = `${base}/events/old`; + const oldTail = await createStream(old, [{ n: 0 }]); + const { id } = await pull([], { pattern: `${base}/events/*` }); + expect((await readSubscription(id)).streams).toContainEqual({ path: old, link_type: "glob", acked_offset: oldTail }); + const nested = `${base}/events/nested/child`; + await createStream(nested, [{ n: 1 }]); + const fresh = `${base}/events/new`; + const freshTail = await createStream(fresh, [{ n: 2 }]); + const claimed = await claim(id); + expect(claimed.streams.find((s) => s.path === old)?.has_pending).toBe(false); + expect(claimed.streams.find((s) => s.path === fresh)?.has_pending).toBe(true); + expect((await readSubscription(id)).streams.some((s) => s.path === nested)).toBe(false); + expect(await json(await progress(id, claimed, [{ stream: fresh, offset: freshTail }]))).toEqual({ ok: true, next_wake: false }); + const nextTail = await append(old, { n: 3 }); + const next = await claim(id); + expect(next.generation).toBeGreaterThan(claimed.generation); + await json(await progress(id, next, [{ stream: old, offset: nextTail }])); + }); + + test("double-star matches zero and multiple path segments", async () => { + const base = `${prefix()}/events`; + const { id } = await pull([], { pattern: `${base}/**` }); + await createStream(base, [{ n: 1 }]); + await createStream(`${base}/a/b`, [{ n: 2 }]); + await poll(async () => { + const paths = (await readSubscription(id)).streams.map((s) => s.path); + return paths.includes(base) && paths.includes(`${base}/a/b`) ? true : undefined; + }); + }); + + test("deep valid paths discover pattern subscriptions without database query limits", async () => { + const base = prefix(); + const source = `${base}/${Array(520).fill("s").join("/")}`; + expect(Buffer.byteLength(source)).toBeLessThan(2048); + const { id, wake } = await pull([], { pattern: `${base}/**` }); + const offset = await createStream(source, [{ depth: 520 }]); + await poll(async () => (await json(await request(wake))).some((event: any) => event.subscription_id === id) ? true : undefined); + const claimed = await claim(id); + expect(claimed.streams).toHaveLength(1); + expect(claimed.streams[0]).toMatchObject({ path: source, link_type: "glob", tail_offset: offset, has_pending: true }); + expect(await json(await request(`${source}?offset=${encodeURIComponent(claimed.streams[0].acked_offset)}`))).toEqual([{ depth: 520 }]); + expect(await json(await progress(id, claimed, [{ stream: source, offset }]))).toEqual({ ok: true, next_wake: false }); + }); + + test("explicit removal preserves glob cursor and reconfirmation preserves membership changes", async () => { + const base = prefix(); + const source = `${base}/source`; + await createStream(source); + const { id, config } = await pull([source, source], { pattern: `${base}/*` }); + const offset = await append(source, { n: 1 }); + const claimed = await claim(id); + await json(await progress(id, claimed, [{ stream: source, offset }], false)); + expect((await request(`__ds/subscriptions/${id}/streams/${encodeURIComponent(source)}`, "DELETE")).status).toBe(204); + expect((await readSubscription(id)).streams).toContainEqual({ path: source, link_type: "glob", acked_offset: offset }); + const extra = `${prefix()}/absent`; + expect((await request(`__ds/subscriptions/${id}/streams`, "POST", { streams: [source, extra] })).status).toBe(204); + await json(await request(`__ds/subscriptions/${id}`, "PUT", { ...config, streams: [source] })); + const links = (await readSubscription(id)).streams; + expect(links.find((s) => s.path === source)?.acked_offset).toBe(offset); + expect(links.some((s) => s.path === extra && s.link_type === "explicit")).toBe(true); + expect((await request(`__ds/subscriptions/${id}`, "PUT", { ...config, description: "different" })).status).toBe(409); + }); + + test("ack batches reject atomically and acknowledged offsets resume real reads", async () => { + const base = prefix(); + const a = `${base}/a`, b = `${base}/b`; + const beginning = await createStream(a); + await createStream(b); + const { id } = await pull([a, b]); + const first = await append(a, { n: 1 }); + const second = await append(b, { n: 2 }); + const claimed = await claim(id); + for (const bad of [ + { stream: `${base}/unknown`, offset: first }, + { stream: b, offset: "not-an-offset" }, + { stream: b, offset: "9999999999999999_9999999999999999" }, + { stream: a, offset: beginning }, + ]) { + const result = await json(await progress(id, claimed, [{ stream: a, offset: first }, bad], false), 409); + expect(result.error.code).toBe("INVALID_OFFSET"); + expect((await readSubscription(id)).streams.find((s) => s.path === a)?.acked_offset).toBe(beginning); + } + await json(await progress(id, claimed, [{ stream: a, offset: first }], false)); + expect((await progress(id, claimed, [{ stream: a, offset: beginning }], false)).status).toBe(409); + await json(await progress(id, claimed, [{ stream: b, offset: second }])); + await append(a, { n: 3 }); + expect(await json(await request(`${a}?offset=${encodeURIComponent(first)}`))).toEqual([{ n: 3 }]); + }); + + test("byte-stream acknowledgment resumes at the returned offset without replay or skipped bytes", async () => { + const source = `${prefix()}/bytes`; + streams.push(source); + expect((await fetch(`${root}/${source}`, { + method: "PUT", headers: { "content-type": "application/octet-stream" }, body: new Uint8Array(), + signal: AbortSignal.timeout(15_000), + })).status).toBe(201); + const { id } = await pull([source]); + const first = new Uint8Array([0, 255, 10, 195, 169]); + expect((await fetch(`${root}/${source}`, { + method: "POST", headers: { "content-type": "application/octet-stream" }, body: first, + signal: AbortSignal.timeout(15_000), + })).status).toBe(204); + const claimed = await claim(id); + const offset = claimed.streams.find((s) => s.path === source)!.tail_offset; + expect(new Uint8Array(await (await request(source)).arrayBuffer())).toEqual(first); + await json(await progress(id, claimed, [{ stream: source, offset }])); + const second = new Uint8Array([13, 0, 128, 254]); + expect((await fetch(`${root}/${source}`, { + method: "POST", headers: { "content-type": "application/octet-stream" }, body: second, + signal: AbortSignal.timeout(15_000), + })).status).toBe(204); + const resumed = await request(`${source}?offset=${encodeURIComponent(offset)}`); + expect(resumed.status).toBe(200); + expect(new Uint8Array(await resumed.arrayBuffer())).toEqual(second); + const current = await claim(id); + await json(await progress(id, current, [{ stream: source, offset: current.streams[0].tail_offset }])); + }); + + test("new forks start after inherited and sub-offset bytes, then wake on new appends", async () => { + const source = `${prefix()}/source`, base = `${prefix()}/forks`; + streams.push(source); + expect((await fetch(`${root}/${source}`, { + method: "PUT", headers: { "content-type": "text/plain" }, body: "hello", + signal: AbortSignal.timeout(15_000), + })).status).toBe(201); + const { id } = await pull([], { pattern: `${base}/*` }); + for (const [suffix, inherited, subOffset] of [["full", "hello", undefined], ["partial", "hel", "3"]] as const) { + const fork = `${base}/${suffix}`; + streams.push(fork); + expect((await fetch(`${root}/${fork}`, { + method: "PUT", + signal: AbortSignal.timeout(15_000), + headers: { + "content-type": "text/plain", "stream-forked-from": new URL(`${root}/${source}`).pathname, + ...(subOffset === undefined ? {} : { + "stream-fork-offset": "0000000000000000_0000000000000000", "stream-fork-sub-offset": subOffset, + }), + }, + })).status).toBe(201); + const inheritedTail = await tail(fork); + await poll(async () => (await readSubscription(id)).streams.find((s) => s.path === fork)?.acked_offset === inheritedTail ? true : undefined); + expect(await (await request(fork)).text()).toBe(inherited); + expect((await json(await request(`__ds/subscriptions/${id}/claim`, "POST", { worker: "historical" }), 409)).error.code).toBe("NO_PENDING_WORK"); + expect((await fetch(`${root}/${fork}`, { + method: "POST", headers: { "content-type": "text/plain" }, body: "!", + signal: AbortSignal.timeout(15_000), + })).status).toBe(204); + const claimed = await claim(id); + const link = claimed.streams.find((s) => s.path === fork)!; + expect(link).toMatchObject({ acked_offset: inheritedTail, has_pending: true }); + expect(await (await request(`${fork}?offset=${encodeURIComponent(inheritedTail)}`)).text()).toBe("!"); + expect(await json(await progress(id, claimed, [{ stream: fork, offset: link.tail_offset }]))).toEqual({ ok: true, next_wake: false }); + } + }); + + test("synchronous webhook completion acknowledges only its transmitted snapshot", async () => { + let finish!: () => void; + const held = new Promise((resolve) => { finish = resolve; }); + const hook = await receiver(async (index) => { + if (index === 0) await held; + return { done: true }; + }); + const source = `${prefix()}/source`; + await createStream(source); + const id = await subscribe({ type: "webhook", streams: [source], webhook: { url: hook.url }, lease_ttl_ms: 10_000 }); + const first = await append(source, { n: 1 }); + const wake1 = await hook.next(); + expect(wake1.streams.find((s) => s.path === source)?.tail_offset).toBe(first); + const second = await append(source, { n: 2 }); + finish(); + const wake2 = await hook.next(1); + expect(wake2.generation).toBeGreaterThan(wake1.generation); + expect(wake2.wake_id).not.toBe(wake1.wake_id); + expect(wake2.streams.find((s) => s.path === source)?.acked_offset).toBe(first); + expect(wake2.streams.find((s) => s.path === source)?.tail_offset).toBe(second); + await poll(async () => (await readSubscription(id)).streams.find((s) => s.path === source)?.acked_offset === second ? true : undefined); + }); + + test("late HTTP completion cannot acknowledge an expired wake's replacement", async () => { + let finish!: () => void; + const held = new Promise((resolve) => { finish = resolve; }); + const hook = await receiver(async (index) => { + if (index === 0) { await held; return { done: true }; } + return {}; + }); + const source = `${prefix()}/source`; + const beginning = await createStream(source); + const id = await subscribe({ type: "webhook", streams: [source], webhook: { url: hook.url }, lease_ttl_ms: 1000 }); + const offset = await append(source, { n: 1 }); + const old = await hook.next(); + const current = await hook.next(1); + expect(current.generation).toBeGreaterThan(old.generation); + finish(); + await sleep(150); + expect((await readSubscription(id)).streams.find((s) => s.path === source)?.acked_offset).toBe(beginning); + const stale = await json(await request(`__ds/subscriptions/${id}/callback`, "POST", { + wake_id: old.wake_id, generation: old.generation, done: true, + }, old.callback_token), 409); + expect(stale.error.code).toBe("FENCED"); + await json(await request(`__ds/subscriptions/${id}/callback`, "POST", { + wake_id: current.wake_id, generation: current.generation, acks: [{ stream: source, offset }], done: true, + }, current.callback_token)); + }); + + test("concurrent claims have one winner and heartbeats extend ownership", async () => { + const source = `${prefix()}/source`; + await createStream(source); + const lease = 3000; + const { id, wake } = await pull([source], { lease_ttl_ms: lease }); + await append(source, { n: 1 }); + await poll(async () => (await json(await request(wake))).some((event: any) => event.subscription_id === id) ? true : undefined); + const attempts = await Promise.all(Array.from({ length: 6 }, (_, i) => request(`__ds/subscriptions/${id}/claim`, "POST", { worker: `worker-${i}` }))); + expect(attempts.filter((r) => r.status === 200)).toHaveLength(1); + const winner = await json(attempts.find((r) => r.status === 200)!); + for (const loser of attempts.filter((r) => r.status !== 200)) { + const busy = await json(loser, 409); + expect(busy.error.code).toBe("ALREADY_CLAIMED"); + expect(busy.error.current_holder).toMatch(/^worker-/); + } + // Keep ownership past the original lease without assuming that six actor + // round trips finish within a fraction of a one-second lease under load. + const heartbeatUntil = Date.now() + lease + 200; + do { + await json(await progress(id, winner, [], false)); + await sleep(300); + } while (Date.now() < heartbeatUntil); + expect((await json(await request(`__ds/subscriptions/${id}/claim`, "POST", { worker: "too-early" }), 409)).error.code).toBe("ALREADY_CLAIMED"); + await sleep(lease + 200); + const replacement = await claim(id, "after-expiry"); + expect(replacement.generation).toBeGreaterThan(winner.generation); + expect((await json(await progress(id, winner), 409)).error.code).toBe("FENCED"); + expect((await json(await request(`__ds/subscriptions/${id}/release`, "POST", { + wake_id: winner.wake_id, generation: winner.generation, + }, winner.token), 409)).error.code).toBe("FENCED"); + }); + + test("claim capabilities reject tampering, other subscriptions and deleted incarnations", async () => { + const source = `${prefix()}/source`; + await createStream(source); + const first = await pull([source]); + const other = await pull([source]); + await append(source, { n: 1 }); + const old = await claim(first.id); + const currentOther = await claim(other.id); + const badToken = `${old.token.slice(0, -8)}${old.token.at(-8) === "A" ? "B" : "A"}${old.token.slice(-7)}`; + expect((await progress(first.id, { ...old, token: badToken })).status).toBe(401); + expect((await progress(other.id, { ...currentOther, token: old.token })).status).toBe(401); + expect((await request(`__ds/subscriptions/${first.id}`, "DELETE")).status).toBe(204); + expect((await progress(first.id, old)).status).toBe(409); + await subscribe(first.config, first.id); + await append(source, { n: 2 }); + const replacement = await claim(first.id); + expect(replacement.generation).toBeGreaterThan(old.generation); + expect((await json(await progress(first.id, { ...replacement, token: old.token }), 409)).error.code).toBe("FENCED"); + }); + + test("Engine application capabilities enforce signature, purpose, audience, tenant and namespace", async () => { + const source = `${prefix()}/source`; + const beginning = await createStream(source); + const { id } = await pull([source], { lease_ttl_ms: 60_000 }); + const offset = await append(source, { n: 1 }); + const claimed = await claim(id); + const [headerPart, payloadPart, signaturePart] = claimed.token.split("."); + const header = JSON.parse(Buffer.from(headerPart, "base64url").toString()); + const payload = JSON.parse(Buffer.from(payloadPart, "base64url").toString()); + expect(header).toMatchObject({ alg: "EdDSA", typ: "rivet-app+jwt" }); + expect(payload).toMatchObject({ rivet_ver: 1, aud: "durable-streams", rivet_purpose: "durable-streams.claim" }); + expect(payload.exp - payload.iat).toBeGreaterThan(0); + expect(payload.exp - payload.iat).toBeLessThanOrEqual(86_400); + const ds = payload.app.durable_streams; + expect(ds).toMatchObject({ root, subscription_id: id, wake_id: claimed.wake_id, generation: claimed.generation, operations: ["ack", "release"] }); + const key = (await engine("/.well-known/jwks.json")).keys.find((key: any) => key.kid === header.kid); + expect(verify(null, Buffer.from(`${headerPart}.${payloadPart}`), createPublicKey({ key, format: "jwk" }), Buffer.from(signaturePart, "base64url"))).toBe(true); + expect((await engine("/auth/tokens/inspect", "GET", undefined, claimed.token, 401)).code).toBe("invalid_token"); + + const issue = (claims = ds, audience = "durable-streams", purpose = "durable-streams.claim", token = process.env.RIVET_TOKEN) => + engine("/auth/application-tokens", "POST", { audience, purpose, duration: 86_400, claims: { durable_streams: claims } }, token); + for (const [claims, audience, purpose, status] of [ + [{ ...ds, root: `${root}/other` }, "durable-streams", "durable-streams.claim", 401], + [{ ...ds, tenant_scope: "other-tenant" }, "durable-streams", "durable-streams.claim", 401], + [ds, "other-audience", "durable-streams.claim", 401], + [ds, "durable-streams", "durable-streams.callback", 401], + [{ ...ds, operations: ["callback"] }, "durable-streams", "durable-streams.claim", 401], + [{ ...ds, generation: claimed.generation + 1 }, "durable-streams", "durable-streams.claim", 409], + ] as const) { + const issued = await issue(claims, audience, purpose); + expect((await progress(id, { ...claimed, token: issued.token })).status).toBe(status); + } + + const admin = process.env.CONFORMANCE_ENGINE_ADMIN_TOKEN; + if (!admin) throw new Error("Conformance Engine admin credential is required"); + const credential = async (namespace: string, duration: number) => { + return engine("/auth/tokens", "POST", { + namespace, duration, + grants: [{ resource: "application_token", target: "any", operations: ["create"] }], + }, admin); + }; + const otherNamespace = `ds-other-${randomUUID()}`; + await engine("/namespaces", "POST", { name: otherNamespace, display_name: "Other signing namespace" }, admin); + const other = await credential(otherNamespace, 60); + expect((await progress(id, { ...claimed, token: (await issue(ds, undefined, undefined, other.token)).token })).status).toBe(401); + const shortCaller = await credential(process.env.RIVET_NAMESPACE!, 3); + const short = await issue(ds, undefined, undefined, shortCaller.token); + expect(short.expires_ts).toBeLessThanOrEqual(shortCaller.expires_ts); + expect(short.expires_ts - short.issued_ts).toBeLessThanOrEqual(3000); + await sleep(Math.max(0, short.expires_ts - Date.now()) + 1100); + expect((await progress(id, { ...claimed, token: short.token })).status).toBe(401); + expect((await readSubscription(id)).streams[0].acked_offset).toBe(beginning); + await json(await progress(id, claimed, [{ stream: source, offset }])); + }); + + test("Engine operator rotation updates DS discovery without changing callback authority", async () => { + const admin = process.env.CONFORMANCE_ENGINE_ADMIN_TOKEN; + if (!admin) throw new Error("Conformance Engine admin credential is required"); + const source = `${prefix()}/source`; + await createStream(source); + const hook = await receiver(); + const id = await subscribe({ type: "webhook", streams: [source], webhook: { url: hook.url }, lease_ttl_ms: 60_000 }); + const offset = await append(source, { n: 1 }); + const notification = await hook.next(); + const sub = await json(await request(`__ds/subscriptions/${id}`)); + expect(sub.webhook.signing).toMatchObject({ alg: "ed25519", jwks_url: `${root}/__ds/jwks.json` }); + const metadata = await engine("/auth/webhooks/ensure", "POST"); + const management = `/namespaces/${encodeURIComponent(process.env.RIVET_NAMESPACE!)}/auth/webhook-keys`; + const before = await engine(management, "GET", undefined, admin); + expect(sub.webhook.signing.kid).toBe(before.active_kid); + const staged = await engine(`${management}/rotate`, "POST", { expected_generation: before.generation }, admin); + expect(staged.active_kid).toBe(before.active_kid); + expect(typeof staged.pending_kid).toBe("string"); + const published = await json(await request("__ds/jwks.json")); + expect(published.keys.some((key: any) => key.kid === staged.pending_kid)).toBe(true); + expect(published).toEqual(await engine(`/namespaces/${metadata.namespace_id}/auth/webhooks/jwks.json`)); + const revoked = await engine(`${management}/revoke`, "POST", { + request_id: randomUUID(), expected_generation: staged.generation, kids: [before.active_kid], + }, admin); + const current = await json(await request("__ds/jwks.json")); + expect(current.keys.some((key: any) => key.kid === revoked.active_kid)).toBe(true); + expect(current.keys.some((key: any) => key.kid === before.active_kid)).toBe(false); + await json(await request(`__ds/subscriptions/${id}/callback`, "POST", { + wake_id: notification.wake_id, generation: notification.generation, done: true, acks: [{ stream: source, offset }], + }, notification.callback_token)); + }); + + test("missing wake streams recover after the application creates them", async () => { + const base = prefix(); + const source = `${base}/source`, wake = `${base}/wake`; + const id = await subscribe({ type: "pull-wake", streams: [source], wake_stream: wake }); + await createStream(source, [{ n: 1 }]); + expect((await request(wake, "HEAD")).status).toBe(404); + await createStream(wake); + await poll(async () => { + const events = await json(await request(wake)); + return events.some((event: any) => event.subscription_id === id) ? true : undefined; + }); + expect((await claim(id)).streams.find((s) => s.path === source)?.has_pending).toBe(true); + }); + + test("explicit subscriptions recover across stream deletion and recreation", async () => { + const source = `${prefix()}/source`; + await createStream(source); + const { id } = await pull([source]); + const offset = await append(source, { n: 1 }); + const first = await claim(id); + await json(await progress(id, first, [{ stream: source, offset }])); + expect((await request(source, "DELETE")).status).toBe(204); + const recreated = await createStream(source, [{ n: 2 }]); + const second = await claim(id); + expect(second.streams.find((s) => s.path === source)?.has_pending).toBe(true); + expect(second.generation).toBeGreaterThan(first.generation); + await json(await progress(id, second, [{ stream: source, offset: recreated }])); + }); + + test.each(["pull-wake", "webhook"])("%s fences active capabilities when a stream is recreated at the same offset", async (type) => { + const source = `${prefix()}/source`; + await createStream(source); + const hook = type === "webhook" ? await receiver() : undefined; + const pulling = type === "pull-wake" ? await pull([source]) : undefined; + const id = pulling?.id ?? await subscribe({ type, streams: [source], webhook: { url: hook!.url }, lease_ttl_ms: 10_000 }); + const oldTail = await append(source, { n: 1 }); + const old: Wake | Notification = hook ? await hook.next() : await claim(id); + expect((await request(source, "DELETE")).status).toBe(204); + const newTail = await createStream(source, [{ n: 2 }]); + expect(newTail).toBe(oldTail); + if (pulling) { + await poll(async () => (await json(await request(pulling.wake))).some((event: any) => event.subscription_id === id && event.generation > old.generation) ? true : undefined); + } + const current = hook ? await hook.next(1) : await claim(id); + const action = hook ? "callback" : "ack"; + const token = (value: Wake | Notification) => hook ? (value as Notification).callback_token : value.token; + const stale = await json(await request(`__ds/subscriptions/${id}/${action}`, "POST", { + wake_id: old.wake_id, generation: old.generation, acks: [{ stream: source, offset: oldTail }], done: true, + }, token(old)), 409); + expect(stale.error.code).toBe("FENCED"); + await json(await request(`__ds/subscriptions/${id}/${action}`, "POST", { + wake_id: current.wake_id, generation: current.generation, acks: [{ stream: source, offset: newTail }], done: true, + }, token(current))); + }); + + test("large persisted memberships and wake snapshots survive database storage limits", async () => { + const base = prefix(); + const source = `${base}/source`; + await createStream(source); + const absent = Array.from({ length: 512 }, (_, index) => `${base}/missing-${index}-${"x".repeat(240)}`); + const { id } = await pull(absent); + expect((await request(`__ds/subscriptions/${id}/streams`, "POST", { streams: [source] })).status).toBe(204); + const subscription = await readSubscription(id); + expect(subscription.streams).toHaveLength(513); + expect(Buffer.byteLength(JSON.stringify(subscription))).toBeGreaterThan(128 * 1024); + const offset = await append(source, { n: 1 }); + const claimed = await claim(id); + expect(claimed.streams).toHaveLength(513); + expect(claimed.streams.find((s) => s.path === source)?.has_pending).toBe(true); + expect(await json(await progress(id, claimed, [{ stream: source, offset }]))).toEqual({ ok: true, next_wake: false }); + }); + + test("Unicode and space paths round-trip through encoded callback URLs and membership", async () => { + const source = `${prefix()}/café room/雪`; + const canonical = new URL(`${root}/${source}`).pathname.slice(new URL(root).pathname.length + 1); + const hook = await receiver(); + const id = await subscribe({ type: "webhook", streams: [source], webhook: { url: hook.url }, lease_ttl_ms: 10_000 }, `extra id ${randomUUID()}`); + const offset = await createStream(source, [{ n: 1 }]); + const notification = await hook.next(); + expect(notification.callback_url).toBe(`${root}/__ds/subscriptions/${encodeURIComponent(id)}/callback`); + expect(notification.streams.find((s) => s.path === canonical)?.tail_offset).toBe(offset); + expect(await json(await request(canonical))).toEqual([{ n: 1 }]); + const response = await fetch(notification.callback_url, { + method: "POST", + headers: { "content-type": "application/json", authorization: `Bearer ${notification.callback_token}` }, + body: JSON.stringify({ wake_id: notification.wake_id, generation: notification.generation, acks: [{ stream: source, offset }], done: true }), + signal: AbortSignal.timeout(15_000), + }); + expect(await json(response)).toEqual({ ok: true, next_wake: false }); + expect((await request(`__ds/subscriptions/${id}/streams/${encodeURIComponent(source)}`, "DELETE")).status).toBe(204); + expect((await readSubscription(id)).streams).toEqual([]); + }); + + test("encoded slash and literal star paths retain distinct stream identities", async () => { + const base = prefix(); + const encodedSlash = `${base}/a%2Fb`; + const nested = `${base}/a/b`; + const literalStar = `${base}/%2A`; + await createStream(encodedSlash, [{ kind: "encoded slash" }]); + await createStream(nested, [{ kind: "nested" }]); + await createStream(literalStar); + const { id } = await pull([encodedSlash, nested, literalStar]); + expect((await readSubscription(id)).streams.map((s) => s.path).sort()).toEqual([encodedSlash, nested, literalStar].sort()); + expect(await json(await request(encodedSlash))).toEqual([{ kind: "encoded slash" }]); + expect(await json(await request(nested))).toEqual([{ kind: "nested" }]); + const offset = await append(literalStar, { kind: "literal star" }); + const claimed = await claim(id); + expect(claimed.streams.find((s) => s.path === literalStar)?.has_pending).toBe(true); + expect(claimed.streams.filter((s) => s.path !== literalStar).every((s) => !s.has_pending)).toBe(true); + expect(await json(await progress(id, claimed, [{ stream: literalStar, offset }]))).toEqual({ ok: true, next_wake: false }); + expect((await request(`__ds/subscriptions/${id}/streams/${encodeURIComponent(encodedSlash)}`, "DELETE")).status).toBe(204); + expect((await readSubscription(id)).streams.map((s) => s.path).sort()).toEqual([nested, literalStar].sort()); + expect((await request(`__ds/subscriptions/${id}/streams/${encodeURIComponent(literalStar)}`, "DELETE")).status).toBe(204); + expect((await readSubscription(id)).streams.map((s) => s.path)).toEqual([nested]); + }); + + test("overlapping wake patterns suppress only their own notifications", async () => { + const base = prefix(); + const source = `${base}/source`, wake = `${base}/wake`; + await createStream(source); + const wakeBeginning = await createStream(wake); + const observer = await pull([wake]); + const id = await subscribe({ type: "pull-wake", pattern: `${base}/**`, wake_stream: wake }); + const offset = await append(source, { n: 1 }); + await poll(async () => (await json(await request(observer.wake))).some((event: any) => event.subscription_id === observer.id) ? true : undefined); + const claimed = await claim(id); + expect(await json(await progress(id, claimed, [{ stream: source, offset }]))).toEqual({ ok: true, next_wake: false }); + expect((await json(await request(`__ds/subscriptions/${id}/claim`, "POST", { worker: "already-done" }), 409)).error.code).toBe("NO_PENDING_WORK"); + const observed = await claim(observer.id); + const link = observed.streams.find((s) => s.path === wake)!; + expect(link.has_pending).toBe(true); + expect(link.acked_offset).toBe(wakeBeginning); + const events = await json(await request(wake)); + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ type: "wake", subscription_id: id }); + await json(await progress(observer.id, observed, [{ stream: wake, offset: link.tail_offset }])); + await sleep(300); + expect(await json(await request(wake))).toEqual(events); + }); + + test("fan-out beyond one page preserves notifications and current acknowledgment tails", async () => { + const base = prefix(); + const source = `${base}/source`, wake = `${base}/wake`; + await createStream(source); + await createStream(wake); + const ids = Array.from({ length: 70 }, (_, index) => `extra-fanout-${base.split("/").at(-1)}-${String(index).padStart(3, "0")}`); + await concurrent(ids, (id) => subscribe({ type: "pull-wake", streams: [source], wake_stream: wake }, id)); + await append(source, { n: 1 }); + const currentTail = await append(source, { n: 2 }); + const claimed = await claim(ids[69]); + expect(claimed.streams.find((s) => s.path === source)?.tail_offset).toBe(currentTail); + expect(await json(await progress(ids[69], claimed, [{ stream: source, offset: currentTail }]))).toEqual({ ok: true, next_wake: false }); + // This second high-sort subscription is left unclaimed until publication, + // proving the durable fan-out cursor traversed beyond the first 64 rows. + await poll(async () => (await json(await request(wake))).some((event: any) => event.subscription_id === ids[68]) ? true : undefined, 30_000); + const next = await claim(ids[68]); + expect(await json(await progress(ids[68], next, [{ stream: source, offset: currentTail }]))).toEqual({ ok: true, next_wake: false }); + }, 90_000); + + test("registration racing appends cannot skip a write made after registration returns", async () => { + const base = prefix(); + const source = `${base}/source`, wake = `${prefix()}/wake`; + await createStream(source, [{ phase: "historical" }]); + await createStream(wake); + const [id] = await Promise.all([ + subscribe({ type: "pull-wake", pattern: `${base}/*`, wake_stream: wake }), + append(source, { phase: "racing-a" }), + append(source, { phase: "racing-b" }), + ]); + const marker = randomUUID(); + const markerTail = await append(source, { marker }); + let covered = false; + for (let attempt = 0; attempt < 4; attempt++) { + const claimed = await claim(id); + const link = claimed.streams.find((s) => s.path === source)!; + expect(link.has_pending).toBe(true); + if (link.tail_offset === markerTail) { + expect(await json(await request(`${source}?offset=${encodeURIComponent(link.acked_offset)}`))).toContainEqual({ marker }); + covered = true; + } + await json(await progress(id, claimed, [{ stream: source, offset: link.tail_offset }])); + if (covered) break; + } + expect(covered).toBe(true); + }); + + test("rejects private IPv4, mapped IPv6, local IPv6 and credential-bearing destinations", async () => { + for (const url of [ + "http://10.0.0.1/hook", "https://10.0.0.1/hook", "https://169.254.169.254/hook", + "https://[fc00::1]/hook", "https://[fe80::1]/hook", "https://[::ffff:10.0.0.1]/hook", + "http://user:password@127.0.0.1/hook", + ]) { + const response = await request(`__ds/subscriptions/unsafe-${randomUUID()}`, "PUT", { + type: "webhook", streams: [`${prefix()}/source`], webhook: { url }, + }); + expect((await json(response, 400)).error.code).toBe("WEBHOOK_URL_REJECTED"); + } + }); + + test("webhook redirects are not followed and failed delivery is exposed", async () => { + const target = await receiver(); + const redirect = await receiver(() => ({ status: 302, headers: { location: target.url }, body: {} })); + const source = `${prefix()}/source`; + await createStream(source); + const id = await subscribe({ type: "webhook", streams: [source], webhook: { url: redirect.url }, lease_ttl_ms: 10_000 }); + await append(source, { n: 1 }); + await redirect.next(); + await poll(async () => (await readSubscription(id)).status === "failed" ? true : undefined); + expect(target.notifications).toHaveLength(0); + const jwksResponse = await request("__ds/jwks.json"); + expect(jwksResponse.headers.get("content-type")).toBe("application/jwk-set+json"); + expect(jwksResponse.headers.get("cache-control")).toBe("public, max-age=300"); + const jwks = await json(jwksResponse); + for (const key of jwks.keys) { + expect(Object.keys(key).sort()).toEqual(["alg", "crv", "kid", "kty", "use", "x"]); + } + }); + + test("reserves only the first __ds segment and enforces control methods", async () => { + expect((await request("__ds/unknown", "PUT", [{ n: 1 }])).status).toBe(404); + expect((await request("__ds/unknown")).status).toBe(404); + const source = `${prefix()}/__ds/ordinary`; + await createStream(source, [{ n: 1 }]); + expect(await json(await request(source))).toEqual([{ n: 1 }]); + const { id } = await pull([source]); + for (const action of ["claim", "ack", "callback", "release"]) { + expect((await request(`__ds/subscriptions/${id}/${action}`)).status).toBe(405); + } + }); + + test("rejects raw dot segments before reserved-route and mount normalization", async () => { + const target = `${prefix()}/escaped`; + const mount = new URL(root).pathname; + // fetch normalizes dot segments before sending; node:http preserves the + // request target so these exercise the server's routing boundary. + for (const path of [ + `${mount}/ordinary/../__ds/unknown`, + `${mount}/ordinary/%2e%2e/__ds/unknown`, + `${mount}/__ds/../${target}`, + `${mount}/__ds/.%2e/${target}`, + `${mount}/./${target}`, + `${mount}/%2e/${target}`, + `${mount}/../stream/${target}`, + `${mount}/%2E%2E/stream/${target}`, + ]) { + const status = await new Promise((resolve, reject) => { + const req = httpRequest(new URL(origin!), { + method: "PUT", path, headers: { "content-type": "application/json" }, timeout: 15_000, + }, (res) => { + res.resume(); + res.once("end", () => resolve(res.statusCode!)); + res.once("error", reject); + }); + req.once("error", reject); + req.once("timeout", () => req.destroy(new Error("Raw path request timed out"))); + req.end(JSON.stringify([{ escaped: true }])); + }); + expect(status, path).toBe(400); + } + expect((await request(target, "HEAD")).status).toBe(404); + expect((await request("__ds/unknown", "HEAD")).status).toBe(404); + await createStream(target, [{ ordinary: true }]); + expect(await json(await request(target))).toEqual([{ ordinary: true }]); + }); +}); diff --git a/tests/conformance/vitest.config.ts b/tests/conformance/vitest.config.ts new file mode 100644 index 0000000..541c824 --- /dev/null +++ b/tests/conformance/vitest.config.ts @@ -0,0 +1,13 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + setupFiles: ["./caller-auth.ts"], + include: ["conformance.test.ts", "subscriptions.test.ts"], + // Upstream TTL cases have short windows that are sensitive to test contention. + fileParallelism: false, + maxConcurrency: 1, + // Randomized cases perform hundreds of real-Engine storage operations. + testTimeout: 180_000, + }, +}); diff --git a/tests/e2e.rs b/tests/e2e.rs index 8c23b5b..5c8629f 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -1,25 +1,77 @@ +use std::time::Duration; + use bytes::Bytes; use http::{HeaderMap, HeaderValue, Method, StatusCode}; use rivet_durable_streams::{ACTOR_NAME, register}; -use rivetkit::Registry; -use rivetkit_client::GetOrCreateOptions; +use rivetkit::{Registry, ServeConfig}; +use rivetkit_client::{Client, ClientConfig, GetOrCreateOptions}; use serde_json::json; +use sha2::{Digest, Sha256}; +use tokio_util::sync::CancellationToken; #[tokio::test] #[ignore = "requires a matching local Rivet Engine"] async fn create_append_read_sse_and_delete() -> anyhow::Result<()> { let mut registry = Registry::new(); register(&mut registry); - let runtime = rivetkit::test::setup(registry).await?; - let actor = runtime.client().get_or_create( + let shutdown = CancellationToken::new(); + let _shutdown_guard = shutdown.clone().drop_guard(); + let mut serve = None; + let mut runtime = None; + let client = if let Ok(endpoint) = std::env::var("RIVET_TEST_ENDPOINT") { + let pool = format!("durable-streams-e2e-{}", uuid::Uuid::new_v4()); + let token = std::env::var("RIVET_TOKEN").unwrap_or_else(|_| "dev".into()); + let namespace = std::env::var("RIVET_NAMESPACE").unwrap_or_else(|_| "default".into()); + let config = ServeConfig { + endpoint: endpoint.clone(), + token: Some(token.clone()), + namespace: namespace.clone(), + pool_name: pool.clone(), + ..ServeConfig::default() + }; + serve = Some(tokio::spawn( + registry.serve_with_config(config, shutdown.clone()), + )); + Client::new( + ClientConfig::new(endpoint) + .token(&token) + .header("authorization", format!("Bearer {token}")) + .namespace(namespace) + .pool_name(pool), + ) + } else { + runtime = Some(rivetkit::test::setup(registry).await?); + runtime.as_ref().unwrap().client().clone() + }; + let path = format!("/integration/{}", uuid::Uuid::new_v4()); + let mut hash = Sha256::new(); + hash.update(b"test\0"); + hash.update(path.as_bytes()); + let actor = client.get_or_create( ACTOR_NAME, - vec![format!("durable-stream-{}", uuid::Uuid::new_v4())], + vec![format!("{:x}", hash.finalize())], GetOrCreateOptions { - create_with_input: Some(json!({ "path": "/integration", "tenant_scope": "test" })), + create_with_input: Some(json!({ "path": path, "tenant_scope": "test" })), ..GetOrCreateOptions::default() }, )?; + // The public registry API has no readiness observer. A successful HEAD + // proves the new pool and actor are ready without retrying mutations. + tokio::time::timeout(Duration::from_secs(30), async { + loop { + if let Ok(response) = actor + .fetch("protocol", Method::HEAD, HeaderMap::new(), None) + .await + && response.status() == StatusCode::NOT_FOUND + { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + }) + .await?; + let response = actor .fetch( "protocol", @@ -66,8 +118,15 @@ async fn create_append_read_sse_and_delete() -> anyhow::Result<()> { let body = response.text().await?; assert!(body.contains("event: data\n")); assert!(body.contains(r#"data:[{"n":1},{"n":2}]"#)); - assert!(body.contains("event: control\n")); - assert!(body.contains(r#"\"streamClosed\":true"#)); + let control = body + .split("\n\n") + .find(|frame| frame.starts_with("event: control\n")) + .and_then(|frame| frame.lines().find_map(|line| line.strip_prefix("data:"))) + .expect("SSE control frame"); + assert_eq!( + serde_json::from_str::(control)?["streamClosed"], + true + ); let response = actor .fetch("protocol", Method::DELETE, HeaderMap::new(), None) @@ -78,7 +137,13 @@ async fn create_append_read_sse_and_delete() -> anyhow::Result<()> { .await?; assert_eq!(response.status(), StatusCode::NOT_FOUND); - runtime.shutdown().await; + shutdown.cancel(); + if let Some(serve) = serve { + tokio::time::timeout(Duration::from_secs(30), serve).await???; + } + if let Some(runtime) = runtime { + runtime.shutdown().await; + } Ok(()) } diff --git a/tests/subscription_recovery.py b/tests/subscription_recovery.py new file mode 100644 index 0000000..29dbaa2 --- /dev/null +++ b/tests/subscription_recovery.py @@ -0,0 +1,456 @@ +#!/usr/bin/env python3 +"""Real-Engine restart/fault tests. Requires the fault-injection binary, never a test HTTP endpoint.""" +import argparse +import base64 +import concurrent.futures +import http.server +import json +import os +import pathlib +import queue +import signal +import socket +import subprocess +import tempfile +import threading +import time +import urllib.error +import urllib.request +import uuid +from conformance.provision_signing import provision + +DEFAULT_TOKEN = object() + + +def free_port(): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +class Receiver: + def __init__(self): + self.requests = queue.Queue() + self.status = 200 + self.done = False + owner = self + + class Handler(http.server.BaseHTTPRequestHandler): + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["content-length"]))) + owner.requests.put((time.monotonic(), body)) + self.send_response(owner.status) + self.send_header("content-type", "application/json") + self.end_headers() + self.wfile.write(json.dumps({"done": owner.done}).encode()) + + def log_message(self, *_args): + pass + + self.server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=self.server.serve_forever, daemon=True).start() + self.url = f"http://127.0.0.1:{self.server.server_port}/hook" + + def next(self, timeout=25): + return self.requests.get(timeout=timeout) + + def close(self): + self.server.shutdown() + self.server.server_close() + + +class Server: + def __init__(self, binary, engine, logs): + self.binary = str(pathlib.Path(binary).resolve()) + self.engine = engine + self.namespace = os.environ.get("RIVET_NAMESPACE", "default") + self.engine_token = os.environ["RIVET_TOKEN"] + self.logs = pathlib.Path(logs) + self.port = free_port() + self.base = f"http://127.0.0.1:{self.port}/v1/stream" + self.tenant = "recovery-" + uuid.uuid4().hex + self.pool = "subscription-recovery-" + uuid.uuid4().hex + self.process = None + self.starts = 0 + self.marker = self.logs / "checkpoint" + self.release_marker = self.logs / "checkpoint.release" + + def start(self, fault=None, pause=None): + self.starts += 1 + self.marker.unlink(missing_ok=True) + self.release_marker.unlink(missing_ok=True) + env = dict(os.environ, RIVET_ENDPOINT=self.engine, RIVET_NAMESPACE=self.namespace, RIVET_TOKEN=self.engine_token, + RIVET_POOL_NAME=self.pool, DS_TENANT_SCOPE=self.tenant, + DS_SUBSCRIPTIONS_MANAGEMENT_TOKEN="recovery-test-management", + DS_SUBSCRIPTIONS_ALLOW_UNAUTHENTICATED="0", DS_SUBSCRIPTIONS_ALLOW_LOCAL_HTTP="1", + PUBLIC_ORIGIN=f"http://127.0.0.1:{self.port}", RUST_LOG="warn") + env.pop("DS_SUBSCRIPTION_FAULT", None) + env.pop("DS_SUBSCRIPTION_PAUSE_POINT", None) + env.pop("DS_SUBSCRIPTION_PAUSE_MARKER", None) + if fault: + env.update(DS_SUBSCRIPTION_FAULT=fault, DS_SUBSCRIPTION_FAULT_MARKER=str(self.marker)) + if pause: + env.update(DS_SUBSCRIPTION_PAUSE_POINT=pause, DS_SUBSCRIPTION_PAUSE_MARKER=str(self.marker)) + with open(self.logs / f"server-{self.starts}.log", "wb") as log: + self.process = subprocess.Popen([self.binary, "--host", "0.0.0.0", "--port", str(self.port)], + env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) + self.poll(lambda: self.request("GET", "ready", expected=(404,))[0] == 404, timeout=45) + self.poll(lambda: self.request("GET", "__ds/jwks.json", token=None)[0] == 200, timeout=60) + + def stop(self): + if self.process and self.process.poll() is None: + os.killpg(self.process.pid, signal.SIGINT) + try: + self.process.wait(timeout=10) + except subprocess.TimeoutExpired: + os.killpg(self.process.pid, signal.SIGKILL) + self.process.wait(timeout=10) + + def restart(self, fault=None, pause=None): + self.stop() + self.start(fault, pause) + + def request(self, method, path, body=None, token=DEFAULT_TOKEN, expected=(200, 201, 204)): + data = None if body is None else json.dumps(body).encode() + headers = {"content-type": "application/json"} + if token is DEFAULT_TOKEN: + token = "recovery-test-management" if path.startswith("__ds/") else self.engine_token + if token: + headers["authorization"] = "Bearer " + token + url = path if path.startswith("http") else self.base + "/" + path + request = urllib.request.Request(url, method=method, data=data, headers=headers) + try: + response = urllib.request.urlopen(request, timeout=35) + except urllib.error.HTTPError as error: + response = error + raw = response.read() + try: + value = json.loads(raw) if raw else None + except json.JSONDecodeError: + value = raw.decode(errors="replace") + error_code = value.get("error", {}).get("code") if isinstance(value, dict) and isinstance(value.get("error"), dict) else None + assert response.status in expected, (method, path, response.status, error_code) + return response.status, value + + @staticmethod + def poll(condition, timeout=25): + end = time.monotonic() + timeout + last = None + while time.monotonic() < end: + try: + result = condition() + if result: + return result + except (OSError, AssertionError) as error: + last = error + time.sleep(0.1) + raise AssertionError(f"Polling timed out: {last}") + + def sub(self, name, stream, receiver=None, lease=30000): + config = {"type": "webhook" if receiver else "pull-wake", "streams": [stream], "lease_ttl_ms": lease} + if receiver: + config["webhook"] = {"url": receiver.url} + else: + config["wake_stream"] = "wake/" + name + self.request("PUT", config["wake_stream"], []) + self.request("PUT", "__ds/subscriptions/" + name, config) + + def crash_request(self, checkpoint, method, path, body): + self.poll(lambda: self.request("HEAD", path)[0] == 200, timeout=60) + with concurrent.futures.ThreadPoolExecutor() as executor: + future = executor.submit(self.request, method, path, body) + self.poll(lambda: self.marker.exists(), timeout=30) + assert self.marker.read_text() == checkpoint + try: + future.result(timeout=40) + except (OSError, AssertionError): + pass + # The faulted actor can exit separately from the HTTP facade. Once the + # checkpoint is confirmed, stop any surviving facade for a cold restart. + if self.process.poll() is None: + os.killpg(self.process.pid, signal.SIGKILL) + self.process.wait(timeout=10) + self.start() + + +def run(binary, engine, logs): + s = Server(binary, engine, logs) + receiver = Receiver() + try: + s.start() + # Public management is denied without the explicitly configured credential. + s.request("PUT", "__ds/subscriptions/unauthorized", {}, token=None, expected=(401,)) + s.sub("append-crash", "events/append-crash", receiver) + # Explicit operations and configuration conflicts must not depend on + # unrelated runner pools (the preceding Rust e2e pool has stopped). + s.request("POST", "__ds/subscriptions/append-crash/streams", {"streams": ["events/append-crash"]}) + s.request("PUT", "__ds/subscriptions/append-crash", { + "type": "webhook", "pattern": "events/**", "webhook": {"url": receiver.url}, + }, expected=(409,)) + s.request("PUT", "events/append-crash", []) + keys = s.request("GET", "__ds/jwks.json", token=None)[1] + s.restart("after_stream_commit") + s.crash_request("after_stream_commit", "POST", "events/append-crash", {"after": "commit"}) + _, wake = receiver.next() + assert wake["subscription_id"] == "append-crash" + assert keys == s.request("GET", "__ds/jwks.json", token=None)[1] + ack = {"wake_id": wake["wake_id"], "generation": wake["generation"], "done": True, + "acks": [{"stream": "events/append-crash", "offset": wake["streams"][0]["tail_offset"]}]} + s.request("POST", wake["callback_url"], ack, token=wake["callback_token"]) + print("PASS restart after stream commit before notification; signing key persisted", flush=True) + s.request("DELETE", "__ds/subscriptions/append-crash") + + s.sub("wake-crash", "events/wake-crash", receiver) + s.request("PUT", "events/wake-crash", []) + s.restart("after_wake_persist") + s.crash_request("after_wake_persist", "POST", "events/wake-crash", {"n": 1}) + # The persisted 60-second dispatch reservation fences an attempt whose + # process disappeared. Recovery must wait for it without a control call. + _, wake = receiver.next(timeout=75) + assert wake["subscription_id"] == "wake-crash" + s.request("POST", wake["callback_url"], {"wake_id": wake["wake_id"], "generation": wake["generation"], "done": True, + "acks": [{"stream": "events/wake-crash", "offset": wake["streams"][0]["tail_offset"]}]}, token=wake["callback_token"]) + print("PASS restart with persisted webhook wake before delivery", flush=True) + s.request("DELETE", "__ds/subscriptions/wake-crash") + + s.sub("pull-crash", "events/pull-crash") + s.request("PUT", "events/pull-crash", []) + s.restart("after_wake_append_commit") + s.crash_request("after_wake_append_commit", "POST", "events/pull-crash", {"n": 1}) + events = s.poll(lambda: s.request("GET", "wake/pull-crash")[1]) + assert len(events) == 1, events + claimed = s.request("POST", "__ds/subscriptions/pull-crash/claim", {"worker": "first"})[1] + s.restart() + busy = s.request("POST", "__ds/subscriptions/pull-crash/claim", {"worker": "second"}, expected=(409,))[1] + assert busy["error"]["code"] == "ALREADY_CLAIMED" + s.request("POST", "__ds/subscriptions/pull-crash/ack", {"wake_id": claimed["wake_id"], "generation": claimed["generation"], "done": True, + "acks": [{"stream": "events/pull-crash", "offset": claimed["streams"][0]["tail_offset"]}]}, token=claimed["token"]) + assert len(s.request("GET", "wake/pull-crash")[1]) == 1 + print("PASS restart after pull wake append before completion; deduplication and active claim survive", flush=True) + s.request("DELETE", "__ds/subscriptions/pull-crash") + + s.sub("expired-claim", "events/expired-claim", lease=3000) + s.request("PUT", "events/expired-claim", {"n": 1}) + claimed = s.poll(lambda: s.request("POST", "__ds/subscriptions/expired-claim/claim", {"worker": "old"})[1]) + acknowledged = claimed["streams"][0]["tail_offset"] + s.request("POST", "__ds/subscriptions/expired-claim/ack", { + "wake_id": claimed["wake_id"], "generation": claimed["generation"], + "acks": [{"stream": "events/expired-claim", "offset": acknowledged}], + }, token=claimed["token"]) + s.request("POST", "events/expired-claim", {"n": 2}) + # Stop immediately so graceful draining cannot process lease expiry; + # the persisted deadline must be enforced by the restarted owner. + os.killpg(s.process.pid, signal.SIGKILL) + s.process.wait(timeout=10) + time.sleep(3.1) + s.start() + stale = s.request("POST", "__ds/subscriptions/expired-claim/ack", { + "wake_id": claimed["wake_id"], "generation": claimed["generation"], "done": True, + }, token=claimed["token"], expected=(409,))[1] + assert stale["error"]["code"] == "FENCED" + current = s.poll(lambda: s.request("POST", "__ds/subscriptions/expired-claim/claim", {"worker": "new"})[1]) + assert current["generation"] > claimed["generation"] and current["wake_id"] != claimed["wake_id"] + assert current["streams"][0]["acked_offset"] == acknowledged + assert current["streams"][0]["has_pending"] + s.request("POST", "__ds/subscriptions/expired-claim/ack", { + "wake_id": current["wake_id"], "generation": current["generation"], "done": True, + "acks": [{"stream": "events/expired-claim", "offset": current["streams"][0]["tail_offset"]}], + }, token=current["token"]) + print("PASS persisted claim expires while offline; replacement fences old token and retains cursor", flush=True) + s.request("DELETE", "__ds/subscriptions/expired-claim") + + receiver.status = 503 + s.sub("retry", "events/retry", receiver, lease=600000) + s.request("PUT", "events/retry", {"n": 1}) + first_time, first = receiver.next() + second_time, second = receiver.next() + third_time, third = receiver.next() + assert first["wake_id"] == second["wake_id"] == third["wake_id"] + assert second_time - first_time >= .7 and third_time - second_time >= 1.4 + s.poll(lambda: s.request("GET", "__ds/subscriptions/retry")[1]["status"] == "failed") + s.restart() + receiver.status, receiver.done = 200, True + fourth_time, fourth = receiver.next() + assert fourth["wake_id"] == third["wake_id"] + assert fourth_time - third_time >= 3.0, "Restart reset retry backoff" + s.poll(lambda: s.request("GET", "__ds/subscriptions/retry")[1]["streams"][0]["acked_offset"] == fourth["streams"][0]["tail_offset"]) + print("PASS failed delivery retry count/deadline and wake identity survive restart", flush=True) + s.request("DELETE", "__ds/subscriptions/retry") + + # Missing signing grants are permanent failures. Restoring credentials + # alone must not silently retry; the operator explicitly re-confirms. + normal_token = s.engine_token + s.stop() + s.engine_token = provision(engine, s.namespace, os.environ["CONFORMANCE_ENGINE_ADMIN_TOKEN"], application_tokens=False) + s.start() + receiver.done = False + s.sub("signing-denied", "events/signing-denied", receiver) + s.request("PUT", "events/signing-denied", {"n": 1}) + + def signing_failed(): + subscription = s.request("GET", "__ds/subscriptions/signing-denied")[1] + error = subscription.get("delivery_error") or {} + return subscription if subscription["status"] == "failed" and error.get("status") == 403 else None + + failed = s.poll(signing_failed) + assert failed["delivery_error"]["retry_at"] is None + assert receiver.requests.empty(), "Denied signer delivered an unsigned webhook" + s.engine_token = normal_token + s.restart() + time.sleep(1.2) + assert signing_failed(), "Restart cleared the persisted signing failure" + assert receiver.requests.empty(), "Corrected credential retried without operator re-confirmation" + s.sub("signing-denied", "events/signing-denied", receiver) + _, delivered = receiver.next() + s.request("POST", delivered["callback_url"], { + "wake_id": delivered["wake_id"], "generation": delivered["generation"], "done": True, + "acks": [{"stream": "events/signing-denied", "offset": delivered["streams"][0]["tail_offset"]}], + }, token=delivered["callback_token"]) + assert s.request("GET", "__ds/subscriptions/signing-denied")[1]["delivery_error"] is None + print("PASS permanent Engine403 persists across restart; corrected credentials resume on same-config PUT", flush=True) + s.request("DELETE", "__ds/subscriptions/signing-denied") + + s.restart(pause="after_application_issuance") + s.sub("claim-race", "events/claim-race", lease=60000) + s.request("PUT", "events/claim-race", {"original": True}) + original_event = s.poll(lambda: s.request("GET", "wake/claim-race")[1])[0] + with concurrent.futures.ThreadPoolExecutor() as executor: + claiming = executor.submit(s.request, "POST", "__ds/subscriptions/claim-race/claim", + {"worker": "before-recreation"}, expected=(409,)) + s.poll(lambda: s.marker.exists(), timeout=8) + assert s.marker.read_text() == "after_application_issuance" + s.request("DELETE", "events/claim-race") + s.request("PUT", "events/claim-race", {"recreated": True}) + # Ordinary wake-stream reads establish that catalog fanout observed + # the new incarnation without a subscription control operation. + replacement_event = s.poll(lambda: next((event for event in s.request("GET", "wake/claim-race")[1] + if event["generation"] > original_event["generation"]), None), timeout=5) + s.release_marker.write_text("release") + assert claiming.result(timeout=15)[1]["error"]["code"] == "FENCED" + claimed = s.request("POST", "__ds/subscriptions/claim-race/claim", {"worker": "after-recreation"})[1] + assert claimed["generation"] == replacement_event["generation"] + s.request("POST", "__ds/subscriptions/claim-race/ack", { + "wake_id": claimed["wake_id"], "generation": claimed["generation"], "done": True, + "acks": [{"stream": "events/claim-race", "offset": claimed["streams"][0]["tail_offset"]}], + }, token=claimed["token"]) + s.request("DELETE", "__ds/subscriptions/claim-race") + print("PASS source recreation during Engine issuance fences old claim and publishes replacement", flush=True) + + s.restart(pause="after_webhook_signing") + s.sub("webhook-race", "events/webhook-race", receiver, lease=60000) + s.request("PUT", "events/webhook-race", {"original": True}) + s.poll(lambda: s.marker.exists(), timeout=8) + assert s.marker.read_text() == "after_webhook_signing" + assert receiver.requests.empty(), "Webhook was delivered before its signing barrier" + s.request("DELETE", "events/webhook-race") + s.request("PUT", "events/webhook-race", {"recreated": True}) + assert s.request("GET", "events/webhook-race")[1] == [{"recreated": True}] + # Source mutations await the durable catalog notification. The pump + # remains paused; no subscription control operation repairs its state. + s.release_marker.write_text("release") + # No subscription control call or additional write may rescue this wake. + _, replacement = receiver.next() + assert replacement["generation"] > 1, "Old signed snapshot reached the receiver" + s.request("POST", replacement["callback_url"], { + "wake_id": replacement["wake_id"], "generation": replacement["generation"], "done": True, + "acks": [{"stream": "events/webhook-race", "offset": replacement["streams"][0]["tail_offset"]}], + }, token=replacement["callback_token"]) + s.request("DELETE", "__ds/subscriptions/webhook-race") + print("PASS source recreation during Engine webhook signing suppresses old delivery and wakes replacement", flush=True) + + # Same IDs and paths in another tenant resolve to different actors. + prior_keys = s.request("GET", "__ds/jwks.json", token=None)[1] + s.stop() + s.tenant += "-other" + s.start() + s.request("GET", "__ds/subscriptions/retry", expected=(404,)) + s.request("GET", "events/retry", expected=(404,)) + # Webhook keys belong to the Engine namespace; DS tenants still fence + # capabilities and isolate actors while sharing its public signer. + assert s.request("GET", "__ds/jwks.json", token=None)[1] == prior_keys + print("PASS tenant isolation and default management authorization", flush=True) + finally: + s.stop() + receiver.close() + + +def run_migration(legacy_binary, binary, engine, logs): + logs = pathlib.Path(logs) / "migration" + logs.mkdir(exist_ok=True) + s = Server(legacy_binary, engine, logs) + # Keep the legacy catalog scan bounded independently of the thousands of + # stream actors left by the official suite in its own namespace. + s.namespace = "ds-migration-" + uuid.uuid4().hex + s.engine_token = provision(engine, s.namespace, os.environ["CONFORMANCE_ENGINE_ADMIN_TOKEN"]) + receiver = Receiver() + + def token_header(token): + part = token.split(".")[0] + return json.loads(base64.urlsafe_b64decode(part + "=" * (-len(part) % 4))) + + try: + s.start() + # Register patterns before creating sources: the preserved old SDK + # cannot authenticate keyed actor lookups against a secured Engine. + s.request("PUT", "__ds/subscriptions/legacy-hook", { + "type": "webhook", "pattern": "events/legacy-hook", "lease_ttl_ms": 600000, + "webhook": {"url": receiver.url}, + }) + s.request("PUT", "wake/legacy-claim", []) + s.request("PUT", "__ds/subscriptions/legacy-claim", { + "type": "pull-wake", "pattern": "events/legacy-claim", "lease_ttl_ms": 600000, + "wake_stream": "wake/legacy-claim", + }) + old_keys = s.request("GET", "__ds/jwks.json", token=None)[1]["keys"] + s.request("PUT", "events/legacy-hook", {"legacy": True}) + _, callback = receiver.next() + s.request("PUT", "events/legacy-claim", {"legacy": True}) + claimed = s.poll(lambda: s.request("POST", "__ds/subscriptions/legacy-claim/claim", {"worker": "legacy"})[1]) + assert token_header(callback["callback_token"])["alg"] == "HS256" + assert token_header(claimed["token"])["alg"] == "HS256" + s.stop() + s.binary = str(pathlib.Path(binary).resolve()) + s.start() + new_keys = s.request("GET", "__ds/jwks.json", token=None)[1]["keys"] + assert all(key in new_keys for key in old_keys), "Upgrade discarded in-flight webhook verification keys" + assert len(new_keys) > len(old_keys), "Engine webhook keys were not published after cutover" + callback_body = { + "wake_id": callback["wake_id"], "generation": callback["generation"], "done": True, + "acks": [{"stream": "events/legacy-hook", "offset": callback["streams"][0]["tail_offset"]}], + } + s.request("POST", callback["callback_url"], callback_body, token=callback["callback_token"]) + assert s.request("POST", callback["callback_url"], callback_body, token=callback["callback_token"], expected=(409,))[1]["error"]["code"] == "FENCED" + s.request("POST", "__ds/subscriptions/legacy-claim/ack", { + "wake_id": claimed["wake_id"], "generation": claimed["generation"], "done": True, + "acks": [{"stream": "events/legacy-claim", "offset": claimed["streams"][0]["tail_offset"]}], + }, token=claimed["token"]) + s.request("POST", "events/legacy-claim", {"engine": True}) + fresh = s.poll(lambda: s.request("POST", "__ds/subscriptions/legacy-claim/claim", {"worker": "engine"})[1]) + assert token_header(fresh["token"])["typ"] == "rivet-app+jwt" + assert fresh["generation"] > claimed["generation"] + s.request("POST", "events/legacy-hook", {"engine": True}) + _, fresh_callback = receiver.next() + assert token_header(fresh_callback["callback_token"])["typ"] == "rivet-app+jwt" + print("PASS live legacy callback and claim survive cutover; all new capabilities use Engine JWTs", flush=True) + finally: + s.stop() + receiver.close() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--binary", default="target/debug/rivet-durable-streams") + parser.add_argument("--engine", default="http://127.0.0.1:17420") + parser.add_argument("--logs", default=None) + parser.add_argument("--legacy-binary", default=os.environ.get("CONFORMANCE_LEGACY_BINARY")) + args = parser.parse_args() + if args.logs: + pathlib.Path(args.logs).mkdir(parents=True, exist_ok=True) + run(args.binary, args.engine, args.logs) + if args.legacy_binary: + run_migration(args.legacy_binary, args.binary, args.engine, args.logs) + else: + with tempfile.TemporaryDirectory(prefix="ds-subscription-recovery-") as logs: + run(args.binary, args.engine, logs) + if args.legacy_binary: + run_migration(args.legacy_binary, args.binary, args.engine, logs)