diff --git a/.custom-gcl.yml b/.custom-gcl.yml new file mode 100644 index 000000000..a86822eff --- /dev/null +++ b/.custom-gcl.yml @@ -0,0 +1,5 @@ +version: v2.13.1 +plugins: + - module: "go.kenn.io/kit" + import: "go.kenn.io/kit/lint/gclplugin" + version: "v0.25.1-0.20260918202731-04a175847323" diff --git a/.gitattributes b/.gitattributes index 082de49a0..d6f4e02b1 100644 --- a/.gitattributes +++ b/.gitattributes @@ -17,3 +17,6 @@ internal/vcard/registry/data/*.csv -text whitespace=blank-at-eol,blank-at-eof,sp # line endings on Windows checkouts. internal/slack/testdata/slackdump/standard/general/attachments/** -text internal/slack/testdata/slackdump/standard/__uploads/** -text + +# Keep the pinned upstream Vec1 source byte-for-byte reproducible. +internal/vector/sqlitevec/vec1/vec1.c -text whitespace=-blank-at-eol,-blank-at-eof diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 7f5368070..5da19b50b 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -1,3 +1,8 @@ self-hosted-runner: labels: - windows-11-vs2026-arm + - namespace-profile-linux-4cpu + - namespace-profile-linux-8cpu + - namespace-profile-windows-8cpu + - nscloud-ubuntu-24.04-arm64-4x8 + - namespace-profile-mac-6cpu diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index b45baef5e..88ce02905 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -1,8 +1,7 @@ name: CI # The PR dispatcher always calls the main-pinned workflow. That workflow -# independently routes same-repository Linux jobs to the managed public fleet -# and fork jobs to GitHub-hosted runners. +# runs PR jobs on Namespace, including fork PRs. on: pull_request: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8f51e7395..c5ba35a29 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,6 +3,10 @@ name: CI permissions: contents: read +# The | separator falls back to the module's source repo on any proxy error, not just 404/410. +env: + GOPROXY: https://proxy.golang.org|direct + on: workflow_call: push: @@ -17,12 +21,12 @@ jobs: matrix: include: - goarch: amd64 - runner: ubuntu-latest + runner: namespace-profile-linux-8cpu go_arch_name: amd64 - goarch: arm64 - runner: ubuntu-24.04-arm + runner: nscloud-ubuntu-24.04-arm64-4x8 go_arch_name: arm64 - runs-on: ${{ contains(fromJSON('["ubuntu-latest","ubuntu-24.04","ubuntu-22.04"]'), matrix.runner) && github.repository == 'kenn-io/msgvault' && (github.event_name == 'push' && github.ref == 'refs/heads/main' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.base.repo.full_name == github.repository)) && 'kenn-linux-x64-public' || matrix.runner }} + runs-on: ${{ matrix.runner }} container: image: ubuntu:22.04 steps: @@ -56,11 +60,13 @@ jobs: VERSION="ci-$COMMIT" mkdir -p dist - LDFLAGS="-s -w -X go.kenn.io/msgvault/cmd/msgvault/cmd.Version=${VERSION} -X go.kenn.io/msgvault/cmd/msgvault/cmd.Commit=${COMMIT} -X go.kenn.io/msgvault/cmd/msgvault/cmd.BuildDate=$(date -u +%Y-%m-%dT%H:%M:%SZ) -extldflags '-lstdc++ -lm'" + CGO_ENABLED=0 go build -trimpath -buildvcs=false -o dist/msgvault-codex-bridge ./cmd/msgvault-codex-bridge + BRIDGE_SHA256=$(sha256sum dist/msgvault-codex-bridge | cut -d' ' -f1) + LDFLAGS="-s -w -X go.kenn.io/msgvault/cmd/msgvault/cmd.Version=${VERSION} -X go.kenn.io/msgvault/cmd/msgvault/cmd.Commit=${COMMIT} -X go.kenn.io/msgvault/cmd/msgvault/cmd.BuildDate=$(date -u +%Y-%m-%dT%H:%M:%SZ) -X go.kenn.io/msgvault/internal/peoplesweep.codexBridgeSHA256=${BRIDGE_SHA256} -extldflags '-lstdc++ -lm'" go build -tags "fts5 sqlite_vec" -trimpath -buildvcs=false -ldflags="$LDFLAGS" -o dist/msgvault ./cmd/msgvault echo "--- Binary info ---" - file dist/msgvault + file dist/msgvault dist/msgvault-codex-bridge ldd dist/msgvault || true echo "--- Runtime requirements ---" @@ -72,18 +78,23 @@ jobs: echo "$SMOKE_OUT" echo "$SMOKE_OUT" | grep -q "$VERSION" || { echo "FATAL: version output doesn't match CI build version"; exit 1; } - # Hosted Ubuntu, like the PostgreSQL lanes: on the managed Linux runners the - # unsharded packages alone take over 30 minutes and the largest packages - # hit go test's one-hour timeout, while hosted runners finish in minutes. test: - runs-on: ubuntu-24.04 + runs-on: namespace-profile-linux-8cpu steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: go-version-file: go.mod - cache: 'true' + cache: false + + - name: Install SQLite development headers + run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev + + - name: Configure Namespace Go cache + uses: namespacelabs/nscloud-cache-action@1124a6f3ce44e5cf84cc22111530961f4d2a15f9 # v1 + with: + cache: go - name: Check PostgreSQL build coverage run: make pg-shipped-only-check @@ -130,34 +141,47 @@ jobs: package: ./internal/store - name: api package: ./internal/api - runs-on: ubuntu-24.04 + runs-on: namespace-profile-linux-8cpu steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: go-version-file: go.mod - cache: 'true' + cache: false + + - name: Install SQLite development headers + run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev + + - name: Configure Namespace Go cache + uses: namespacelabs/nscloud-cache-action@1124a6f3ce44e5cf84cc22111530961f4d2a15f9 # v1 + with: + cache: go - name: Test package in isolated shards run: scripts/test-package-shards.sh "${{ matrix.package }}" 4 "fts5 sqlite_vec" 60m test-macos-15: - if: github.repository == 'kenn-io/msgvault' && github.event_name == 'push' && github.ref == 'refs/heads/main' - runs-on: macos-15 + if: github.repository == 'kenn-io/msgvault' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) + runs-on: namespace-profile-mac-6cpu steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: go-version-file: go.mod - cache: 'true' + cache: false + + - name: Configure Namespace Go cache + uses: namespacelabs/nscloud-cache-action@1124a6f3ce44e5cf84cc22111530961f4d2a15f9 # v1 + with: + cache: go - name: Test run: make test frontend: - runs-on: ${{ github.repository == 'kenn-io/msgvault' && (github.event_name == 'push' && github.ref == 'refs/heads/main' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.base.repo.full_name == github.repository)) && 'kenn-linux-x64-public' || 'ubuntu-24.04' }} + runs-on: namespace-profile-linux-8cpu container: image: mcr.microsoft.com/playwright:v1.61.1-noble@sha256:5b8f294aff9041b7191c34a4bab3ac270157a28774d4b0660e9743297b697e48 steps: @@ -174,7 +198,7 @@ jobs: rm -f /etc/apt/sources.list.d/nodesource.list for attempt in 1 2 3; do if apt-get update -o Acquire::Retries=3 -o Acquire::http::Timeout=30 \ - && apt-get install -y --no-install-recommends -o Acquire::Retries=3 -o Acquire::http::Timeout=30 gcc g++ make unzip; then + && apt-get install -y --no-install-recommends -o Acquire::Retries=3 -o Acquire::http::Timeout=30 gcc g++ make unzip libsqlite3-dev; then exit 0 fi echo "apt attempt $attempt failed; retrying" >&2 @@ -188,7 +212,7 @@ jobs: - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: go-version-file: go.mod - cache: ${{ github.repository == 'kenn-io/msgvault' && (github.event_name == 'push' && github.ref == 'refs/heads/main' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.base.repo.full_name == github.repository)) && 'false' || 'true' }} + cache: 'true' - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 with: @@ -220,7 +244,7 @@ jobs: if-no-files-found: ignore web-e2e: - runs-on: ${{ github.repository == 'kenn-io/msgvault' && (github.event_name == 'push' && github.ref == 'refs/heads/main' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.base.repo.full_name == github.repository)) && 'kenn-linux-x64-public' || 'ubuntu-24.04' }} + runs-on: namespace-profile-linux-8cpu container: image: mcr.microsoft.com/playwright:v1.61.1-noble@sha256:5b8f294aff9041b7191c34a4bab3ac270157a28774d4b0660e9743297b697e48 steps: @@ -229,7 +253,7 @@ jobs: - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: go-version-file: go.mod - cache: ${{ github.repository == 'kenn-io/msgvault' && (github.event_name == 'push' && github.ref == 'refs/heads/main' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.base.repo.full_name == github.repository)) && 'false' || 'true' }} + cache: 'true' - name: Install container build tools # Mirror hiccups have hung plain apt-get for 15+ minutes; bounded @@ -242,7 +266,7 @@ jobs: rm -f /etc/apt/sources.list.d/nodesource.list for attempt in 1 2 3; do if apt-get update -o Acquire::Retries=3 -o Acquire::http::Timeout=30 \ - && apt-get install -y --no-install-recommends -o Acquire::Retries=3 -o Acquire::http::Timeout=30 gcc g++ make unzip; then + && apt-get install -y --no-install-recommends -o Acquire::Retries=3 -o Acquire::http::Timeout=30 gcc g++ make unzip libsqlite3-dev; then exit 0 fi echo "apt attempt $attempt failed; retrying" >&2 @@ -279,14 +303,19 @@ jobs: large-archive-benchmark: if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' - runs-on: macos-15 + runs-on: namespace-profile-mac-6cpu steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: go-version-file: go.mod - cache: 'true' + cache: false + + - name: Configure Namespace Go cache + uses: namespacelabs/nscloud-cache-action@1124a6f3ce44e5cf84cc22111530961f4d2a15f9 # v1 + with: + cache: go - name: Run mixed-archive analytical reference benchmark run: | @@ -302,18 +331,23 @@ jobs: if-no-files-found: error test-windows: - runs-on: windows-latest + runs-on: namespace-profile-windows-8cpu steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: go-version-file: go.mod - cache: 'true' + cache: false + + - name: Configure Namespace Go cache + uses: namespacelabs/nscloud-cache-action@1124a6f3ce44e5cf84cc22111530961f4d2a15f9 # v1 + with: + cache: go # sqlite-vec's CGo bindings #include , which isn't on - # the default windows-latest include path. Install it from MSYS2 - # (already present on the runner) and point CGO_CFLAGS at it. + # the default include path. Install it from MSYS2 and point + # CGO_CFLAGS at it. Keep the runner's default compiler for DuckDB. # # arrow-go/v18's cdata helpers.h declares its helpers as plain # `inline` (not `static inline`). Under C99 that requires exactly @@ -322,10 +356,11 @@ jobs: # external copy, and --allow-multiple-definition lets MinGW's ld # pick the first and continue instead of erroring on the # duplicates that flag produces. - - name: Install sqlite3 dev headers + - name: Install SQLite development headers shell: pwsh run: | C:\msys64\usr\bin\pacman.exe -S --noconfirm --needed mingw-w64-x86_64-sqlite3 + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - name: Build shell: pwsh @@ -335,6 +370,20 @@ jobs: CGO_LDFLAGS: "-Wl,--allow-multiple-definition" run: go build -tags "fts5 sqlite_vec" -o msgvault.exe ./cmd/msgvault + # Run the same custom linter as `make custom-gcl` from PowerShell. + - name: Lint + shell: pwsh + env: + CGO_ENABLED: "1" + CGO_CFLAGS: "-IC:/msys64/mingw64/include -fgnu89-inline" + CGO_LDFLAGS: "-Wl,--allow-multiple-definition" + GOFLAGS: "-buildvcs=false" + run: | + $version = (Select-String -Path .custom-gcl.yml -Pattern '^version: (\S+)').Matches[0].Groups[1].Value + go run "github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$version" custom --destination "$env:RUNNER_TEMP" --name custom-gcl + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + & "$env:RUNNER_TEMP/custom-gcl.exe" run ./... + - name: Test shell: pwsh env: @@ -363,19 +412,25 @@ jobs: path: msgvault.exe test-windows-slow-packages: - runs-on: windows-latest + runs-on: namespace-profile-windows-8cpu steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: go-version-file: go.mod - cache: 'true' + cache: false + + - name: Configure Namespace Go cache + uses: namespacelabs/nscloud-cache-action@1124a6f3ce44e5cf84cc22111530961f4d2a15f9 # v1 + with: + cache: go - - name: Install sqlite3 dev headers + - name: Install SQLite development headers shell: pwsh run: | C:\msys64\usr\bin\pacman.exe -S --noconfirm --needed mingw-w64-x86_64-sqlite3 + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - name: Test slow packages shell: pwsh @@ -395,17 +450,14 @@ jobs: fail-fast: false matrix: include: - - name: cli - package: ./cmd/msgvault/cmd - # Windows SQLite I/O makes this package's aggregate shard duration - # exceed one hour: the passing shard alone ran 406 tests in 3373.8s - # (~56m) and the next shard timed out at 60m while still actively - # executing schema setup (no deadlock; tests kept progressing). - # Give only this package the larger per-shard budget. - timeout: 120m - - name: store - package: ./internal/store - timeout: 60m + # A 4-core runner is already saturated by one package's shards, so the + # two slowest packages each split across 3 runners. + - { name: cli-1, package: ./cmd/msgvault/cmd, part: 1, parts: 3, timeout: 60m } + - { name: cli-2, package: ./cmd/msgvault/cmd, part: 2, parts: 3, timeout: 60m } + - { name: cli-3, package: ./cmd/msgvault/cmd, part: 3, parts: 3, timeout: 60m } + - { name: store-1, package: ./internal/store, part: 1, parts: 3, timeout: 60m } + - { name: store-2, package: ./internal/store, part: 2, parts: 3, timeout: 60m } + - { name: store-3, package: ./internal/store, part: 3, parts: 3, timeout: 60m } - name: sync-vector-embed package: ./internal/sync package2: ./internal/vector/embed @@ -413,19 +465,25 @@ jobs: - name: vector-sqlite package: ./internal/vector/sqlitevec timeout: 60m - runs-on: windows-latest + runs-on: namespace-profile-windows-8cpu steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: go-version-file: go.mod - cache: 'true' + cache: false + + - name: Configure Namespace Go cache + uses: namespacelabs/nscloud-cache-action@1124a6f3ce44e5cf84cc22111530961f4d2a15f9 # v1 + with: + cache: go - - name: Install sqlite3 dev headers + - name: Install SQLite development headers shell: pwsh run: | C:\msys64\usr\bin\pacman.exe -S --noconfirm --needed mingw-w64-x86_64-sqlite3 + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - name: Test package in isolated shards shell: pwsh @@ -441,12 +499,14 @@ jobs: -Package $package ` -ShardCount 4 ` -Tags "fts5 sqlite_vec" ` - -Timeout ${{ matrix.timeout }} + -Timeout ${{ matrix.timeout }} ` + -PartIndex ${{ matrix.part || 1 }} ` + -PartCount ${{ matrix.parts || 1 }} if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } } test-windows-pack-restore: - runs-on: windows-latest + runs-on: namespace-profile-windows-8cpu timeout-minutes: 20 steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -454,12 +514,18 @@ jobs: - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: go-version-file: go.mod - cache: 'true' + cache: false - - name: Install sqlite3 dev headers + - name: Configure Namespace Go cache + uses: namespacelabs/nscloud-cache-action@1124a6f3ce44e5cf84cc22111530961f4d2a15f9 # v1 + with: + cache: go + + - name: Install SQLite development headers shell: pwsh run: | C:\msys64\usr\bin\pacman.exe -S --noconfirm --needed mingw-w64-x86_64-sqlite3 + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - name: Measure packed and loose restore layouts shell: pwsh @@ -486,7 +552,7 @@ jobs: # tag is required; without it these files compile out and ship no tests, # so this code would otherwise have zero CI coverage. test-pgvector: - runs-on: ubuntu-latest + runs-on: namespace-profile-linux-8cpu services: postgres: image: pgvector/pgvector:pg16 @@ -510,7 +576,15 @@ jobs: - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: go-version-file: go.mod - cache: 'true' + cache: false + + - name: Install SQLite development headers + run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev + + - name: Configure Namespace Go cache + uses: namespacelabs/nscloud-cache-action@1124a6f3ce44e5cf84cc22111530961f4d2a15f9 # v1 + with: + cache: go - name: Enable pgvector extension run: | @@ -531,7 +605,7 @@ jobs: # unique constraints) surface here rather than slipping past a single # pass. test-postgres: - runs-on: ubuntu-latest + runs-on: namespace-profile-linux-8cpu services: postgres: image: postgres:16 @@ -555,7 +629,15 @@ jobs: - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: go-version-file: go.mod - cache: 'true' + cache: false + + - name: Install SQLite development headers + run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev + + - name: Configure Namespace Go cache + uses: namespacelabs/nscloud-cache-action@1124a6f3ce44e5cf84cc22111530961f4d2a15f9 # v1 + with: + cache: go # This lane uses a stock postgres:16 image (no pgvector extension), so it # must NOT build the pgvector tag — those tests CREATE EXTENSION "vector" @@ -590,7 +672,7 @@ jobs: package: ./internal/store - name: api package: ./internal/api - runs-on: ubuntu-latest + runs-on: namespace-profile-linux-8cpu services: postgres: image: postgres:16 @@ -614,7 +696,15 @@ jobs: - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: go-version-file: go.mod - cache: 'true' + cache: false + + - name: Install SQLite development headers + run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev + + - name: Configure Namespace Go cache + uses: namespacelabs/nscloud-cache-action@1124a6f3ce44e5cf84cc22111530961f4d2a15f9 # v1 + with: + cache: go # Shipped build (no pgvector tag) against a stock postgres:16 image, as # in test-postgres above. diff --git a/.github/workflows/codex-isolation.yml b/.github/workflows/codex-isolation.yml new file mode 100644 index 000000000..486813550 --- /dev/null +++ b/.github/workflows/codex-isolation.yml @@ -0,0 +1,40 @@ +name: Codex isolation + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +jobs: + pinned-artifact: + # Ubuntu 22.04 permits the unprivileged user namespaces used by Bubblewrap. + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version-file: go.mod + cache: 'true' + - name: Install Bubblewrap + run: sudo apt-get update && sudo apt-get install -y bubblewrap + - name: Download pinned Codex artifact + run: | + mkdir -p "$RUNNER_TEMP/codex-artifact" + cd "$RUNNER_TEMP/codex-artifact" + curl -fsSL --retry 3 https://github.com/openai/codex/releases/download/rust-v0.156.0/codex-x86_64-unknown-linux-musl.tar.gz -o codex.tar.gz + echo '3d49d9af25a5168cfc51e50e520ab238b23083c259ae7c14f89b007cb2545c7b codex.tar.gz' | sha256sum -c - + tar -xzf codex.tar.gz + echo '78a11f06e0a2dda42d13fba1d50dc62e8cbdb2d5f69789722f4d4d99b5cdbe30 codex-x86_64-unknown-linux-musl' | sha256sum -c - + echo "MSGVAULT_CODEX_PINNED_EXECUTABLE=$PWD/codex-x86_64-unknown-linux-musl" >> "$GITHUB_ENV" + - name: Exercise launcher and protocol without real credentials + run: | + CGO_ENABLED=0 go build -trimpath -buildvcs=false -o "$RUNNER_TEMP/codex-artifact/msgvault-codex-bridge" ./cmd/msgvault-codex-bridge + chmod 755 "$RUNNER_TEMP/codex-artifact/msgvault-codex-bridge" + bridge_digest=$(sha256sum "$RUNNER_TEMP/codex-artifact/msgvault-codex-bridge" | cut -d' ' -f1) + go test -tags "fts5 sqlite_vec" -c -ldflags="-X go.kenn.io/msgvault/internal/peoplesweep.codexBridgeSHA256=$bridge_digest" -o "$RUNNER_TEMP/codex-artifact/peoplesweep.test" ./internal/peoplesweep + cd internal/peoplesweep + MSGVAULT_CODEX_TEST_DEFAULT_BRIDGE=1 "$RUNNER_TEMP/codex-artifact/peoplesweep.test" -test.timeout=5m -test.run='^TestCodex' -test.count=1 diff --git a/.github/workflows/docker-pr.yml b/.github/workflows/docker-pr.yml index 1f9773a87..35c27d803 100644 --- a/.github/workflows/docker-pr.yml +++ b/.github/workflows/docker-pr.yml @@ -1,8 +1,6 @@ name: Docker -# Bootstrap the split between validation and publishing on a GitHub-hosted -# runner. Follow-up PRs can return this dispatcher to docker.yml@main once -# that read-only reusable workflow is present on the default branch. +# Run the main-pinned build and smoke checks on Namespace. on: pull_request: paths: @@ -25,36 +23,4 @@ permissions: read-all jobs: validate: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 - - - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 - with: - bun-version: 1.3.14 - - - name: Build browser application - run: make web-assets-check - - - name: Build amd64 image - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 - with: - context: . - platforms: linux/amd64 - push: false - load: true - tags: msgvault:test - build-args: | - VERSION=test - COMMIT=${{ github.sha }} - BUILD_DATE=test - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Smoke test (amd64) - run: scripts/smoke-container.sh msgvault:test + uses: kenn-io/msgvault/.github/workflows/docker.yml@main diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 59383d4a0..a04b1c237 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,6 +1,7 @@ name: Docker on: + workflow_dispatch: workflow_call: permissions: read-all @@ -8,7 +9,7 @@ permissions: read-all jobs: # PR validation: build and smoke-test only, no registry access validate: - runs-on: ${{ github.repository == 'kenn-io/msgvault' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.base.repo.full_name == github.repository && 'kenn-linux-x64-public' || 'ubuntu-latest' }} + runs-on: namespace-profile-linux-4cpu permissions: contents: read diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 029aa92d7..82793e3db 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -11,7 +11,7 @@ on: jobs: docs: - runs-on: ubuntu-24.04 + runs-on: namespace-profile-linux-4cpu steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 diff --git a/.github/workflows/workflow-validation.yml b/.github/workflows/workflow-validation.yml index d7ff9115e..8a636a4c5 100644 --- a/.github/workflows/workflow-validation.yml +++ b/.github/workflows/workflow-validation.yml @@ -1,6 +1,7 @@ name: Workflow validation on: + workflow_dispatch: pull_request: paths: - ".github/workflows/**" @@ -10,7 +11,7 @@ permissions: jobs: actionlint: - runs-on: ubuntu-latest + runs-on: namespace-profile-linux-4cpu steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - uses: raven-actions/actionlint@3d39aea434753780c3b3d4a1a31c854b4dbf49d7 # v2.2.0 diff --git a/.gitignore b/.gitignore index bd02c851e..056423e87 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Go binaries /msgvault +/msgvault-codex-bridge /msgvault.exe /mimeshootout diff --git a/.golangci.yml b/.golangci.yml index ddff5704c..56960c73d 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -59,6 +59,7 @@ linters: - ineffassign - intrange - iotamixing + - kennlint - loggercheck - makezero - mirror @@ -203,8 +204,30 @@ linters: # use errors.Is to distinguish, which wrapping would break. - .Err( + custom: + # Kit's analyzers, built into custom-gcl by `make lint`. Only sleeptest + # runs. The stock sqlclosecheck and rowserrcheck above stay; Kit's + # copies and its other analyzers are off until they get their own pass. + kennlint: + type: module + description: Kit sleeptest, time.Sleep in tests outside synctest. + original-url: go.kenn.io/kit/lint + settings: + disable: + - errtext + - nohttpmux + - rowserrcheck + - sqlcheck + - sqlclosecheck + - testifyhelper + exclusions: rules: + # Workspace varies in Linux enrollment tests excluded from Windows builds. + - linters: [unparam] + path: internal/peoplesweep/codex_app_server_test\.go + text: "syntheticCodexAuth - workspace" + # Tests legitimately repeat fixture strings - linters: [goconst] path: _test\.go diff --git a/.roborev.toml b/.roborev.toml index 55a4b9438..9a14e5e67 100644 --- a/.roborev.toml +++ b/.roborev.toml @@ -54,4 +54,8 @@ repo-ignored local credential files, dotenv files, OAuth client JSON, or other dotfiles into those artifacts. Do not flag literal example strings in documentation snippets as leaked secrets unless they are plausible real credentials. + +The Jev adapter passes a plain http.Client to Docbank's TypeSafe client, +which refuses redirects on its own copy. Do not flag redirect handling +in internal/vector/rerank/jev.go. """ diff --git a/Dockerfile b/Dockerfile index 6d863bcb5..c3b69d022 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ RUN cd web && bun run generate && bun run build # Go build stage. # Pin by digest for reproducibility; update periodically. -FROM golang:1.27.0-bookworm@sha256:484ef6066fa69acb059fdfeda7ba2b8f7391f2ef6abc6f9b8411e669ebd56466 AS builder +FROM golang:1.27.1-bookworm@sha256:69a7b9788769bec032d238959b61854e9ae87f57be9029ec04e9885fabf99195 AS builder # Install build dependencies for CGO (SQLite, DuckDB). # libsqlite3-dev provides sqlite3.h, required to compile the sqlite-vec @@ -26,6 +26,9 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins WORKDIR /src +# Fall back to the module's source repo on any proxy error, not just 404/410. +ENV GOPROXY=https://proxy.golang.org|direct + # Download dependencies first (layer caching) COPY go.mod go.sum ./ RUN go mod download @@ -42,13 +45,17 @@ ARG COMMIT=unknown ARG BUILD_DATE=unknown # Note: Module path must match go.mod (go.kenn.io/msgvault) -RUN CGO_ENABLED=1 go build \ +RUN CGO_ENABLED=0 go build -trimpath -buildvcs=false \ + -o /msgvault-codex-bridge ./cmd/msgvault-codex-bridge \ + && bridge_digest=$(sha256sum /msgvault-codex-bridge | cut -d' ' -f1) \ + && CGO_ENABLED=1 go build \ -tags "fts5 sqlite_vec" \ -trimpath \ -ldflags="-s -w \ -X go.kenn.io/msgvault/cmd/msgvault/cmd.Version=${VERSION} \ -X go.kenn.io/msgvault/cmd/msgvault/cmd.Commit=${COMMIT} \ - -X go.kenn.io/msgvault/cmd/msgvault/cmd.BuildDate=${BUILD_DATE}" \ + -X go.kenn.io/msgvault/cmd/msgvault/cmd.BuildDate=${BUILD_DATE} \ + -X go.kenn.io/msgvault/internal/peoplesweep.codexBridgeSHA256=${bridge_digest}" \ -o /msgvault \ ./cmd/msgvault @@ -61,6 +68,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins tzdata \ wget \ libstdc++6 \ + bubblewrap \ && rm -rf /var/lib/apt/lists/* # Create non-root user @@ -69,6 +77,7 @@ RUN groupadd --gid 1000 msgvault \ # Copy binary from builder COPY --from=builder /msgvault /usr/local/bin/msgvault +COPY --from=builder --chown=msgvault:msgvault /msgvault-codex-bridge /usr/local/bin/msgvault-codex-bridge # Set up data directory with correct ownership ENV MSGVAULT_HOME=/data diff --git a/Makefile b/Makefile index ec47697b7..7c8de0800 100644 --- a/Makefile +++ b/Makefile @@ -50,6 +50,7 @@ GO_INSTALL_BIN := $(shell go env GOPATH)/bin endif GOLANGCI_LINT_BIN := $(GO_INSTALL_BIN)/golangci-lint CI_TOOLS_BIN := $(shell git rev-parse --path-format=absolute --git-path ci-tools/bin) +CUSTOM_GCL_BIN := $(CI_TOOLS_BIN)/custom-gcl$(shell go env GOEXE) GOVULNCHECK_BIN := $(CI_TOOLS_BIN)/govulncheck # Build tags for the PostgreSQL test lane (test-pg). Must be the full build set: @@ -70,7 +71,7 @@ PG_TEST_TAGS := fts5 sqlite_vec pgvector # in both configurations, so test-pg-both runs just these in the shipped-build # configuration. Verified by `make pg-shipped-only-check`, which re-derives the # closure from `go list`. -PG_SHIPPED_ONLY_PKGS := ./cmd/msgvault ./cmd/msgvault/cmd ./internal/api ./internal/mcp ./internal/scheduler ./internal/store ./internal/vector/chunkmatch ./internal/vector/document ./internal/vector/embed ./internal/vector/hybrid ./internal/vector/pgvector ./scripts/contextual-retrieval-eval +PG_SHIPPED_ONLY_PKGS := ./cmd/msgvault ./cmd/msgvault/cmd ./internal/api ./internal/daemonclient ./internal/mcp ./internal/scheduler ./internal/store ./internal/vector/chunkmatch ./internal/vector/document ./internal/vector/embed ./internal/vector/hybrid ./internal/vector/pgvector ./scripts/contextual-retrieval-eval OPENAPI_ARTIFACTS := api/openapi.yaml pkg/client/openapi.yaml pkg/client/generated WEB_INSTALL_STAMP := web/node_modules/.msgvault-install-stamp @@ -87,23 +88,38 @@ export GOLANGCI_LINT_CACHE # serialize one another while duplicate runners in one worktree can wait. GOLANGCI_LINT_TMP ?= $(GOLANGCI_LINT_CACHE)/tmp -.PHONY: build build-release install clean test test-unsharded test-shards test-v test-pg test-pg-shipped test-pg-shipped-unsharded test-pg-both pg-shipped-only-check require-test-db fmt lint-tools lint lint-ci vuln-tools vulncheck testify-helper-check tidy openapi api-generate openapi-check api-check web-install web-generate web-check web-test web-test-browser web-e2e web-build web-embed web-assets-check smoke-web-release shootout run-shootout install-hooks bench vcard-registry-check vcard-registry-update docs-install docs-build docs-serve docs-check docs-fixture-test docs-fixture-check docs-fixture-smoke docs-web-screenshots docs-screenshots docs-assets-branch docs-generated-assets-branch docs-deploy-staging docs-deploy help +.PHONY: build build-release install clean test test-unsharded test-shards test-v test-pg test-pg-shipped test-pg-shipped-unsharded test-pg-both pg-shipped-only-check require-test-db fmt lint-tools custom-gcl lint lint-ci vuln-tools vulncheck testify-helper-check tidy openapi api-generate openapi-check api-check web-install web-generate web-check web-test web-test-browser web-e2e web-build web-embed web-assets-check smoke-web-release shootout run-shootout install-hooks bench vcard-registry-check vcard-registry-update docs-install docs-build docs-serve docs-check docs-fixture-test docs-fixture-check docs-fixture-smoke docs-web-screenshots docs-screenshots docs-assets-branch docs-generated-assets-branch docs-deploy-staging docs-deploy help # Build the binary (debug) build: web-embed +ifeq ($(shell go env GOOS),linux) + CGO_ENABLED=0 go build -trimpath -buildvcs=false -o msgvault-codex-bridge ./cmd/msgvault-codex-bridge + chmod 755 msgvault-codex-bridge + @bridge_digest=$$(sha256sum msgvault-codex-bridge | cut -d' ' -f1); \ + CGO_ENABLED=1 go build -tags "$(BUILD_TAGS)" -ldflags="$(LDFLAGS) -X go.kenn.io/msgvault/internal/peoplesweep.codexBridgeSHA256=$$bridge_digest" -o msgvault ./cmd/msgvault +else CGO_ENABLED=1 go build -tags "$(BUILD_TAGS)" -ldflags="$(LDFLAGS)" -o msgvault ./cmd/msgvault +endif @chmod +x msgvault # Build with optimizations (release) build-release: web-embed +ifeq ($(shell go env GOOS),linux) + CGO_ENABLED=0 go build -trimpath -buildvcs=false -o msgvault-codex-bridge ./cmd/msgvault-codex-bridge + chmod 755 msgvault-codex-bridge + @bridge_digest=$$(sha256sum msgvault-codex-bridge | cut -d' ' -f1); \ + CGO_ENABLED=1 go build -tags "$(BUILD_TAGS)" -ldflags="$(LDFLAGS_RELEASE) -X go.kenn.io/msgvault/internal/peoplesweep.codexBridgeSHA256=$$bridge_digest" -trimpath -o msgvault ./cmd/msgvault +else CGO_ENABLED=1 go build -tags "$(BUILD_TAGS)" -ldflags="$(LDFLAGS_RELEASE)" -trimpath -o msgvault ./cmd/msgvault +endif @chmod +x msgvault # Install to ~/.local/bin, $GOBIN, or $GOPATH/bin -install: web-embed - @if [ -d "$(HOME)/.local/bin" ]; then \ +install: build + @set -e; if [ -d "$(HOME)/.local/bin" ]; then \ echo "Installing to ~/.local/bin/msgvault"; \ - CGO_ENABLED=1 go build -tags "$(BUILD_TAGS)" -ldflags="$(LDFLAGS)" -o "$(HOME)/.local/bin/msgvault" ./cmd/msgvault; \ + install -m 755 msgvault "$(HOME)/.local/bin/msgvault"; \ + if [ "$$(go env GOOS)" = linux ]; then install -m 755 msgvault-codex-bridge "$(HOME)/.local/bin/msgvault-codex-bridge"; fi; \ else \ INSTALL_DIR="$${GOBIN:-$$(go env GOBIN)}"; \ if [ -z "$$INSTALL_DIR" ]; then \ @@ -112,12 +128,13 @@ install: web-embed fi; \ mkdir -p "$$INSTALL_DIR"; \ echo "Installing to $$INSTALL_DIR/msgvault"; \ - CGO_ENABLED=1 go build -tags "$(BUILD_TAGS)" -ldflags="$(LDFLAGS)" -o "$$INSTALL_DIR/msgvault" ./cmd/msgvault; \ + install -m 755 msgvault "$$INSTALL_DIR/msgvault"; \ + if [ "$$(go env GOOS)" = linux ]; then install -m 755 msgvault-codex-bridge "$$INSTALL_DIR/msgvault-codex-bridge"; fi; \ fi # Clean build artifacts clean: - rm -f msgvault msgvault.exe mimeshootout + rm -f msgvault msgvault.exe msgvault-codex-bridge mimeshootout rm -rf bin/ # Scale the SQLite suite when both CPU and memory budgets allow it. An explicit @@ -275,7 +292,7 @@ api-generate: set -e; tmp="$$(mktemp)"; trap 'rm -f "$$tmp"' EXIT; go run ./cmd/msgvault openapi > "$$tmp"; if [ -f api/openapi.yaml ] && cmp -s "$$tmp" api/openapi.yaml; then rm "$$tmp"; else mv "$$tmp" api/openapi.yaml; fi; trap - EXIT set -e; tmp="$$(mktemp)"; trap 'rm -f "$$tmp"' EXIT; go run ./cmd/msgvault openapi --version 3.0 --format yaml > "$$tmp"; if [ -f pkg/client/openapi.yaml ] && cmp -s "$$tmp" pkg/client/openapi.yaml; then rm "$$tmp"; else mv "$$tmp" pkg/client/openapi.yaml; fi; trap - EXIT cd pkg/client/generated && find . -maxdepth 1 -type f -name '*.go' ! -name 'generate.go' -delete && go tool -modfile=../../../tools/oapi-codegen/go.mod oapi-codegen -config config.yaml ../openapi.yaml - go run ./internal/codegenfix/cmd pkg/client/generated/types.go + go run ./internal/codegenfix/cmd pkg/client/generated/types.go pkg/client/generated/client.go openapi-check: api-generate @git diff --exit-code -- $(OPENAPI_ARTIFACTS) || (echo "OpenAPI generated assets are stale; run 'make api-generate' and commit the changes." >&2; exit 1) @@ -309,13 +326,15 @@ web-check: web-install web-test: cd web && bun run test -web-test-browser: +# test:browser runs everything under web/tests, including tests/e2e specs +# whose fixtures spawn the repo-root msgvault daemon, so the binary must +# exist before Playwright starts. Same invariant as web-e2e below. +web-test-browser: build cd web && bun run test:browser -# Task 20 browser gates use the same digest-pinned Playwright environment as -# web-test-browser in CI. Traces, screenshots, and video are retained only for -# failures by web/playwright.config.ts. -web-e2e: +# Browser gates use the same digest-pinned Playwright environment as CI. +# Build the real daemon and embedded UI before Playwright test timeouts start. +web-e2e: build cd web && bun run test:e2e web-build: web-generate @@ -346,18 +365,25 @@ smoke-web-release: fmt: go fmt ./... -# Install the pinned linter used by CI. +# Install the pinned linter used by CI; lint-ci runs this first, so skip it when already installed. lint-tools: + @if [ "$$("$(GOLANGCI_LINT_BIN)" version --short 2>/dev/null)" = "$(GOLANGCI_LINT_VERSION:v%=%)" ]; then exit 0; fi; \ go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) +# Build golangci-lint with the plugins in .custom-gcl.yml into the +# repository-owned tool path. Strip repo-local Git variables so a build run +# from the commit hook does not inherit GIT_DIR. +custom-gcl: lint-tools + @mkdir -p "$(CI_TOOLS_BIN)" + @unset_args=$$(git rev-parse --local-env-vars 2>/dev/null | sed 's/^/-u /' | tr '\n' ' '); \ + env $$unset_args GOFLAGS=-buildvcs=false "$(GOLANGCI_LINT_BIN)" custom \ + --destination "$(CI_TOOLS_BIN)" --name custom-gcl \ + --version "$(GOLANGCI_LINT_VERSION)" + # Run linter (auto-fix) -lint: - @if ! command -v golangci-lint >/dev/null 2>&1; then \ - echo "golangci-lint not found. Install: https://golangci-lint.run/usage/install/" >&2; \ - exit 1; \ - fi +lint: custom-gcl @mkdir -p "$(GOLANGCI_LINT_TMP)" - TMPDIR="$(GOLANGCI_LINT_TMP)" golangci-lint run --fix ./... + TMPDIR="$(GOLANGCI_LINT_TMP)" "$(CUSTOM_GCL_BIN)" run --fix ./... # Check the shared Huma API contract. huma-check: @@ -366,9 +392,9 @@ huma-check: .PHONY: huma-check # Run linter (CI, no auto-fix) -lint-ci: lint-tools testify-helper-check +lint-ci: custom-gcl testify-helper-check @mkdir -p "$(GOLANGCI_LINT_TMP)" - TMPDIR="$(GOLANGCI_LINT_TMP)" "$(GOLANGCI_LINT_BIN)" run ./... + TMPDIR="$(GOLANGCI_LINT_TMP)" "$(CUSTOM_GCL_BIN)" run ./... @if [ -n "$$GITHUB_PATH" ]; then \ $(MAKE) --no-print-directory vuln-tools; \ printf '%s\n' "$(CI_TOOLS_BIN)" >> "$$GITHUB_PATH"; \ @@ -383,7 +409,7 @@ vuln-tools: vulncheck: vuln-tools "$(GOVULNCHECK_BIN)" -tags "$(BUILD_TAGS)" ./... -# Enforce testify helper usage in assertion-heavy tests +# Enforce testify helper usage and named sub-second polling budgets in tests testify-helper-check: go run ./cmd/testify-helper-check -tags="$(BUILD_TAGS)" ./... @@ -461,13 +487,18 @@ docs-assets-branch: docs-generated-assets-branch: bash docs/screenshots/update-generated-assets-branch.sh +# Build locally so the sibling website/ directory is available to both deploys. # Deploy docs to Vercel staging docs-deploy-staging: - cd docs && vercel + cd docs && vercel pull --yes --environment=preview + cd docs && vercel build + cd docs && vercel deploy --prebuilt # Deploy docs to Vercel production docs-deploy: - cd docs && vercel --prod + cd docs && vercel pull --yes --environment=production + cd docs && vercel build --prod + cd docs && vercel deploy --prebuilt --prod # Build the MIME shootout tool shootout: @@ -493,7 +524,7 @@ help: @echo " lint - Run linter (auto-fix)" @echo " lint-ci - Run linter (CI, no auto-fix; also runs testify-helper-check)" @echo " vulncheck - Run the pinned Go vulnerability scanner" - @echo " testify-helper-check - Enforce testify helper usage in assertion-heavy tests" + @echo " testify-helper-check - Enforce testify helpers and polling budgets in tests" @echo " tidy - Tidy go.mod" @echo " vcard-registry-check - Network-check IANA registry drift (manual; not CI)" @echo " vcard-registry-update - Update the vendored IANA vCard registry" diff --git a/README.md b/README.md index cd9c53108..9599085c6 100644 --- a/README.md +++ b/README.md @@ -13,11 +13,11 @@

Documentation · - Setup Guide · + Setup Guide · Interactive TUI

-**The system of record for your communications and relationships.** +**Keep your communications and relationships in an archive you own.** msgvault is a local-first, open-source archive for email, chat, meetings, calendars, and contacts. Keep your history on your own hardware, find messages @@ -25,9 +25,8 @@ and files, and connect the addresses and handles that belong to the same person. Use the browser, terminal, CLI, HTTP API, or an AI assistant through MCP. > **Alpha software.** APIs, storage format, and CLI flags may change. Back up -> your data. This README describes current `main`; see -> [the changelog](docs/changelog.md#unreleased) for unreleased features -> and upgrade steps. +> your data. This README follows current `main`; see the +> [0.21.0 changelog](docs/changelog.md#0210) for features and upgrade steps. ## What you can do @@ -56,7 +55,6 @@ for the choices and consent steps. ## Installation - **macOS / Linux:** ```bash curl -fsSL https://msgvault.io/install.sh | bash @@ -121,9 +119,10 @@ first sync, and running on your own server. | I want to… | Read | |---|---| | Understand the product | [Product overview](https://msgvault.io/) and [archive lifecycle](https://msgvault.io/guide/) | -| Catch up after 0.19 | [Changelog and upgrade notes](docs/changelog.md#unreleased) | +| Upgrade an existing archive | [Changelog and upgrade notes](docs/changelog.md#0210) | | Search messages and attachments | [Searching](docs/usage/searching.md) and [document indexing](docs/usage/document-indexing.md) | | Maintain contacts and relationships | [People and profiles](docs/usage/people.md) | +| Import local text-message history | [Text message imports](docs/usage/text-messages.md) | | Configure optional AI features | [Recommended configuration](docs/usage/recommended-configuration.md) | | Run msgvault on a server | [Remote deployment](docs/guides/remote-deployment.md) | | Back up or free mailbox space | [Backup](docs/usage/backup.md) and [deleting email](docs/usage/deletion.md) | diff --git a/api/openapi.yaml b/api/openapi.yaml index 3eeac289a..20bbe7cab 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -46,16 +46,121 @@ components: next_run: format: date-time type: string + pending: + type: boolean + queued: + type: boolean running: type: boolean schedule: type: string + started_at: + format: date-time + type: string required: - email - running - next_run - schedule type: object + Action: + additionalProperties: true + properties: + assignee_email: + type: string + assignee_name: + type: string + description: + type: string + due_date: + type: string + locator: + type: string + ordinal: + format: int64 + type: integer + origin: + type: string + source_id: + type: string + source_status: + type: string + status: + type: string + title: + type: string + required: + - ordinal + - title + - status + - origin + - locator + type: object + ActionCoverage: + additionalProperties: true + properties: + available: + format: int64 + type: integer + meeting_count: + format: int64 + type: integer + partial: + format: int64 + type: integer + unavailable: + format: int64 + type: integer + unsupported: + format: int64 + type: integer + required: + - meeting_count + - available + - partial + - unsupported + - unavailable + type: object + ActionRow: + additionalProperties: true + properties: + action: + $ref: "#/components/schemas/Action" + meeting: + $ref: "#/components/schemas/MeetingRef" + required: + - meeting + - action + type: object + ActionsPage: + additionalProperties: true + properties: + archive_uid: + type: string + coverage: + $ref: "#/components/schemas/ActionCoverage" + next_cursor: + type: string + rows: + items: + $ref: "#/components/schemas/ActionRow" + type: array + schema_version: + format: int64 + type: integer + scope: + $ref: "#/components/schemas/ScopeProvenance" + total_count: + format: int64 + type: integer + required: + - schema_version + - archive_uid + - rows + - total_count + - coverage + - scope + type: object ActivityRef: additionalProperties: true properties: @@ -182,6 +287,12 @@ components: items: type: string type: array + sender_selections: + additionalProperties: + items: + type: string + type: array + type: object source_ids: items: format: int64 @@ -241,12 +352,17 @@ components: type: integer identifier: type: string + sender_keys: + items: + type: string + type: array type: type: string required: - id - type - identifier + - sender_keys type: object AgentTokenView: additionalProperties: true @@ -627,6 +743,22 @@ components: BackupFreezeEndResponse: additionalProperties: true type: object + BasisTotals: + additionalProperties: true + properties: + basis: + type: string + count: + format: int64 + type: integer + total_seconds: + format: double + type: number + required: + - basis + - count + - total_seconds + type: object BodySearchContext: additionalProperties: true properties: @@ -674,6 +806,8 @@ components: type: string oauth_app_explicit: type: boolean + write: + type: boolean required: - email type: object @@ -827,6 +961,8 @@ components: type: string scope_escalation_oauth_app: type: string + scope_escalation_source_type: + type: string stdout: type: string required: @@ -1037,6 +1173,58 @@ components: required: - type type: object + CacheBuildAccepted: + additionalProperties: true + properties: + cache: + $ref: "#/components/schemas/CacheFreshness" + job_id: + type: string + status: + type: string + required: + - status + - job_id + type: object + CacheBuildStatus: + additionalProperties: true + properties: + accepted_at: + format: date-time + type: string + error: + type: string + finished_at: + format: date-time + type: string + job_id: + type: string + status: + type: string + required: + - job_id + - status + - accepted_at + type: object + CacheFreshness: + additionalProperties: true + properties: + building: + type: boolean + generation: + type: string + pending_additions: + format: int64 + type: integer + published_at: + format: date-time + type: string + stale_reason: + type: string + required: + - generation + - published_at + type: object CacheStats: additionalProperties: true properties: @@ -1079,97 +1267,301 @@ components: required: - status type: object - CancelDeletionResponse: + CalendarConflict: additionalProperties: true properties: - id: + calendar_ids: + items: + type: string + type: array + end: + format: date-time type: string - status: + start: + format: date-time type: string required: - - id - - status + - calendar_ids + - start + - end type: object - Candidate: + CalendarEventTarget: additionalProperties: true properties: - already_confirmed: - type: boolean - classification: - enum: - - confirmed - - strong - - weak + start: + $ref: "#/components/schemas/GCalEventDateTime" + summary: type: string - first_seen_at: - format: date-time + required: + - start + type: object + CalendarPlannedWrite: + additionalProperties: true + properties: + action: type: string - identifier: + calendar_id: type: string - last_seen_at: - format: date-time + destination: type: string - normalized_identifier: + event: + $ref: "#/components/schemas/GCalEventInput" + event_id: type: string - provider_states: + target: + $ref: "#/components/schemas/CalendarEventTarget" + required: + - action + - calendar_id + type: object + CalendarRequest: + additionalProperties: false + not: + properties: + action: + enum: + - move + - respond + type: string + scope: + enum: + - future + type: string + required: + - action + - scope + type: object + properties: + account: + type: string + action: + enum: + - create + - update + - delete + - move + - respond + - freebusy + - conflicts + type: string + add_attendees: items: type: string type: array - received_message_count: - format: int64 - type: integer - sent_message_count: - format: int64 - type: integer - signals: + calendar_id: + description: Required for event mutations. For freebusy or conflicts, use this as the target or omit it and provide calendar_ids. + type: string + calendar_ids: + description: Availability target IDs. When nonempty, only these calendars are checked; calendar_id may be omitted. items: type: string type: array - required: - - identifier - - normalized_identifier - - classification - - already_confirmed - - signals - - provider_states - - sent_message_count - - received_message_count - - first_seen_at - - last_seen_at - type: object - CardDAVAccountRequest: - additionalProperties: false - properties: - base_url: + destination: type: string - enabled: + dry_run: type: boolean - oauth_app: + event: + $ref: "#/components/schemas/GCalEventInput" + event_id: type: string - password: + expected_plan_fingerprint: + description: Optional precondition from a prior dry run; rejects the mutation if the OAuth account, planned writes, or normalized notification mode differs. type: string - writeOnly: true - provider: + original_start: + type: string + read_only: + type: boolean + response: enum: - - "" - - google + - accepted + - declined + - tentative type: string - schedule: + scope: + enum: + - single + - future + - all type: string - username: + send_updates: + enum: + - none + - all + - externalOnly + type: string + time_max: + format: date-time + type: string + time_min: + format: date-time + type: string + time_zone: type: string required: - - base_url - - username - - enabled + - action + - account type: object - CardDAVAccountResponse: + CalendarResult: additionalProperties: true properties: - base_url: + account: type: string - books: - format: int64 + calendar_id: + type: string + conflicts: + items: + $ref: "#/components/schemas/CalendarConflict" + type: array + dry_run: + type: boolean + error: + type: string + freebusy: + $ref: "#/components/schemas/GCalFreeBusyResponse" + outcome_code: + description: Machine-readable classification for a partial provider write + enum: + - calendar_partial + - calendar_outcome_unknown + type: string + outcome_unknown: + type: boolean + plan: + items: + $ref: "#/components/schemas/CalendarPlannedWrite" + type: array + plan_fingerprint: + description: Opaque comparison token bound to the OAuth account, planned writes, and normalized send_updates; valid within one daemon process + type: string + send_updates: + type: string + writes: + items: + $ref: "#/components/schemas/CalendarWriteReceipt" + type: array + required: + - account + - calendar_id + - send_updates + - dry_run + - plan + - writes + type: object + CalendarWriteReceipt: + additionalProperties: true + properties: + action: + type: string + archive_error: + type: string + archived: + type: boolean + calendar_id: + type: string + event: + $ref: "#/components/schemas/GCalEvent" + message_id: + format: int64 + type: integer + required: + - action + - calendar_id + - event + - archived + type: object + CancelDeletionResponse: + additionalProperties: true + properties: + id: + type: string + status: + type: string + required: + - id + - status + type: object + Candidate: + additionalProperties: true + properties: + already_confirmed: + type: boolean + classification: + enum: + - confirmed + - strong + - weak + type: string + first_seen_at: + format: date-time + type: string + identifier: + type: string + last_seen_at: + format: date-time + type: string + normalized_identifier: + type: string + provider_states: + items: + type: string + type: array + received_message_count: + format: int64 + type: integer + sent_message_count: + format: int64 + type: integer + signals: + items: + type: string + type: array + required: + - identifier + - normalized_identifier + - classification + - already_confirmed + - signals + - provider_states + - sent_message_count + - received_message_count + - first_seen_at + - last_seen_at + type: object + CardDAVAccountRequest: + additionalProperties: false + properties: + base_url: + type: string + connection: + type: string + enabled: + type: boolean + oauth_app: + type: string + password: + type: string + writeOnly: true + provider: + enum: + - "" + - google + type: string + schedule: + type: string + username: + type: string + required: + - base_url + - username + - enabled + type: object + CardDAVAccountResponse: + additionalProperties: true + properties: + base_url: + type: string + books: + format: int64 type: integer enabled: type: boolean @@ -1203,6 +1595,11 @@ components: CardDAVBookResponse: additionalProperties: true properties: + account_id: + format: int64 + type: integer + connection: + type: string id: format: int64 type: integer @@ -1379,6 +1776,37 @@ components: required: - conflicts type: object + CardDAVConnectionResponse: + additionalProperties: true + properties: + account_id: + format: int64 + type: integer + connection: + type: string + oauth_app: + type: string + orphaned: + type: boolean + provider: + type: string + status: + $ref: "#/components/schemas/CardDAVStatusResponse" + required: + - connection + - orphaned + - status + type: object + CardDAVConnectionsResponse: + additionalProperties: true + properties: + connections: + items: + $ref: "#/components/schemas/CardDAVConnectionResponse" + type: array + required: + - connections + type: object CardDAVContactSummaryResponse: additionalProperties: true properties: @@ -1408,6 +1836,8 @@ components: CardDAVGoogleAuthorizeRequest: additionalProperties: false properties: + connection: + type: string email: type: string oauth_app: @@ -1421,6 +1851,8 @@ components: CardDAVGoogleAuthorizeResponse: additionalProperties: true properties: + connection: + type: string state: type: string url: @@ -1532,9 +1964,14 @@ components: CardDAVRunResponse: additionalProperties: true properties: + account_id: + format: int64 + type: integer books: format: int64 type: integer + connection: + type: string created: format: int64 type: integer @@ -1583,6 +2020,7 @@ components: format: int64 type: integer required: + - account_id - id - trigger - full @@ -1663,6 +2101,8 @@ components: CardDAVSyncRequest: additionalProperties: false properties: + connection: + type: string full: type: boolean type: object @@ -1823,6 +2263,11 @@ components: items: $ref: "#/components/schemas/CliAccountResponse" type: array + as_of: + format: date-time + type: string + stale: + type: boolean required: - accounts type: object @@ -2144,6 +2589,18 @@ components: - body_text - body_html type: object + CliOriginalMessageResponse: + additionalProperties: true + properties: + message: + $ref: "#/components/schemas/MessageRecord" + mime: + contentEncoding: base64 + type: string + required: + - message + - mime + type: object CliRebuildFTSEvent: additionalProperties: true properties: @@ -2198,6 +2655,25 @@ components: required: - stats type: object + CodexModel: + additionalProperties: true + properties: + default_reasoning_effort: + type: string + display_name: + type: string + id: + type: string + supported_efforts: + items: + type: string + type: array + required: + - id + - display_name + - default_reasoning_effort + - supported_efforts + type: object CommunicationService: additionalProperties: true properties: @@ -2267,6 +2743,47 @@ components: required: - reported_score type: object + ConnectionSyncOutcome: + additionalProperties: true + properties: + account_id: + format: int64 + type: integer + books: + format: int64 + type: integer + connection: + type: string + created: + format: int64 + type: integer + error_code: + type: string + error_message: + type: string + removed: + format: int64 + type: integer + run_id: + format: int64 + type: integer + status: + enum: + - succeeded + - partial + - failed + type: string + updated: + format: int64 + type: integer + required: + - connection + - status + - books + - created + - updated + - removed + type: object ContactState: additionalProperties: true properties: @@ -2480,6 +2997,10 @@ components: CreatePersonRequest: additionalProperties: false properties: + display_name: + type: + - string + - "null" participant_id: format: int64 type: integer @@ -2659,11 +3180,15 @@ components: - activity - activity_truncated type: object - DecideIdentityMatchRequest: + DecideIdentityMatchReviewedRequest: additionalProperties: false properties: notes: type: string + review_token: + type: string + required: + - review_token type: object DecidePersonMergeCandidateRequest: additionalProperties: false @@ -2845,6 +3370,29 @@ components: - categories - organizations type: object + Disclosure: + additionalProperties: true + properties: + endpoint: + type: string + model_id: + type: string + packet_schema: + type: string + policy_version: + type: string + question_version: + type: string + retention_declaration: + type: string + required: + - endpoint + - model_id + - packet_schema + - retention_declaration + - policy_version + - question_version + type: object DiscoverError: additionalProperties: true properties: @@ -2945,6 +3493,22 @@ components: - rejected - applied type: object + DocumentFailureDiagnostic: + additionalProperties: true + properties: + canonical_blob_hash: + type: string + detail: + type: string + reason_code: + type: string + state: + type: string + required: + - canonical_blob_hash + - reason_code + - state + type: object DocumentIndexRebuildStatus: additionalProperties: true properties: @@ -2981,6 +3545,12 @@ components: failed_attempts: format: int64 type: integer + failures: + items: + $ref: "#/components/schemas/DocumentFailureDiagnostic" + type: array + failures_exhausted: + type: boolean ineligible_role_occurrences: format: int64 type: integer @@ -3034,7 +3604,9 @@ components: format: int64 type: integer required: - - profile_exists + - failures + - failures_exhausted + - profile_exists - profile_enabled - exact_consent - extraction_attempts @@ -3558,6 +4130,33 @@ components: - provider_idempotent - detail type: object + DurationTotals: + additionalProperties: true + properties: + average_known_seconds: + format: double + type: + - number + - "null" + known_duration_count: + format: int64 + type: integer + meeting_count: + format: int64 + type: integer + total_known_seconds: + format: double + type: number + unknown_duration_count: + format: int64 + type: integer + required: + - meeting_count + - known_duration_count + - unknown_duration_count + - total_known_seconds + - average_known_seconds + type: object Employment: additionalProperties: true properties: @@ -4737,6 +5336,222 @@ components: - retryable - bytes type: object + GCalAttendee: + additionalProperties: false + properties: + additionalGuests: + format: int64 + type: integer + comment: + type: string + displayName: + type: string + email: + type: string + optional: + type: boolean + organizer: + description: Provider-controlled attendee role; do not send in event input. + readOnly: true + type: boolean + resource: + type: boolean + responseStatus: + description: Provider-controlled RSVP state; use the self RSVP operation to change your response. + readOnly: true + type: string + self: + description: Provider-controlled marker for the authenticated attendee; do not send in event input. + readOnly: true + type: boolean + type: object + GCalBusyPeriod: + additionalProperties: true + properties: + end: + format: date-time + type: string + start: + format: date-time + type: string + required: + - start + - end + type: object + GCalCalendarBusy: + additionalProperties: true + properties: + busy: + items: + $ref: "#/components/schemas/GCalBusyPeriod" + type: array + errors: + items: + $ref: "#/components/schemas/GCalCalendarError" + type: array + required: + - busy + type: object + GCalCalendarError: + additionalProperties: true + properties: + domain: + type: string + reason: + type: string + required: + - reason + type: object + GCalEvent: + additionalProperties: true + properties: + attendees: + items: + $ref: "#/components/schemas/GCalAttendee" + type: array + created: + format: date-time + type: string + creator: + $ref: "#/components/schemas/GCalPerson" + description: + type: string + end: + $ref: "#/components/schemas/GCalEventDateTime" + etag: + type: string + eventType: + type: string + hangoutLink: + type: string + htmlLink: + type: string + iCalUID: + type: string + id: + type: string + location: + type: string + organizer: + $ref: "#/components/schemas/GCalPerson" + originalStartTime: + $ref: "#/components/schemas/GCalEventDateTime" + recurrence: + items: + type: string + type: array + recurringEventId: + type: string + reminders: + $ref: "#/components/schemas/GCalReminders" + sequence: + format: int64 + type: integer + start: + $ref: "#/components/schemas/GCalEventDateTime" + status: + type: string + summary: + type: string + transparency: + type: string + updated: + format: date-time + type: string + visibility: + type: string + type: object + GCalEventDateTime: + additionalProperties: false + properties: + date: + type: string + dateTime: + format: date-time + type: string + timeZone: + type: string + type: object + GCalEventInput: + additionalProperties: false + properties: + attendees: + items: + $ref: "#/components/schemas/GCalAttendee" + type: array + attendeesOmitted: + description: Internal self-RSVP marker returned in plans. Do not send this field in a control request. + readOnly: true + type: boolean + description: + description: Omit to preserve the existing value; use an empty string to clear it on update. JSON null is not accepted. + type: string + end: + $ref: "#/components/schemas/GCalEventDateTime" + id: + description: Server-assigned event ID returned in plans. Do not send this field in a control request. + readOnly: true + type: string + location: + description: Omit to preserve the existing value; use an empty string to clear it on update. JSON null is not accepted. + type: string + recurrence: + items: + type: string + type: array + reminders: + $ref: "#/components/schemas/GCalReminders" + start: + $ref: "#/components/schemas/GCalEventDateTime" + summary: + description: Omit to preserve the existing value; use an empty string to clear it on update. JSON null is not accepted. + type: string + type: object + GCalFreeBusyResponse: + additionalProperties: true + properties: + calendars: + additionalProperties: + $ref: "#/components/schemas/GCalCalendarBusy" + type: object + required: + - calendars + type: object + GCalPerson: + additionalProperties: true + properties: + displayName: + type: string + email: + type: string + self: + type: boolean + type: object + GCalReminder: + additionalProperties: false + properties: + method: + type: string + minutes: + format: int64 + type: integer + required: + - method + - minutes + type: object + GCalReminders: + additionalProperties: false + properties: + overrides: + items: + $ref: "#/components/schemas/GCalReminder" + type: array + useDefault: + type: boolean + required: + - useDefault + - overrides + type: object GenerationSummary: additionalProperties: true properties: @@ -4919,6 +5734,8 @@ components: HybridSearchResponse: additionalProperties: true properties: + accelerator: + type: string generation: $ref: "#/components/schemas/HybridGenerationSummary" has_more: @@ -4941,6 +5758,8 @@ components: scope_source_count: format: int64 type: integer + timings: + $ref: "#/components/schemas/HybridSearchTimings" took_ms: format: int64 type: integer @@ -4952,8 +5771,26 @@ components: - has_more - generation - took_ms + - timings - results type: object + HybridSearchTimings: + additionalProperties: true + properties: + hydration_ms: + format: int64 + type: integer + query_embedding_ms: + format: int64 + type: integer + retrieval_ms: + format: int64 + type: integer + required: + - query_embedding_ms + - retrieval_ms + - hydration_ms + type: object IdentityConfirmationOutcome: additionalProperties: true properties: @@ -5019,8 +5856,14 @@ components: IdentityMatchCandidate: additionalProperties: true properties: + actionable: + type: boolean + application_pending: + type: boolean basis: type: string + blocker: + type: string confidence: format: double type: number @@ -5044,15 +5887,21 @@ components: type: integer left_kind: type: string + left_person: + $ref: "#/components/schemas/IdentityMatchPersonBinding" normalized_value: type: string notes: type: string + review_token: + type: string right_id: format: int64 type: integer right_kind: type: string + right_person: + $ref: "#/components/schemas/IdentityMatchPersonBinding" scope_kind: type: string scope_value: @@ -5063,6 +5912,10 @@ components: type: string source_ref: type: string + source_support: + items: + $ref: "#/components/schemas/IdentityMatchSourceSupport" + type: array state: type: string updated_at: @@ -5078,6 +5931,8 @@ components: - state - source - evidence + - actionable + - application_pending - created_at - updated_at type: object @@ -5119,6 +5974,10 @@ components: type: integer source: type: string + source_support: + items: + $ref: "#/components/schemas/IdentityMatchSourceSupport" + type: array required: - id - candidate_id @@ -5126,6 +5985,67 @@ components: - source - created_at type: object + IdentityMatchJudgment: + additionalProperties: true + properties: + blockers: + items: + type: string + type: array + candidate_id: + format: int64 + type: integer + created_at: + format: date-time + type: string + error_class: + type: string + fingerprint: + type: string + id: + format: int64 + type: integer + model_id: + type: string + outcome: + type: string + policy_version: + type: string + probability: + format: double + type: number + question_version: + type: string + retry_after: + format: date-time + type: string + status: + type: string + required: + - id + - candidate_id + - fingerprint + - model_id + - question_version + - policy_version + - blockers + - outcome + - status + - created_at + type: object + IdentityMatchPersonBinding: + additionalProperties: true + properties: + person_id: + format: int64 + type: integer + revision: + format: int64 + type: integer + required: + - person_id + - revision + type: object IdentityMatchRejectResponse: additionalProperties: true properties: @@ -5144,6 +6064,18 @@ components: - identity_revision - cache_state type: object + IdentityMatchSourceSupport: + additionalProperties: true + properties: + is_conservative: + type: boolean + source_id: + format: int64 + type: integer + required: + - source_id + - is_conservative + type: object IdentitySearchHTTPRequest: additionalProperties: false properties: @@ -5443,6 +6375,11 @@ components: - transcript_segments type: object properties: + action_items: + items: + $ref: "#/components/schemas/MeetingActionItem" + maxItems: 1000 + type: array attendees: items: $ref: "#/components/schemas/MeetingPerson" @@ -5478,30 +6415,153 @@ components: - external_id - started_at type: object - MeetingImportRequest: + MeetingActionItem: additionalProperties: false properties: - meeting: - $ref: "#/components/schemas/Meeting" - source: - $ref: "#/components/schemas/Source" - required: - - source - - meeting - type: object - MeetingImportResponse: - additionalProperties: true - properties: - message_id: - format: int64 - type: integer - source_id: - format: int64 - type: integer - source_message_id: + assignee_email: + format: email type: string - status: - enum: + assignee_name: + maxLength: 256 + type: string + description: + maxLength: 65536 + type: string + due_date: + maxLength: 256 + type: string + source_id: + maxLength: 256 + type: string + status: + maxLength: 128 + type: string + title: + maxLength: 4096 + minLength: 1 + type: string + required: + - title + type: object + MeetingActionsRequest: + additionalProperties: false + not: + required: + - scope + - explore + type: object + properties: + assignee_email: + type: string + cursor: + type: string + explore: + $ref: "#/components/schemas/MeetingExploreScope" + limit: + format: int64 + maximum: 200 + minimum: 1 + type: integer + query: + maxLength: 256 + type: string + scope: + $ref: "#/components/schemas/MeetingScopeRequest" + status: + enum: + - pending + - completed + - cancelled + - unknown + type: string + type: object + MeetingContextRequest: + additionalProperties: false + oneOf: + - properties: + message_ids: + items: + format: int64 + maximum: 9007199254740991 + minimum: 1 + type: integer + maxItems: 100 + minItems: 1 + type: array + required: + - message_ids + type: object + - properties: + selection: + $ref: "#/components/schemas/ExploreSelection" + required: + - selection + type: object + properties: + format: + enum: + - json + - markdown + type: string + include_transcript: + type: boolean + max_bytes: + format: int64 + maximum: 1048576 + minimum: 4096 + type: integer + message_ids: + items: + format: int64 + maximum: 9007199254740991 + minimum: 1 + type: integer + maxItems: 100 + minItems: 1 + type: array + selection: + $ref: "#/components/schemas/ExploreSelection" + type: object + MeetingExploreScope: + additionalProperties: false + properties: + cache_revision: + type: string + candidate_snapshot_id: + type: string + predicate: + $ref: "#/components/schemas/ExploreHTTPRequest" + search_provenance: + $ref: "#/components/schemas/SearchProvenance" + required: + - predicate + - cache_revision + - search_provenance + type: object + MeetingImportRequest: + additionalProperties: false + properties: + meeting: + $ref: "#/components/schemas/Meeting" + source: + $ref: "#/components/schemas/Source" + required: + - source + - meeting + type: object + MeetingImportResponse: + additionalProperties: true + properties: + message_id: + format: int64 + type: integer + source_id: + format: int64 + type: integer + source_message_id: + type: string + status: + enum: - created - updated type: string @@ -5511,16 +6571,162 @@ components: - message_id - source_message_id type: object + MeetingMetricsRequest: + additionalProperties: false + not: + required: + - scope + - explore + type: object + properties: + explore: + $ref: "#/components/schemas/MeetingExploreScope" + scope: + $ref: "#/components/schemas/MeetingScopeRequest" + type: object MeetingPerson: additionalProperties: false + anyOf: + - properties: + email: + format: email + type: string + required: + - email + type: object + - properties: + phone: + maxLength: 64 + type: string + required: + - phone + type: object properties: email: format: email type: string + id: + description: Stable identifier for this person in the import source + maxLength: 200 + type: string name: type: string + phone: + description: International phone number starting with + or 00; normalized to E.164 + maxLength: 64 + type: string + type: object + MeetingRef: + additionalProperties: true + properties: + archive_path: + type: string + conversation_id: + format: int64 + type: integer + message_id: + format: int64 + type: integer + occurred_at: + format: date-time + type: + - string + - "null" + source_id: + format: int64 + type: integer + source_identifier: + type: string + source_message_id: + type: string + source_type: + type: string + title: + type: string required: - - email + - message_id + - conversation_id + - source_id + - source_type + - source_identifier + - source_message_id + - title + - occurred_at + - archive_path + type: object + MeetingScopeRequest: + additionalProperties: false + allOf: + - not: + required: + - person_id + - participant_id + type: object + - not: + required: + - person_id + - participant_ids + type: object + - not: + required: + - participant_id + - participant_ids + type: object + properties: + after: + format: date-time + type: string + before: + format: date-time + type: string + deletion: + enum: + - any + - active + - deleted + type: string + domains: + items: + type: string + maxItems: 100 + minItems: 1 + type: array + message_ids: + items: + format: int64 + maximum: 9007199254740991 + minimum: 1 + type: integer + maxItems: 100 + type: array + participant_id: + format: int64 + maximum: 9007199254740991 + minimum: 1 + type: integer + participant_ids: + items: + format: int64 + maximum: 9007199254740991 + minimum: 1 + type: integer + maxItems: 100 + minItems: 1 + type: array + person_id: + format: int64 + maximum: 9007199254740991 + minimum: 1 + type: integer + source_ids: + items: + format: int64 + maximum: 9007199254740991 + minimum: 1 + type: integer + maxItems: 100 + minItems: 1 + type: array type: object MergeOrganizationBody: additionalProperties: false @@ -5644,6 +6850,38 @@ components: - page_size - messages type: object + MessageRecord: + additionalProperties: true + properties: + account: + type: string + conversation_id: + format: int64 + type: integer + last_sync_at: + format: date-time + type: + - string + - "null" + message_id: + format: int64 + type: integer + source_conversation_id: + type: string + source_id: + format: int64 + type: integer + source_message_id: + type: string + source_type: + type: string + required: + - conversation_id + - source_conversation_id + - source_id + - account + - source_type + type: object MessageSummary: additionalProperties: true properties: @@ -5708,23 +6946,78 @@ components: - has_attachments - size_bytes type: object - MutationResult: - additionalProperties: true - properties: - name: - type: string - source_count: - format: int64 - type: integer - required: - - name - type: object - NetworkEdge: + Metrics: additionalProperties: true properties: - end_date: + archive_uid: type: string - id: + duration_by_basis: + items: + $ref: "#/components/schemas/BasisTotals" + type: array + first_meeting_at: + format: date-time + type: + - string + - "null" + last_meeting_at: + format: date-time + type: + - string + - "null" + months: + items: + $ref: "#/components/schemas/MonthTotals" + type: array + schema_version: + format: int64 + type: integer + scope: + $ref: "#/components/schemas/ScopeProvenance" + totals: + $ref: "#/components/schemas/DurationTotals" + undated_count: + format: int64 + type: integer + required: + - schema_version + - archive_uid + - totals + - first_meeting_at + - last_meeting_at + - undated_count + - duration_by_basis + - months + - scope + type: object + MonthTotals: + additionalProperties: true + properties: + month: + type: string + totals: + $ref: "#/components/schemas/DurationTotals" + required: + - month + - totals + type: object + MutationResult: + additionalProperties: true + properties: + name: + type: string + source_count: + format: int64 + type: integer + required: + - name + type: object + NetworkEdge: + additionalProperties: true + properties: + end_date: + type: string + id: type: string kind: enum: @@ -5946,6 +7239,11 @@ components: OperationRunDetail: additionalProperties: false properties: + account_id: + format: int64 + type: integer + connection: + type: string counters: items: $ref: "#/components/schemas/OperationPublicCounter" @@ -6022,6 +7320,11 @@ components: OperationRunSummary: additionalProperties: false properties: + account_id: + format: int64 + type: integer + connection: + type: string counters: items: $ref: "#/components/schemas/OperationPublicCounter" @@ -7137,6 +8440,34 @@ components: - limit - offset type: object + PacketResult: + additionalProperties: true + properties: + content: + type: string + content_bytes: + format: int64 + type: integer + format: + type: string + omitted_message_ids: + items: + format: int64 + type: integer + type: array + schema_version: + format: int64 + type: integer + truncated: + type: boolean + required: + - schema_version + - format + - content + - content_bytes + - truncated + - omitted_message_ids + type: object PartialDate: additionalProperties: false properties: @@ -7356,6 +8687,261 @@ components: format: int64 type: integer type: object + PeopleCodexLoginRequest: + additionalProperties: false + properties: + name: + minLength: 1 + type: string + required: + - name + type: object + PeopleCodexLoginResponse: + additionalProperties: true + properties: + local_deadline: + format: date-time + type: string + session_id: + type: string + user_code: + type: string + verification_url: + type: string + required: + - session_id + - verification_url + - user_code + - local_deadline + type: object + PeopleCodexLoginStatusResponse: + additionalProperties: true + properties: + state: + type: string + required: + - state + type: object + PeopleCodexModelsResponse: + additionalProperties: true + properties: + models: + items: + $ref: "#/components/schemas/CodexModel" + type: array + required: + - models + type: object + PeopleCodexProfileRequest: + additionalProperties: false + properties: + allow_sensitive: + type: boolean + allowed_sources: + items: + type: string + minItems: 1 + type: array + model: + minLength: 1 + type: string + reasoning_effort: + minLength: 1 + type: string + retention_posture: + minLength: 1 + type: string + source_since: + type: string + source_until: + type: string + training_posture: + minLength: 1 + type: string + required: + - model + - reasoning_effort + - retention_posture + - training_posture + - allowed_sources + - source_since + - allow_sensitive + type: object + PeopleInferenceCheckResponse: + additionalProperties: true + properties: + fingerprint: + type: string + model: + type: string + ok: + type: boolean + usage: + $ref: "#/components/schemas/TokenUsage" + required: + - ok + - fingerprint + - model + - usage + type: object + PeopleInferenceConsentRequest: + additionalProperties: false + properties: + confirmed: + type: boolean + fingerprint: + type: string + required: + - fingerprint + - confirmed + type: object + PeopleInferenceKeyWriteRequest: + additionalProperties: false + properties: + value: + minLength: 1 + type: string + required: + - value + type: object + PeopleInferencePresetCreateRequest: + additionalProperties: false + properties: + allow_sensitive: + type: boolean + allowed_sources: + items: + type: string + minItems: 1 + type: array + model: + minLength: 1 + type: string + preset_id: + enum: + - openai + - openrouter + - venice + type: string + retention_posture: + minLength: 1 + type: string + source_since: + type: string + source_until: + type: string + training_posture: + minLength: 1 + type: string + required: + - preset_id + - model + - retention_posture + - training_posture + - allowed_sources + - source_since + - allow_sensitive + type: object + PeopleInferenceProfileSetting: + additionalProperties: true + properties: + allow_sensitive: + type: boolean + allowed_sources: + items: + type: string + type: array + checked: + type: boolean + consent_active: + type: boolean + credential_configured: + type: boolean + credential_env: + type: string + credential_revision: + type: string + credential_source: + type: string + endpoint: + type: string + fingerprint: + type: string + model: + type: string + name: + type: string + output_mode: + type: string + preset_id: + type: string + protocol: + type: string + retention_posture: + type: string + selected: + type: boolean + source_since: + type: string + source_until: + type: string + training_posture: + type: string + required: + - name + - selected + - protocol + - model + - credential_source + - credential_configured + - checked + - consent_active + - output_mode + - retention_posture + - training_posture + - allowed_sources + - source_since + - allow_sensitive + type: object + PeopleInferenceSelectionRequest: + additionalProperties: false + properties: + name: + minLength: 1 + type: string + required: + - name + type: object + PeopleInferenceSettingsResponse: + additionalProperties: true + properties: + configured_enabled: + type: boolean + configured_fingerprint: + type: string + configured_name: + type: string + pending_restart: + type: boolean + profiles: + items: + $ref: "#/components/schemas/PeopleInferenceProfileSetting" + type: array + running_enabled: + type: boolean + running_fingerprint: + type: string + running_name: + type: string + stored_credentials_supported: + type: boolean + required: + - stored_credentials_supported + - profiles + - configured_enabled + - running_enabled + - pending_restart + type: object PeopleResponse: additionalProperties: true properties: @@ -7498,13 +9084,130 @@ components: type: integer type: array type: object - PersonAttributeConflictResponse: - additionalProperties: true + PersonAgendaCreateRequest: + additionalProperties: false properties: - current_value: - $ref: "#/components/schemas/PersonAttributeValue" - current_value_id: - format: int64 + body: + type: string + labels: + items: + type: string + type: array + list: + type: string + priority: + format: int64 + type: integer + title: + type: string + required: + - title + type: object + PersonAgendaItem: + additionalProperties: true + properties: + body: + type: string + labels: + items: + type: string + type: array + list: + type: string + owner: + type: string + priority: + format: int64 + type: integer + project: + type: string + qualified_ref: + type: string + ref: + type: string + revision: + type: string + state: + type: string + status: + type: string + title: + type: string + uid: + type: string + web_url: + type: string + required: + - uid + - ref + - qualified_ref + - project + - title + - revision + - list + - status + - state + type: object + PersonAgendaLinkRequest: + additionalProperties: false + properties: + list: + type: string + ref: + type: string + required: + - ref + type: object + PersonAgendaMutationResponse: + additionalProperties: true + properties: + item: + $ref: "#/components/schemas/PersonAgendaItem" + required: + - item + type: object + PersonAgendaResult: + additionalProperties: true + properties: + items: + items: + $ref: "#/components/schemas/PersonAgendaItem" + type: array + person_uid: + description: Canonical stable person UID used for new Kata links + type: string + person_uids: + description: Canonical UID followed by retired aliases that still resolve to this person + items: + type: string + type: array + project: + type: string + truncated: + description: More open tasks exist than the agenda result limit + type: boolean + required: + - person_uid + - person_uids + - truncated + - project + - items + type: object + PersonAgendaUpdateRequest: + additionalProperties: false + properties: + list: + type: string + required: + - list + type: object + PersonAttributeConflictResponse: + additionalProperties: true + properties: + current_value: + $ref: "#/components/schemas/PersonAttributeValue" + current_value_id: + format: int64 type: integer error: type: string @@ -7833,6 +9536,10 @@ components: format: int64 type: integer type: array + members: + items: + $ref: "#/components/schemas/PersonClusterMember" + type: array required: - canonical_id - member_ids @@ -7841,6 +9548,8 @@ components: PersonClusterEdge: additionalProperties: true properties: + link_origin: + $ref: "#/components/schemas/PersonClusterLinkOrigin" participant_a: format: int64 type: integer @@ -7851,6 +9560,33 @@ components: - participant_a - participant_b type: object + PersonClusterLinkOrigin: + additionalProperties: true + properties: + basis: + type: string + kind: + type: string + source: + type: string + required: + - kind + type: object + PersonClusterMember: + additionalProperties: true + properties: + display_name: + type: string + email: + type: string + participant_id: + format: int64 + type: integer + phone: + type: string + required: + - participant_id + type: object PersonContactPoint: additionalProperties: true properties: @@ -8730,11 +10466,21 @@ components: type: string is_primary: type: boolean + participant_display_name: + type: string participant_id: format: int64 type: integer provenance: type: string + scope_kind: + type: string + scope_value: + type: string + service_label: + type: string + service_slug: + type: string type: type: string value: @@ -8800,6 +10546,120 @@ components: - sent_count - latest_at type: object + PersonMatchBatchError: + additionalProperties: true + properties: + code: + type: string + message: + type: string + required: + - code + - message + type: object + PersonMatchConsentDecisionRequest: + additionalProperties: false + properties: + disclosure_fingerprint: + type: string + required: + - disclosure_fingerprint + type: object + PersonMatchConsentDecisionResponse: + additionalProperties: true + properties: + changed: + type: boolean + consent_active: + type: boolean + disclosure_fingerprint: + type: string + required: + - disclosure_fingerprint + - consent_active + - changed + type: object + PersonMatchJudgmentHistoryResponse: + additionalProperties: true + properties: + candidate_id: + format: int64 + type: integer + judgments: + items: + $ref: "#/components/schemas/IdentityMatchJudgment" + type: array + limit: + format: int64 + type: integer + next_before_id: + format: int64 + type: integer + required: + - judgments + - limit + - candidate_id + type: object + PersonMatchScoringRequest: + additionalProperties: false + properties: + limit: + format: int64 + type: integer + type: object + PersonMatchScoringResponse: + additionalProperties: true + properties: + error: + $ref: "#/components/schemas/PersonMatchBatchError" + processed: + format: int64 + type: integer + results: + items: + $ref: "#/components/schemas/Result" + type: array + required: + - results + - processed + type: object + PersonMatchScoringStatus: + additionalProperties: true + properties: + batch_size: + format: int64 + type: integer + blocker: + type: string + consent_active: + type: boolean + credential_available: + type: boolean + data_fields: + type: string + disclosure: + $ref: "#/components/schemas/Disclosure" + disclosure_fingerprint: + type: string + enabled: + type: boolean + minimum_probability: + format: double + type: number + model_id: + type: string + ready: + type: boolean + required: + - data_fields + - enabled + - model_id + - minimum_probability + - batch_size + - credential_available + - consent_active + - ready + type: object PersonMedia: additionalProperties: true properties: @@ -9840,6 +11700,8 @@ components: QueryRequest: additionalProperties: false properties: + fresh: + type: boolean sql: type: string required: @@ -9848,6 +11710,8 @@ components: QueryResult: additionalProperties: true properties: + cache: + $ref: "#/components/schemas/CacheFreshness" columns: items: type: string @@ -10321,6 +12185,46 @@ components: - identifier - removed type: object + Result: + additionalProperties: true + properties: + blockers: + items: + type: string + type: array + candidate_id: + format: int64 + type: integer + evidence_classes: + items: + type: string + type: array + model_id: + type: string + packet_schema: + type: string + policy_version: + type: string + probability: + format: double + type: number + proposed_action: + type: string + review_token: + type: string + status: + type: string + required: + - candidate_id + - review_token + - model_id + - packet_schema + - policy_version + - evidence_classes + - proposed_action + - blockers + - status + type: object RunSavedViewRequest: additionalProperties: false properties: @@ -10535,6 +12439,23 @@ components: - running - accounts type: object + ScopeProvenance: + additionalProperties: true + properties: + cache_revision: + type: string + candidate_snapshot_id: + type: string + kind: + type: string + lexical_index_revision: + type: string + vector_generation: + format: int64 + type: integer + required: + - kind + type: object ScoreBreakdown: additionalProperties: true properties: @@ -11219,6 +13140,12 @@ components: type: boolean scheduler_last_error: type: string + scheduler_pending: + type: boolean + scheduler_queued: + type: boolean + scheduler_started_at: + type: string source_type: type: string sync_unavailable_reason: @@ -11354,12 +13281,17 @@ components: active_messages: format: int64 type: integer + as_of: + format: date-time + type: string database_size_bytes: format: int64 type: integer source_deleted_messages: format: int64 type: integer + stale: + type: boolean total_accounts: format: int64 type: integer @@ -11377,6 +13309,8 @@ components: type: integer vector_search: $ref: "#/components/schemas/StatsView" + vector_stats_unavailable: + type: boolean vector_status: type: string vector_text_message_types: @@ -11585,12 +13519,22 @@ components: books: format: int64 type: integer + connections: + items: + $ref: "#/components/schemas/ConnectionSyncOutcome" + type: array created: format: int64 type: integer removed: format: int64 type: integer + status: + enum: + - succeeded + - partial + - failed + type: string updated: format: int64 type: integer @@ -11627,14 +13571,6 @@ components: type: - string - "null" - cursor_after: - type: - - string - - "null" - cursor_before: - type: - - string - - "null" error_message: type: - string @@ -11679,8 +13615,6 @@ components: - messages_updated - errors_count - error_message - - cursor_before - - cursor_after type: object TargetDescriptor: additionalProperties: true @@ -12057,6 +13991,101 @@ components: - limit - messages type: object + ThreadMessage: + additionalProperties: true + properties: + attachment_count: + format: int64 + type: integer + cc: + items: + $ref: "#/components/schemas/Address" + type: array + deleted_from_source_at: + format: date-time + type: + - string + - "null" + from: + items: + $ref: "#/components/schemas/Address" + type: array + has_raw: + type: boolean + id: + format: int64 + type: integer + sent_at: + format: date-time + type: + - string + - "null" + source_message_id: + type: string + subject: + type: string + to: + items: + $ref: "#/components/schemas/Address" + type: array + required: + - id + - source_message_id + - subject + - from + - to + - cc + - has_raw + - attachment_count + type: object + ThreadPage: + additionalProperties: true + properties: + account: + type: string + conversation_id: + format: int64 + type: integer + has_more: + type: boolean + last_sync_at: + format: date-time + type: + - string + - "null" + message_id: + format: int64 + type: integer + messages: + items: + $ref: "#/components/schemas/ThreadMessage" + type: array + offset: + format: int64 + type: integer + source_conversation_id: + type: string + source_id: + format: int64 + type: integer + source_message_id: + type: string + source_type: + type: string + total: + format: int64 + type: integer + required: + - total + - offset + - has_more + - messages + - conversation_id + - source_conversation_id + - source_id + - account + - source_type + type: object TimelineRow: additionalProperties: true properties: @@ -12121,6 +14150,19 @@ components: required: - refresh_token type: object + TokenUsage: + additionalProperties: true + properties: + input_tokens: + format: int64 + type: integer + output_tokens: + format: int64 + type: integer + required: + - input_tokens + - output_tokens + type: object TotalStatsResponse: additionalProperties: true properties: @@ -12329,6 +14371,10 @@ components: type: string status: type: string + text_enabled: + type: boolean + visual_enabled: + type: boolean required: - status type: object @@ -12439,7 +14485,7 @@ components: type: apiKey info: title: msgvault API - version: 2.26.0 + version: 3.1.0 openapi: 3.1.0 paths: /api/ping: @@ -13321,6 +15367,121 @@ paths: summary: End a backup freeze window tags: - API + /api/v1/cache-builds/{job_id}: + get: + operationId: getCacheBuildStatus + parameters: + - description: Analytics cache build job ID + in: path + name: job_id + required: true + schema: + type: string + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/CacheBuildStatus" + description: OK + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Get analytics cache build status + tags: + - API + /api/v1/calendar/control: + post: + operationId: controlCalendar + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/CalendarRequest" + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/CalendarResult" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "413": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "415": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "502": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Control a live calendar event or query availability + tags: + - Calendar /api/v1/carddav/account: put: operationId: saveCardDAVAccount @@ -13446,6 +15607,13 @@ paths: /api/v1/carddav/books: get: operationId: listCardDAVBooks + parameters: + - description: Saved connection name; omit for all connections + in: query + name: connection + schema: + pattern: ^[a-z][a-z0-9_-]{0,63}$ + type: string responses: "200": content: @@ -13453,6 +15621,12 @@ paths: schema: $ref: "#/components/schemas/CardDAVBooksResponse" description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error "500": content: application/json: @@ -13733,31 +15907,17 @@ paths: summary: Resolve a CardDAV conflict tags: - API - /api/v1/carddav/google/authorize: - post: - description: Start sign-in from the msgvault Web UI. The Origin header must match the redirect_uri origin, and redirect_uri must be the Web UI's root URL. For terminal authorization, use msgvault carddav authorize-google. - operationId: beginGoogleCardDAVAuthorization - parameters: - - description: Origin of the msgvault Web UI, matching redirect_uri - in: header - name: Origin - required: true - schema: - type: string - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/CardDAVGoogleAuthorizeRequest" - required: true + /api/v1/carddav/connections: + get: + operationId: listCardDAVConnections responses: "200": content: application/json: schema: - $ref: "#/components/schemas/CardDAVGoogleAuthorizeResponse" + $ref: "#/components/schemas/CardDAVConnectionsResponse" description: OK - "400": + "500": content: application/json: schema: @@ -13784,12 +15944,66 @@ paths: description: Error security: - apiKey: [] - summary: Start Google Contacts authorization in a browser + summary: List saved CardDAV connections tags: - API - /api/v1/carddav/google/callback: + /api/v1/carddav/google/authorize: post: - operationId: completeGoogleCardDAVAuthorization + description: Start sign-in from the msgvault Web UI. The Origin header must match the redirect_uri origin, and redirect_uri must be the Web UI's root URL. For terminal authorization, use msgvault carddav authorize-google. + operationId: beginGoogleCardDAVAuthorization + parameters: + - description: Origin of the msgvault Web UI, matching redirect_uri + in: header + name: Origin + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/CardDAVGoogleAuthorizeRequest" + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/CardDAVGoogleAuthorizeResponse" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + headers: + Retry-After: + description: Seconds until CardDAV retry is safe + schema: + format: int64 + minimum: 0 + type: integer + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Start Google Contacts authorization in a browser + tags: + - API + /api/v1/carddav/google/callback: + post: + operationId: completeGoogleCardDAVAuthorization requestBody: content: application/json: @@ -14222,6 +16436,12 @@ paths: format: int64 minimum: 1 type: integer + - description: Saved connection name; omit for all connections + in: query + name: connection + schema: + pattern: ^[a-z][a-z0-9_-]{0,63}$ + type: string responses: "200": content: @@ -14261,6 +16481,13 @@ paths: /api/v1/carddav/status: get: operationId: getCardDAVStatus + parameters: + - description: Saved connection name; omit for all connections + in: query + name: connection + schema: + pattern: ^[a-z][a-z0-9_-]{0,63}$ + type: string responses: "200": content: @@ -14268,6 +16495,12 @@ paths: schema: $ref: "#/components/schemas/CardDAVStatusResponse" description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error "500": content: application/json: @@ -15338,6 +17571,80 @@ paths: summary: Get one message for CLI output tags: - API + /api/v1/cli/message/original: + get: + operationId: getCLIMessageOriginal + parameters: + - description: Internal message ID + in: query + name: id + schema: + format: int64 + type: integer + - description: Maximum decoded MIME bytes; omit for an unrestricted export + in: query + name: max_bytes + schema: + format: int64 + type: integer + - description: Provider message ID + in: query + name: source_message_id + schema: + type: string + - description: Source identifier that narrows the lookup + in: query + name: account + schema: + type: string + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/CliOriginalMessageResponse" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "413": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Get one message's original MIME for export + tags: + - API /api/v1/cli/message/raw: get: operationId: getCLIMessageRaw @@ -15398,6 +17705,90 @@ paths: summary: Get one raw message for CLI export tags: - API + /api/v1/cli/message/thread: + get: + operationId: getCLIMessageThread + parameters: + - description: Internal ID of a message in the conversation + in: query + name: id + schema: + format: int64 + type: integer + - description: Provider ID of a message in the conversation + in: query + name: source_message_id + schema: + type: string + - description: Provider conversation ID + in: query + name: thread_id + schema: + type: string + - description: Source identifier that narrows the lookup + in: query + name: account + schema: + type: string + - description: Return fixed membership for the entire conversation; cannot be combined with limit or offset + in: query + name: all + schema: + type: boolean + - description: Messages per page (default 100, max 500) + in: query + name: limit + schema: + format: int64 + type: integer + - description: Messages to skip + in: query + name: offset + schema: + format: int64 + type: integer + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/ThreadPage" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: List one conversation in chronological order for export + tags: + - API /api/v1/cli/rebuild-fts: post: operationId: rebuildCLIFTS @@ -15590,6 +17981,16 @@ paths: post: operationId: syncCLI parameters: + - description: Build the analytics cache after sync even inside the interval + in: query + name: build-cache + schema: + type: boolean + - description: Skip the analytics cache refresh after sync + in: query + name: no-build-cache + schema: + type: boolean - description: Account email or display name to sync in: query name: email @@ -15637,6 +18038,16 @@ paths: post: operationId: syncFullCLI parameters: + - description: Build the analytics cache after sync even inside the interval + in: query + name: build-cache + schema: + type: boolean + - description: Skip the analytics cache refresh after sync + in: query + name: no-build-cache + schema: + type: boolean - description: Account email or display name to sync in: query name: email @@ -17861,10 +20272,9 @@ paths: summary: List reviewable identity match candidates tags: - API - /api/v1/identity/match-candidates/{id}/accept: - post: - description: Accepting is the explicit user confirmation the matching policy requires. The participant link is applied through the normal identity link path, so a match spanning two curated people is refused rather than merged. - operationId: acceptIdentityMatchCandidate + /api/v1/identity/match-candidates/{id}: + get: + operationId: getIdentityMatchCandidate parameters: - description: Identity match candidate ID in: path @@ -17873,17 +20283,12 @@ paths: schema: format: int64 type: integer - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/DecideIdentityMatchRequest" responses: "200": content: application/json: schema: - $ref: "#/components/schemas/IdentityMatchAcceptResponse" + $ref: "#/components/schemas/IdentityMatchCandidate" description: OK "404": content: @@ -17891,14 +20296,6 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "409": - content: - application/json: - schema: - anyOf: - - $ref: "#/components/schemas/PersonMergeRequiredError" - - $ref: "#/components/schemas/ErrorResponse" - description: Conflict "503": content: application/json: @@ -17913,13 +20310,13 @@ paths: description: Error security: - apiKey: [] - summary: Accept an identity match candidate + summary: Get an identity match candidate for review tags: - API - /api/v1/identity/match-candidates/{id}/reject: + /api/v1/identity/match-candidates/{id}/review/accept: post: - description: A rejected suggestion is retained rather than deleted, so the same low-quality inference is not proposed again on the next import. - operationId: rejectIdentityMatchCandidate + description: Requires the exact review token from a fresh list or get response. Changed evidence returns a conflict without making a new decision. + operationId: reviewAcceptIdentityMatchCandidate parameters: - description: Identity match candidate ID in: path @@ -17932,26 +20329,35 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/DecideIdentityMatchRequest" + $ref: "#/components/schemas/DecideIdentityMatchReviewedRequest" + required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/IdentityMatchRejectResponse" + $ref: "#/components/schemas/IdentityMatchAcceptResponse" description: OK - "404": + "400": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "409": + "404": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error + "409": + content: + application/json: + schema: + anyOf: + - $ref: "#/components/schemas/PersonMergeRequiredError" + - $ref: "#/components/schemas/ErrorResponse" + description: Conflict "503": content: application/json: @@ -17966,58 +20372,58 @@ paths: description: Error security: - apiKey: [] - summary: Reject an identity match candidate + summary: Review and accept an identity match candidate tags: - API - /api/v1/identity/unlinks: + /api/v1/identity/match-candidates/{id}/review/reject: post: - operationId: unlinkIdentityParticipants + description: Requires the exact review token from a fresh list or get response. Changed evidence returns a conflict without making a new decision. + operationId: reviewRejectIdentityMatchCandidate + parameters: + - description: Identity match candidate ID + in: path + name: id + required: true + schema: + format: int64 + type: integer requestBody: content: application/json: schema: - $ref: "#/components/schemas/IdentityLinkRequest" + $ref: "#/components/schemas/DecideIdentityMatchReviewedRequest" required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/IdentityLinkResponse" + $ref: "#/components/schemas/IdentityMatchRejectResponse" description: OK - default: + "400": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - security: - - apiKey: [] - summary: Remove a link edge between two participants - tags: - - API - /api/v1/import/meeting: - post: - operationId: importMeeting - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/MeetingImportRequest" - required: true - responses: - "200": + "404": content: application/json: schema: - $ref: "#/components/schemas/MeetingImportResponse" - description: OK - "201": + $ref: "#/components/schemas/ErrorResponse" + description: Error + "409": content: application/json: schema: - $ref: "#/components/schemas/MeetingImportResponse" - description: Created + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error default: content: application/json: @@ -18026,68 +20432,84 @@ paths: description: Error security: - apiKey: [] - summary: Import one meeting + summary: Review and reject an identity match candidate tags: - API - /api/v1/imports: + /api/v1/identity/scoring/consent: post: - operationId: createImportJob + operationId: personMatchScoringConsent requestBody: content: application/json: schema: - $ref: "#/components/schemas/ImportJobRequest" + $ref: "#/components/schemas/PersonMatchConsentDecisionRequest" required: true responses: - "202": + "200": content: application/json: schema: - $ref: "#/components/schemas/ImportJobResponse" - description: Accepted + $ref: "#/components/schemas/PersonMatchConsentDecisionResponse" + description: OK "400": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "401": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "404": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error "409": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "413": + "503": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "415": + default: content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "422": + security: + - apiKey: [] + summary: Grant identity scoring consent + tags: + - API + /api/v1/identity/scoring/history: + get: + operationId: listPersonMatchJudgments + parameters: + - description: Optional candidate ID; zero lists all + in: query + name: candidate_id + schema: + format: int64 + type: integer + - description: Maximum judgments + in: query + name: limit + schema: + format: int64 + type: integer + - description: Older judgments with ID below this cursor + in: query + name: before_id + schema: + format: int64 + type: integer + responses: + "200": content: application/json: schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "500": + $ref: "#/components/schemas/PersonMatchJudgmentHistoryResponse" + description: OK + "400": content: application/json: schema: @@ -18107,33 +20529,38 @@ paths: description: Error security: - apiKey: [] - summary: Start a bounded historical import + summary: List redacted identity scoring judgments tags: - API - /api/v1/imports/{job_id}: - get: - operationId: getImportJob - parameters: - - description: Historical import job ID - in: path - name: job_id - required: true - schema: - type: string + /api/v1/identity/scoring/revoke: + post: + operationId: personMatchScoringRevoke + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/PersonMatchConsentDecisionRequest" + required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/ImportJobResponse" + $ref: "#/components/schemas/PersonMatchConsentDecisionResponse" description: OK - "401": + "400": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "404": + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": content: application/json: schema: @@ -18147,26 +20574,43 @@ paths: description: Error security: - apiKey: [] - summary: Get historical import status + summary: Withdraw identity scoring consent tags: - API - /api/v1/integrations/tasks/search: - get: - operationId: searchIntegrationTasks - parameters: - - description: Task title search within the configured project - in: query - name: q - required: true - schema: - type: string + /api/v1/identity/scoring/run: + post: + operationId: runPersonMatchScoring + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/PersonMatchScoringRequest" + required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/TaskSearchResponse" + $ref: "#/components/schemas/PersonMatchScoringResponse" description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error default: content: application/json: @@ -18175,19 +20619,25 @@ paths: description: Error security: - apiKey: [] - summary: Search tasks in the configured project + summary: Score and journal a bounded batch of identity suggestions tags: - API - /api/v1/integrations/tasks/status: + /api/v1/identity/scoring/status: get: - operationId: getTaskIntegrationStatus + operationId: getPersonMatchScoringStatus responses: "200": content: application/json: schema: - $ref: "#/components/schemas/TaskIntegrationStatusResponse" + $ref: "#/components/schemas/PersonMatchScoringStatus" description: OK + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error default: content: application/json: @@ -18196,18 +20646,24 @@ paths: description: Error security: - apiKey: [] - summary: Get task integration capability status + summary: Get identity scoring configuration and consent status tags: - API - /api/v1/integrations/tasks/test: + /api/v1/identity/unlinks: post: - operationId: testTaskIntegration + operationId: unlinkIdentityParticipants + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/IdentityLinkRequest" + required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/TaskIntegrationStatusResponse" + $ref: "#/components/schemas/IdentityLinkResponse" description: OK default: content: @@ -18217,32 +20673,31 @@ paths: description: Error security: - apiKey: [] - summary: Test task integration discovery, authentication, capabilities, and project + summary: Remove a link edge between two participants tags: - API - /api/v1/messages: - get: - operationId: listMessages - parameters: - - description: One-based page number (default 1; values below 1 are clamped to 1). Non-numeric values are rejected with 400. - in: query - name: page - schema: - format: int64 - type: integer - - description: Page size (default 20, max 100; out-of-range values are clamped). Non-numeric values are rejected with 400. - in: query - name: page_size - schema: - format: int64 - type: integer + /api/v1/import/meeting: + post: + operationId: importMeeting + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/MeetingImportRequest" + required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/MessageListResponse" + $ref: "#/components/schemas/MeetingImportResponse" description: OK + "201": + content: + application/json: + schema: + $ref: "#/components/schemas/MeetingImportResponse" + description: Created default: content: application/json: @@ -18251,31 +20706,25 @@ paths: description: Error security: - apiKey: [] - summary: List messages + summary: Import one meeting tags: - API - /api/v1/messages/changes: - get: - operationId: listChangedMessages - parameters: - - description: "Opaque cursor from the next_cursor of the previous response, sent back verbatim. Do not parse, construct, compare, or order it; its contents may change without notice. Omit, or send it empty, to start from the beginning of the archive. The token is not authenticated: the server does not sign it and cannot tell one it issued from a well-formed one you built, so a fabricated cursor naming this archive is accepted and simply moves your own position. Rejected with 400 invalid_cursor, rather than read as the beginning: a token the server cannot read, one carrying a cursor format this build does not speak, and one issued against a different archive" - in: query - name: cursor - schema: - type: string - - description: Maximum number of rows to return (default 100, max 500; values below 1 fall back to the default) - in: query - name: limit - schema: - format: int64 - type: integer + /api/v1/imports: + post: + operationId: createImportJob + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ImportJobRequest" + required: true responses: - "200": + "202": content: application/json: schema: - $ref: "#/components/schemas/ChangesResponse" - description: OK + $ref: "#/components/schemas/ImportJobResponse" + description: Accepted "400": content: application/json: @@ -18288,163 +20737,2185 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "429": + "404": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "500": + "409": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "503": + "413": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - default: + "415": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - security: - - apiKey: [] - summary: List messages whose content changed since a cursor - tags: + "422": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Start a bounded historical import + tags: + - API + /api/v1/imports/{job_id}: + get: + operationId: getImportJob + parameters: + - description: Historical import job ID + in: path + name: job_id + required: true + schema: + type: string + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/ImportJobResponse" + description: OK + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Get historical import status + tags: + - API + /api/v1/integrations/kata/status: + get: + operationId: getKataIntegrationStatus + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/TaskIntegrationStatusResponse" + description: OK + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Get Kata person agenda availability + tags: + - API + /api/v1/integrations/tasks/search: + get: + operationId: searchIntegrationTasks + parameters: + - description: Task title search within the configured project + in: query + name: q + required: true + schema: + type: string + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/TaskSearchResponse" + description: OK + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Search tasks in the configured project + tags: + - API + /api/v1/integrations/tasks/status: + get: + operationId: getTaskIntegrationStatus + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/TaskIntegrationStatusResponse" + description: OK + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Get task integration capability status + tags: + - API + /api/v1/integrations/tasks/test: + post: + operationId: testTaskIntegration + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/TaskIntegrationStatusResponse" + description: OK + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Test task integration discovery, authentication, capabilities, and project + tags: + - API + /api/v1/meetings/actions: + post: + operationId: listMeetingActionItems + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/MeetingActionsRequest" + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/ActionsPage" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "413": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "415": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: List archived meeting action items + tags: + - Meetings + /api/v1/meetings/context: + post: + operationId: getMeetingContext + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/MeetingContextRequest" + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/PacketResult" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "413": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "415": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Render bounded archived meeting context + tags: + - Meetings + /api/v1/meetings/metrics: + post: + operationId: getMeetingMetrics + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/MeetingMetricsRequest" + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/Metrics" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "413": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "415": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Get archived meeting duration metrics + tags: + - Meetings + /api/v1/messages: + get: + operationId: listMessages + parameters: + - description: One-based page number (default 1; values below 1 are clamped to 1). Non-numeric values are rejected with 400. + in: query + name: page + schema: + format: int64 + type: integer + - description: Page size (default 20, max 100; out-of-range values are clamped). Non-numeric values are rejected with 400. + in: query + name: page_size + schema: + format: int64 + type: integer + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/MessageListResponse" + description: OK + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: List messages + tags: + - API + /api/v1/messages/changes: + get: + operationId: listChangedMessages + parameters: + - description: "Opaque cursor from the next_cursor of the previous response, sent back verbatim. Do not parse, construct, compare, or order it; its contents may change without notice. Omit, or send it empty, to start from the beginning of the archive. The token is not authenticated: the server does not sign it and cannot tell one it issued from a well-formed one you built, so a fabricated cursor naming this archive is accepted and simply moves your own position. Rejected with 400 invalid_cursor, rather than read as the beginning: a token the server cannot read, one carrying a cursor format this build does not speak, and one issued against a different archive" + in: query + name: cursor + schema: + type: string + - description: Maximum number of rows to return (default 100, max 500; values below 1 fall back to the default) + in: query + name: limit + schema: + format: int64 + type: integer + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/ChangesResponse" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "429": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: List messages whose content changed since a cursor + tags: + - API + /api/v1/messages/filter: + get: + operationId: filterMessages + parameters: + - description: Sender email/address filter + in: query + name: sender + schema: + type: string + - description: Sender display-name filter + in: query + name: sender_name + schema: + type: string + - description: Recipient email/address filter + in: query + name: recipient + schema: + type: string + - description: Recipient display-name filter + in: query + name: recipient_name + schema: + type: string + - description: Domain filter + in: query + name: domain + schema: + type: string + - description: Label filter + in: query + name: label + schema: + type: string + - description: Exact case-insensitive RFC 2919 List-Id filter + in: query + name: list_id + schema: + type: string + - description: Message type filter + in: query + name: message_type + schema: + type: string + - description: Named time period + in: query + name: time_period + schema: + type: string + - description: Time bucket granularity + in: query + name: time_granularity + schema: + type: string + - description: Conversation ID + in: query + name: conversation_id + schema: + format: int64 + type: integer + - description: Source ID + in: query + name: source_id + schema: + format: int64 + type: integer + - description: Source IDs; repeat or comma-separate values + in: query + name: source_ids + schema: + items: + format: int64 + type: integer + type: array + - description: Only include messages with attachments + in: query + name: attachments_only + schema: + type: boolean + - description: Exclude deleted messages + in: query + name: hide_deleted + schema: + type: boolean + - description: Lower date/time bound (RFC3339 or YYYY-MM-DD) + in: query + name: after + schema: + type: string + - description: Upper date/time bound (RFC3339 or YYYY-MM-DD) + in: query + name: before + schema: + type: string + - description: Comma-separated aggregate view names to match empty values + in: query + name: empty_targets + schema: + type: string + - description: Zero-based row offset + in: query + name: offset + schema: + format: int64 + type: integer + - description: Maximum number of rows to return (default and max 500; larger values are clamped) + in: query + name: limit + schema: + format: int64 + type: integer + - description: "Sort field: date, size, or subject" + in: query + name: sort + schema: + type: string + - description: "Sort direction: asc or desc" + in: query + name: direction + schema: + type: string + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/FilteredMessagesResponse" + description: OK + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: List filtered messages + tags: + - API + /api/v1/messages/gmail-ids: + get: + operationId: getGmailIDsByFilter + parameters: + - description: Sender email/address filter + in: query + name: sender + schema: + type: string + - description: Sender display-name filter + in: query + name: sender_name + schema: + type: string + - description: Recipient email/address filter + in: query + name: recipient + schema: + type: string + - description: Recipient display-name filter + in: query + name: recipient_name + schema: + type: string + - description: Domain filter + in: query + name: domain + schema: + type: string + - description: Label filter + in: query + name: label + schema: + type: string + - description: Exact case-insensitive RFC 2919 List-Id filter + in: query + name: list_id + schema: + type: string + - description: Message type filter + in: query + name: message_type + schema: + type: string + - description: Named time period + in: query + name: time_period + schema: + type: string + - description: Time bucket granularity + in: query + name: time_granularity + schema: + type: string + - description: Conversation ID + in: query + name: conversation_id + schema: + format: int64 + type: integer + - description: Source ID + in: query + name: source_id + schema: + format: int64 + type: integer + - description: Source IDs; repeat or comma-separate values + in: query + name: source_ids + schema: + items: + format: int64 + type: integer + type: array + - description: Only include messages with attachments + in: query + name: attachments_only + schema: + type: boolean + - description: Exclude deleted messages + in: query + name: hide_deleted + schema: + type: boolean + - description: Lower date/time bound (RFC3339 or YYYY-MM-DD) + in: query + name: after + schema: + type: string + - description: Upper date/time bound (RFC3339 or YYYY-MM-DD) + in: query + name: before + schema: + type: string + - description: Comma-separated aggregate view names to match empty values + in: query + name: empty_targets + schema: + type: string + - description: Zero-based row offset + in: query + name: offset + schema: + format: int64 + type: integer + - description: Maximum number of rows to return (default and max 500; larger values are clamped) + in: query + name: limit + schema: + format: int64 + type: integer + - description: "Sort field: date, size, or subject" + in: query + name: sort + schema: + type: string + - description: "Sort direction: asc or desc" + in: query + name: direction + schema: + type: string + - description: Structured search query + in: query + name: q + schema: + type: string + - description: "Search mode: fast, deep, or aggregate; required with q" + in: query + name: search_mode + schema: + type: string + - description: Aggregate view type; required for aggregate search + in: query + name: view_type + schema: + type: string + - description: Displayed aggregate row key; required for aggregate search + in: query + name: aggregate_key + schema: + type: string + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/GmailIDsResponse" + description: OK + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: List Gmail message IDs matching a filter + tags: + - API + /api/v1/messages/{id}: + get: + operationId: getMessage + parameters: + - description: Message ID + in: path + name: id + required: true + schema: + format: int64 + type: integer + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/MessageDetail" + description: OK + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Get one message + tags: + - API + /api/v1/messages/{id}/inline: + get: + operationId: getMessageInlinePart + parameters: + - description: Message ID + in: path + name: id + required: true + schema: + format: int64 + type: integer + - description: Inline MIME Content-ID + in: query + name: cid + required: true + schema: + type: string + responses: + "200": + content: + application/octet-stream: + schema: + contentMediaType: application/octet-stream + format: binary + type: string + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "415": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "501": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Get an inline MIME part + tags: + - API + /api/v1/messages/{id}/tasks: + get: + operationId: listMessageTasks + parameters: + - description: Archived email message ID + in: path + name: id + required: true + schema: + format: int64 + type: integer + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/TaskLinkLookupResponse" + description: OK + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: List tasks linked to an archived email + tags: + - API + post: + operationId: createOrLinkMessageTask + parameters: + - description: Archived email message ID + in: path + name: id + required: true + schema: + format: int64 + type: integer + - description: Browser-generated retry-stable request ID + in: header + name: X-Request-Id + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/TaskLinkMutationRequest" + required: true + responses: + "201": + content: + application/json: + schema: + $ref: "#/components/schemas/TaskLinkMutationResponse" + description: Created + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Create or link a task for an archived email + tags: + - API + /api/v1/messages/{id}/tasks/{task_id}: + delete: + operationId: unlinkMessageTask + parameters: + - description: Archived email message ID + in: path + name: id + required: true + schema: + format: int64 + type: integer + - description: External task ID + in: path + name: task_id + required: true + schema: + type: string + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/TaskLinkMutationResponse" + description: OK + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Unlink a task from an archived email + tags: + - API + /api/v1/multimodal/build: + post: + operationId: startVisualAttachmentBuild + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/VisualBuildRequest" + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/Status" + description: OK + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Consent and run one bounded visual attachment embedding pass + tags: + - API + /api/v1/multimodal/retire: + post: + operationId: retireVisualAttachmentGeneration + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/VisualRetireRequest" + required: true + responses: + "204": + description: No Content + default: + description: Error + security: + - apiKey: [] + summary: Retire the visual attachment generation + tags: + - Search + /api/v1/multimodal/retry: + post: + operationId: retryVisualAttachmentOwner + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/VisualRetryRequest" + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/Status" + description: OK + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Retry one visual attachment owner + tags: + - API + /api/v1/multimodal/run: + post: + operationId: resumeVisualAttachmentBuild + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/Status" + description: OK + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Resume one bounded visual attachment embedding pass + tags: + - API + /api/v1/multimodal/status: + get: + operationId: getVisualAttachmentStatus + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/Status" + description: OK + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Get visual attachment embedding status + tags: + - API + /api/v1/operations/runs: + get: + operationId: listOperationRuns + parameters: + - description: Exact operation kind + in: query + name: kind + schema: + enum: + - carddav_sync + - document_embedding + - document_extraction + - message_embedding + - person_embedding + - person_enrichment + - person_sweep + - source_sync + - visual_embedding + type: string + - description: Exact semantic operation lane + in: query + name: lane + schema: + enum: + - contacts + - documents + - messages + - person_facts + - visual_attachments + type: string + - description: Exact operation state + in: query + name: state + schema: + enum: + - cancelled + - failed + - partial + - queued + - running + - succeeded + type: string + - description: Inclusive canonical UTC RFC3339 lower bound + in: query + name: started_from + schema: + format: date-time + type: string + - description: Exclusive canonical UTC RFC3339 upper bound + in: query + name: started_before + schema: + format: date-time + type: string + - description: Maximum runs to return (default 25, max 100) + in: query + name: limit + schema: + format: int64 + maximum: 100 + minimum: 1 + type: integer + - description: Opaque cursor bound to this archive and the complete normalized filter set + in: query + name: cursor + schema: + type: string + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/OperationRunsResponse" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/OperationErrorResponse" + description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/OperationErrorResponse" + description: Error + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/OperationErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/OperationErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/OperationErrorResponse" + description: Error + security: + - apiKey: [] + summary: List normalized operation history + tags: + - API + /api/v1/operations/runs/{id}: + get: + operationId: getOperationRun + parameters: + - description: Opaque archive-bound operation run ID + in: path + name: id + required: true + schema: + type: string + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/OperationRunDetail" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/OperationErrorResponse" + description: Error + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/OperationErrorResponse" + description: Error + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/OperationErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/OperationErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/OperationErrorResponse" + description: Error + security: + - apiKey: [] + summary: Get one normalized operation run + tags: + - API + /api/v1/operations/status: + get: + operationId: getOperationStatus + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/OperationStatusResponse" + description: OK + default: + content: + application/json: + schema: + $ref: "#/components/schemas/OperationErrorResponse" + description: Error + security: + - apiKey: [] + summary: Get normalized operation lane status + tags: + - API + /api/v1/organizations: + get: + operationId: listOrganizations + parameters: + - description: Maximum results + in: query + name: limit + schema: + format: int64 + type: integer + - description: Results to skip + in: query + name: offset + schema: + format: int64 + type: integer + - description: Include retired organizations + in: query + name: include_retired + schema: + type: boolean + - description: Normalized-name search + in: query + name: q + schema: + type: string + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/OrganizationsResponse" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: List organizations + tags: + - API + post: + operationId: createOrganization + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/OrganizationCreateBody" + required: true + responses: + "201": + content: + application/json: + schema: + $ref: "#/components/schemas/Organization" + description: Created + headers: + ETag: + description: Strong organization revision tag for optimistic concurrency + schema: + type: string + Location: + description: Canonical URL of the created organization + schema: + type: string + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Create an organization + tags: + - API + /api/v1/organizations/{id}: + delete: + operationId: deleteOrganization + parameters: + - description: Organization ID + in: path + name: id + required: true + schema: + format: int64 + type: integer + - description: Strong ETag returned by the latest organization read + in: header + name: If-Match + required: true + schema: + type: string + responses: + "204": + description: No Content + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "428": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Delete an organization without employment records + tags: + - API + get: + operationId: getOrganization + parameters: + - description: Organization ID + in: path + name: id + required: true + schema: + format: int64 + type: integer + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/OrganizationProfile" + description: OK + headers: + ETag: + description: Strong organization revision tag for optimistic concurrency + schema: + type: string + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Get an organization + tags: + - API + patch: + operationId: patchOrganization + parameters: + - description: Organization ID + in: path + name: id + required: true + schema: + format: int64 + type: integer + - description: Strong ETag returned by the latest organization read + in: header + name: If-Match + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/OrganizationBody" + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/Organization" + description: OK + headers: + ETag: + description: Strong organization revision tag for optimistic concurrency + schema: + type: string + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "428": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Replace an organization's mutable fields + tags: + - API + /api/v1/organizations/{id}/attributes: + get: + operationId: listOrganizationAttributes + parameters: + - description: Organization ID + in: path + name: id + required: true + schema: + format: int64 + type: integer + - description: Include superseded values + in: query + name: include_superseded + schema: + type: boolean + - description: Restrict to one definition + in: query + name: definition_slug + schema: + type: string + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/OrganizationAttributesResponse" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: List organization typed attributes + tags: + - API + post: + operationId: setOrganizationAttribute + parameters: + - description: Organization ID + in: path + name: id + required: true + schema: + format: int64 + type: integer + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/SetOrganizationAttributeBody" + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/OrganizationAttributeWrite" + description: OK + "201": + content: + application/json: + schema: + $ref: "#/components/schemas/OrganizationAttributeWrite" + description: Created + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Set an organization typed attribute + tags: - API - /api/v1/messages/filter: - get: - operationId: filterMessages + /api/v1/organizations/{id}/attributes/{slug}: + delete: + operationId: clearOrganizationAttribute parameters: - - description: Sender email/address filter - in: query - name: sender - schema: - type: string - - description: Sender display-name filter - in: query - name: sender_name + - description: Organization ID + in: path + name: id + required: true schema: - type: string - - description: Recipient email/address filter - in: query - name: recipient + format: int64 + type: integer + - description: Immutable attribute definition slug + in: path + name: slug + required: true schema: type: string - - description: Recipient display-name filter + - description: Ordinal for a multi-valued definition in: query - name: recipient_name + name: ordinal schema: - type: string - - description: Domain filter + format: int64 + type: integer + - description: "Compare-and-swap: the current value ID expected to be superseded" in: query - name: domain + name: expected_value_id schema: - type: string - - description: Label filter + format: int64 + type: integer + - description: Validate and preview without writing in: query - name: label + name: dry_run schema: - type: string - - description: Exact case-insensitive RFC 2919 List-Id filter - in: query - name: list_id + type: boolean + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/OrganizationAttributeWrite" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Supersede an organization typed attribute + tags: + - API + /api/v1/organizations/{id}/employments: + get: + operationId: listOrganizationEmployments + parameters: + - description: Organization ID + in: path + name: id + required: true schema: - type: string - - description: Message type filter + format: int64 + type: integer + - description: Only current employments in: query - name: message_type + name: current_only schema: - type: string - - description: Named time period + type: boolean + - description: Maximum results in: query - name: time_period + name: limit schema: - type: string - - description: Time bucket granularity + format: int64 + type: integer + - description: Results to skip in: query - name: time_granularity + name: offset schema: - type: string - - description: Conversation ID - in: query - name: conversation_id + format: int64 + type: integer + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/EmploymentsResponse" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: List an organization's employment records + tags: + - API + /api/v1/organizations/{id}/history: + get: + operationId: getOrganizationHistory + parameters: + - description: Organization ID + in: path + name: id + required: true schema: format: int64 type: integer - - description: Source ID - in: query - name: source_id + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/OrganizationProfile" + description: OK + headers: + ETag: + description: Strong organization revision tag for optimistic concurrency + schema: + type: string + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Get organization profile history + tags: + - API + /api/v1/organizations/{id}/merge: + post: + operationId: mergeOrganization + parameters: + - description: Organization ID + in: path + name: id + required: true schema: format: int64 type: integer - - description: Source IDs; repeat or comma-separate values - in: query - name: source_ids - schema: - items: - format: int64 - type: integer - type: array - - description: Only include messages with attachments - in: query - name: attachments_only - schema: - type: boolean - - description: Exclude deleted messages - in: query - name: hide_deleted - schema: - type: boolean - - description: Lower date/time bound (RFC3339 or YYYY-MM-DD) - in: query - name: after + - description: Strong ETag returned by the latest organization read + in: header + name: If-Match + required: true schema: type: string - - description: Upper date/time bound (RFC3339 or YYYY-MM-DD) - in: query - name: before + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/MergeOrganizationBody" + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/Organization" + description: OK + headers: + ETag: + description: Strong organization revision tag for optimistic concurrency + schema: + type: string + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "428": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Merge another organization into this organization + tags: + - API + /api/v1/organizations/{id}/profile: + put: + description: Replaces all structured organization profile collections with at most 200 total values. + operationId: putOrganizationProfile + parameters: + - description: Organization ID + in: path + name: id + required: true schema: - type: string - - description: Comma-separated aggregate view names to match empty values - in: query - name: empty_targets + format: int64 + type: integer + - description: Strong ETag returned by the latest organization read + in: header + name: If-Match + required: true schema: type: string - - description: Zero-based row offset - in: query - name: offset + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/OrganizationProfileBody" + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/OrganizationProfile" + description: OK + headers: + ETag: + description: Strong organization revision tag for optimistic concurrency + schema: + type: string + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "413": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "428": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Replace organization profile collections + tags: + - API + /api/v1/organizations/{id}/profile/media/{media_id}/content: + get: + description: Returns the exact inline bytes stored for one media value. URI-only values have no local content and return 404. + operationId: getOrganizationProfileMediaContent + parameters: + - description: Organization ID + in: path + name: id + required: true schema: format: int64 type: integer - - description: Maximum number of rows to return (default and max 500; larger values are clamped) - in: query - name: limit + - description: Structured organization profile media value ID + in: path + name: media_id + required: true schema: format: int64 type: integer - - description: "Sort field: date, size, or subject" - in: query - name: sort - schema: - type: string - - description: "Sort direction: asc or desc" - in: query - name: direction - schema: - type: string + responses: + "200": + content: + "*/*": + schema: + contentMediaType: application/octet-stream + format: binary + type: string + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Download stored inline content for one organization profile media value + tags: + - API + /api/v1/participants/completions: + post: + description: Returns a bounded, typed completion set from the committed observed-person index and current curated profile primitives. The private query stays in the JSON body. + operationId: completeParticipants + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ParticipantCompletionHTTPRequest" + required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/FilteredMessagesResponse" + $ref: "#/components/schemas/ParticipantCompletionHTTPResponse" description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error default: content: application/json: @@ -18453,157 +22924,45 @@ paths: description: Error security: - apiKey: [] - summary: List filtered messages + summary: Complete observed people by typed contact primitives tags: - API - /api/v1/messages/gmail-ids: - get: - operationId: getGmailIDsByFilter - parameters: - - description: Sender email/address filter - in: query - name: sender - schema: - type: string - - description: Sender display-name filter - in: query - name: sender_name - schema: - type: string - - description: Recipient email/address filter - in: query - name: recipient - schema: - type: string - - description: Recipient display-name filter - in: query - name: recipient_name - schema: - type: string - - description: Domain filter - in: query - name: domain - schema: - type: string - - description: Label filter - in: query - name: label - schema: - type: string - - description: Exact case-insensitive RFC 2919 List-Id filter - in: query - name: list_id - schema: - type: string - - description: Message type filter - in: query - name: message_type - schema: - type: string - - description: Named time period - in: query - name: time_period - schema: - type: string - - description: Time bucket granularity - in: query - name: time_granularity - schema: - type: string - - description: Conversation ID - in: query - name: conversation_id - schema: - format: int64 - type: integer - - description: Source ID - in: query - name: source_id - schema: - format: int64 - type: integer - - description: Source IDs; repeat or comma-separate values - in: query - name: source_ids - schema: - items: - format: int64 - type: integer - type: array - - description: Only include messages with attachments - in: query - name: attachments_only - schema: - type: boolean - - description: Exclude deleted messages - in: query - name: hide_deleted - schema: - type: boolean - - description: Lower date/time bound (RFC3339 or YYYY-MM-DD) - in: query - name: after - schema: - type: string - - description: Upper date/time bound (RFC3339 or YYYY-MM-DD) - in: query - name: before - schema: - type: string - - description: Comma-separated aggregate view names to match empty values - in: query - name: empty_targets - schema: - type: string - - description: Zero-based row offset - in: query - name: offset - schema: - format: int64 - type: integer - - description: Maximum number of rows to return (default and max 500; larger values are clamped) - in: query - name: limit - schema: - format: int64 - type: integer - - description: "Sort field: date, size, or subject" - in: query - name: sort - schema: - type: string - - description: "Sort direction: asc or desc" - in: query - name: direction - schema: - type: string - - description: Structured search query - in: query - name: q - schema: - type: string - - description: "Search mode: fast, deep, or aggregate; required with q" - in: query - name: search_mode - schema: - type: string - - description: Aggregate view type; required for aggregate search - in: query - name: view_type - schema: - type: string - - description: Displayed aggregate row key; required for aggregate search - in: query - name: aggregate_key - schema: - type: string + /api/v1/participants/search: + post: + operationId: searchParticipants + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/IdentitySearchHTTPRequest" + required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/GmailIDsResponse" + $ref: "#/components/schemas/ParticipantSearchHTTPResponse" description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + anyOf: + - $ref: "#/components/schemas/ExploreCacheUnavailableResponse" + - $ref: "#/components/schemas/ErrorResponse" + description: Service Unavailable default: content: application/json: @@ -18612,14 +22971,14 @@ paths: description: Error security: - apiKey: [] - summary: List Gmail message IDs matching a filter + summary: Search observed participant clusters tags: - - API - /api/v1/messages/{id}: + - Exploration + /api/v1/participants/{id}: get: - operationId: getMessage + operationId: getParticipant parameters: - - description: Message ID + - description: Observed participant cluster member ID in: path name: id required: true @@ -18631,8 +22990,16 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/MessageDetail" + $ref: "#/components/schemas/PersonSummary" description: OK + "503": + content: + application/json: + schema: + anyOf: + - $ref: "#/components/schemas/ExploreCacheUnavailableResponse" + - $ref: "#/components/schemas/ErrorResponse" + description: Service Unavailable default: content: application/json: @@ -18641,66 +23008,52 @@ paths: description: Error security: - apiKey: [] - summary: Get one message + summary: Get one observed participant cluster tags: - API - /api/v1/messages/{id}/inline: - get: - operationId: getMessageInlinePart + /api/v1/participants/{id}/files/search: + post: + operationId: searchParticipantFiles parameters: - - description: Message ID + - description: Observed participant cluster member ID in: path name: id required: true schema: format: int64 type: integer - - description: Inline MIME Content-ID - in: query - name: cid - required: true - schema: - type: string + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/PersonFileSearchHTTPRequest" + required: true responses: - "200": - content: - application/octet-stream: - schema: - contentMediaType: application/octet-stream - format: binary - type: string - description: OK - "400": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "401": + "200": content: application/json: schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "404": + $ref: "#/components/schemas/PersonFileSearchHTTPResponse" + description: OK + "400": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "415": + "404": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "500": + "409": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "501": + "422": content: application/json: schema: @@ -18710,8 +23063,10 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error + anyOf: + - $ref: "#/components/schemas/ExploreCacheUnavailableResponse" + - $ref: "#/components/schemas/ErrorResponse" + description: Service Unavailable default: content: application/json: @@ -18720,14 +23075,14 @@ paths: description: Error security: - apiKey: [] - summary: Get an inline MIME part + summary: Search one participant cluster's analytical files tags: - - API - /api/v1/messages/{id}/tasks: + - Exploration + /api/v1/participants/{id}/inboxes: get: - operationId: listMessageTasks + operationId: listParticipantInboxes parameters: - - description: Archived email message ID + - description: Observed participant cluster member ID in: path name: id required: true @@ -18739,8 +23094,16 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/TaskLinkLookupResponse" + $ref: "#/components/schemas/PersonInboxResponse" description: OK + "503": + content: + application/json: + schema: + anyOf: + - $ref: "#/components/schemas/ExploreCacheUnavailableResponse" + - $ref: "#/components/schemas/ErrorResponse" + description: Service Unavailable default: content: application/json: @@ -18749,38 +23112,53 @@ paths: description: Error security: - apiKey: [] - summary: List tasks linked to an archived email + summary: List one participant cluster's messaging inboxes tags: - API + /api/v1/participants/{id}/summary: post: - operationId: createOrLinkMessageTask + operationId: getParticipantContextSummary parameters: - - description: Archived email message ID + - description: Observed participant cluster member ID in: path name: id required: true schema: format: int64 type: integer - - description: Browser-generated retry-stable request ID - in: header - name: X-Request-Id - required: true - schema: - type: string requestBody: content: application/json: schema: - $ref: "#/components/schemas/TaskLinkMutationRequest" + $ref: "#/components/schemas/ExploreHTTPRequest" required: true responses: - "201": + "200": content: application/json: schema: - $ref: "#/components/schemas/TaskLinkMutationResponse" - description: Created + $ref: "#/components/schemas/ParticipantContextSummaryHTTPResponse" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + anyOf: + - $ref: "#/components/schemas/ExploreCacheUnavailableResponse" + - $ref: "#/components/schemas/ErrorResponse" + description: Service Unavailable default: content: application/json: @@ -18789,33 +23167,53 @@ paths: description: Error security: - apiKey: [] - summary: Create or link a task for an archived email + summary: Get one participant cluster's contextual analytical summary tags: - - API - /api/v1/messages/{id}/tasks/{task_id}: - delete: - operationId: unlinkMessageTask + - Exploration + /api/v1/participants/{id}/timeline: + post: + operationId: getParticipantTimeline parameters: - - description: Archived email message ID + - description: Observed participant cluster member ID in: path name: id required: true schema: format: int64 type: integer - - description: External task ID - in: path - name: task_id - required: true - schema: - type: string + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ExploreHTTPRequest" + required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/TaskLinkMutationResponse" + $ref: "#/components/schemas/ExploreHTTPResponse" description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + anyOf: + - $ref: "#/components/schemas/ExploreCacheUnavailableResponse" + - $ref: "#/components/schemas/ErrorResponse" + description: Service Unavailable default: content: application/json: @@ -18824,113 +23222,81 @@ paths: description: Error security: - apiKey: [] - summary: Unlink a task from an archived email + summary: Get one participant cluster's canonical activity timeline tags: - - API - /api/v1/multimodal/build: - post: - operationId: startVisualAttachmentBuild - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/VisualBuildRequest" - required: true + - Exploration + /api/v1/people: + get: + description: "Durable people are curated profiles; /api/v1/participants exposes observed analytical groupings. The listing is deliberately unpaginated: persons exist only through explicit promotion, so the set stays small." + operationId: listPeople responses: "200": content: application/json: schema: - $ref: "#/components/schemas/Status" + $ref: "#/components/schemas/PeopleResponse" description: OK - default: + "503": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - security: - - apiKey: [] - summary: Consent and run one bounded visual attachment embedding pass - tags: - - API - /api/v1/multimodal/retire: - post: - operationId: retireVisualAttachmentGeneration - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/VisualRetireRequest" - required: true - responses: - "204": - description: No Content default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" description: Error security: - apiKey: [] - summary: Retire the visual attachment generation + summary: List durable person profiles tags: - - Search - /api/v1/multimodal/retry: + - API post: - operationId: retryVisualAttachmentOwner + description: Returns 201 when a new person is created, or 200 when the cluster is already represented by a person (idempotent re-promotion, which also binds any unbound cluster members). New profiles default to the first nonblank observed cluster name by participant ID. An explicit display_name takes precedence; an empty or whitespace-only string leaves the name unset. Omitted or null uses the default. Re-promotion preserves the existing name, including a cleared name. + operationId: createPerson requestBody: content: application/json: schema: - $ref: "#/components/schemas/VisualRetryRequest" + $ref: "#/components/schemas/CreatePersonRequest" required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/Status" + $ref: "#/components/schemas/Person" description: OK - default: - content: - application/json: + headers: + ETag: + description: Strong person profile revision tag for optimistic concurrency schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - security: - - apiKey: [] - summary: Retry one visual attachment owner - tags: - - API - /api/v1/multimodal/run: - post: - operationId: resumeVisualAttachmentBuild - responses: - "200": + type: string + "201": content: application/json: schema: - $ref: "#/components/schemas/Status" - description: OK - default: + $ref: "#/components/schemas/Person" + description: Created + headers: + ETag: + description: Strong person profile revision tag for optimistic concurrency + schema: + type: string + "409": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - security: - - apiKey: [] - summary: Resume one bounded visual attachment embedding pass - tags: - - API - /api/v1/multimodal/status: - get: - operationId: getVisualAttachmentStatus - responses: - "200": + "503": content: application/json: schema: - $ref: "#/components/schemas/Status" - description: OK + $ref: "#/components/schemas/ErrorResponse" + description: Error default: content: application/json: @@ -18939,272 +23305,232 @@ paths: description: Error security: - apiKey: [] - summary: Get visual attachment embedding status + summary: Promote a participant cluster to a durable person tags: - API - /api/v1/operations/runs: + /api/v1/people/directory: get: - operationId: listOperationRuns + description: Returns one stable, non-sensitive page of promoted durable people. + operationId: listDirectoryPeople parameters: - - description: Exact operation kind + - description: Lexical query over person names, contact points, and organizations in: query - name: kind + name: q + schema: + type: string + - description: Opaque cursor returned by the previous Directory page + in: query + name: cursor + schema: + type: string + - description: Maximum rows to return (default 50, max 100) + in: query + name: limit + schema: + format: int64 + type: integer + - description: "Current contact state: active or inactive" + in: query + name: contact_state schema: - enum: - - carddav_sync - - document_embedding - - document_extraction - - message_embedding - - person_embedding - - person_enrichment - - person_sweep - - source_sync - - visual_embedding type: string - - description: Exact semantic operation lane + - description: Current person category in: query - name: lane + name: category schema: - enum: - - contacts - - documents - - messages - - person_facts - - visual_attachments type: string - - description: Exact operation state + - description: Current organization in: query - name: state + name: organization schema: - enum: - - cancelled - - failed - - partial - - queued - - running - - succeeded type: string - - description: Inclusive canonical UTC RFC3339 lower bound + - description: Primary communication channel in: query - name: started_from + name: primary_channel schema: - format: date-time type: string - - description: Exclusive canonical UTC RFC3339 upper bound + - description: Return people contacted at or after this RFC3339 timestamp in: query - name: started_before + name: last_contact_after schema: format: date-time type: string - - description: Maximum runs to return (default 25, max 100) + - description: Return people contacted at or before this RFC3339 timestamp in: query - name: limit + name: last_contact_before schema: - format: int64 - maximum: 100 - minimum: 1 - type: integer - - description: Opaque cursor bound to this archive and the complete normalized filter set + format: date-time + type: string + - description: "Directory order: name, last_contact_desc, or last_contact_asc" in: query - name: cursor + name: sort schema: + enum: + - name + - last_contact_desc + - last_contact_asc type: string responses: "200": content: application/json: schema: - $ref: "#/components/schemas/OperationRunsResponse" + $ref: "#/components/schemas/DirectoryPeopleResponse" description: OK "400": content: application/json: schema: - $ref: "#/components/schemas/OperationErrorResponse" + $ref: "#/components/schemas/ErrorResponse" description: Error - "409": + "503": content: application/json: schema: - $ref: "#/components/schemas/OperationErrorResponse" + $ref: "#/components/schemas/ErrorResponse" description: Error - "500": + default: content: application/json: schema: - $ref: "#/components/schemas/OperationErrorResponse" + $ref: "#/components/schemas/ErrorResponse" description: Error + security: + - apiKey: [] + summary: Query durable people for the Directory + tags: + - API + /api/v1/people/search: + post: + description: Searches only the curated person vector corpus and returns durable person roots in relevance order. + operationId: searchPeople + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/PersonSearchRequest" + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/PersonSearchResponse" + description: OK "503": content: application/json: schema: - $ref: "#/components/schemas/OperationErrorResponse" + $ref: "#/components/schemas/ErrorResponse" description: Error default: content: application/json: schema: - $ref: "#/components/schemas/OperationErrorResponse" + $ref: "#/components/schemas/ErrorResponse" description: Error security: - apiKey: [] - summary: List normalized operation history + summary: Search durable people semantically tags: - API - /api/v1/operations/runs/{id}: - get: - operationId: getOperationRun + /api/v1/people/{id}: + delete: + description: "Deletion is permanent: the person's participant bindings are removed and its vCard UID is retired forever. Re-promoting the same cluster afterwards creates a new person with a new UID." + operationId: deletePerson parameters: - - description: Opaque archive-bound operation run ID + - description: Durable person ID in: path name: id required: true + schema: + format: int64 + type: integer + - description: Strong ETag returned by the latest person profile read. Must be the exact single tag from that read; the RFC 7232 forms `*` and comma-separated tag lists are not supported. + in: header + name: If-Match + required: true schema: type: string responses: - "200": + "204": + description: No Content + "400": content: application/json: schema: - $ref: "#/components/schemas/OperationRunDetail" - description: OK - "400": + $ref: "#/components/schemas/ErrorResponse" + description: Error + "401": content: application/json: schema: - $ref: "#/components/schemas/OperationErrorResponse" + $ref: "#/components/schemas/ErrorResponse" description: Error "404": content: application/json: schema: - $ref: "#/components/schemas/OperationErrorResponse" + $ref: "#/components/schemas/ErrorResponse" description: Error - "500": + "409": content: application/json: schema: - $ref: "#/components/schemas/OperationErrorResponse" + $ref: "#/components/schemas/ErrorResponse" description: Error - "503": + "428": content: application/json: schema: - $ref: "#/components/schemas/OperationErrorResponse" + $ref: "#/components/schemas/ErrorResponse" description: Error - default: + "500": content: application/json: schema: - $ref: "#/components/schemas/OperationErrorResponse" + $ref: "#/components/schemas/ErrorResponse" description: Error - security: - - apiKey: [] - summary: Get one normalized operation run - tags: - - API - /api/v1/operations/status: - get: - operationId: getOperationStatus - responses: - "200": + "503": content: application/json: schema: - $ref: "#/components/schemas/OperationStatusResponse" - description: OK + $ref: "#/components/schemas/ErrorResponse" + description: Error default: content: application/json: schema: - $ref: "#/components/schemas/OperationErrorResponse" + $ref: "#/components/schemas/ErrorResponse" description: Error security: - apiKey: [] - summary: Get normalized operation lane status + summary: Delete a durable person profile tags: - API - /api/v1/organizations: get: - operationId: listOrganizations + operationId: getPersonProfile parameters: - - description: Maximum results - in: query - name: limit - schema: - format: int64 - type: integer - - description: Results to skip - in: query - name: offset + - description: Durable person ID + in: path + name: id + required: true schema: format: int64 type: integer - - description: Include retired organizations - in: query - name: include_retired - schema: - type: boolean - - description: Normalized-name search - in: query - name: q - schema: - type: string responses: "200": content: application/json: schema: - $ref: "#/components/schemas/OrganizationsResponse" + $ref: "#/components/schemas/Person" description: OK - "400": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "503": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - default: - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - security: - - apiKey: [] - summary: List organizations - tags: - - API - post: - operationId: createOrganization - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/OrganizationCreateBody" - required: true - responses: - "201": - content: - application/json: - schema: - $ref: "#/components/schemas/Organization" - description: Created headers: ETag: - description: Strong organization revision tag for optimistic concurrency - schema: - type: string - Location: - description: Canonical URL of the created organization + description: Strong person profile revision tag for optimistic concurrency schema: type: string - "400": + "404": content: application/json: schema: @@ -19224,35 +23550,43 @@ paths: description: Error security: - apiKey: [] - summary: Create an organization + summary: Get a durable person profile tags: - API - /api/v1/organizations/{id}: - delete: - operationId: deleteOrganization + patch: + operationId: patchPerson parameters: - - description: Organization ID + - description: Durable person ID in: path name: id required: true schema: format: int64 type: integer - - description: Strong ETag returned by the latest organization read + - description: Strong ETag returned by the latest person profile read. Must be the exact single tag from that read; the RFC 7232 forms `*` and comma-separated tag lists are not supported. in: header name: If-Match required: true schema: type: string + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/PatchPersonRequest" + required: true responses: - "204": - description: No Content - "400": + "200": content: application/json: schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error + $ref: "#/components/schemas/Person" + description: OK + headers: + ETag: + description: Strong person profile revision tag for optimistic concurrency + schema: + type: string "404": content: application/json: @@ -19285,13 +23619,14 @@ paths: description: Error security: - apiKey: [] - summary: Delete an organization without employment records + summary: Update a durable person's display name tags: - API + /api/v1/people/{id}/agenda: get: - operationId: getOrganization + operationId: listPersonAgenda parameters: - - description: Organization ID + - description: Durable person ID in: path name: id required: true @@ -19303,14 +23638,9 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/OrganizationProfile" + $ref: "#/components/schemas/PersonAgendaResult" description: OK - headers: - ETag: - description: Strong organization revision tag for optimistic concurrency - schema: - type: string - "400": + "401": content: application/json: schema: @@ -19322,6 +23652,12 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error "503": content: application/json: @@ -19336,119 +23672,71 @@ paths: description: Error security: - apiKey: [] - summary: Get an organization + summary: List a person's live Kata agenda tags: - API - patch: - operationId: patchOrganization + post: + operationId: createPersonAgendaItem parameters: - - description: Organization ID + - description: Durable person ID in: path name: id required: true schema: format: int64 type: integer - - description: Strong ETag returned by the latest organization read + - description: Opaque 1..128-byte retry key in: header - name: If-Match + name: Idempotency-Key required: true schema: + maxLength: 128 + minLength: 1 type: string requestBody: content: application/json: schema: - $ref: "#/components/schemas/OrganizationBody" + $ref: "#/components/schemas/PersonAgendaCreateRequest" required: true responses: - "200": + "201": content: application/json: schema: - $ref: "#/components/schemas/Organization" - description: OK - headers: - ETag: - description: Strong organization revision tag for optimistic concurrency - schema: - type: string + $ref: "#/components/schemas/PersonAgendaMutationResponse" + description: Created "400": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "404": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "409": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "428": + "401": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "503": + "404": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - default: + "409": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - security: - - apiKey: [] - summary: Replace an organization's mutable fields - tags: - - API - /api/v1/organizations/{id}/attributes: - get: - operationId: listOrganizationAttributes - parameters: - - description: Organization ID - in: path - name: id - required: true - schema: - format: int64 - type: integer - - description: Include superseded values - in: query - name: include_superseded - schema: - type: boolean - - description: Restrict to one definition - in: query - name: definition_slug - schema: - type: string - responses: - "200": - content: - application/json: - schema: - $ref: "#/components/schemas/OrganizationAttributesResponse" - description: OK - "400": + "422": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "404": + "428": content: application/json: schema: @@ -19468,13 +23756,14 @@ paths: description: Error security: - apiKey: [] - summary: List organization typed attributes + summary: Create a live Kata item for a person tags: - API + /api/v1/people/{id}/agenda/links: post: - operationId: setOrganizationAttribute + operationId: linkPersonAgendaItem parameters: - - description: Organization ID + - description: Durable person ID in: path name: id required: true @@ -19485,20 +23774,14 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/SetOrganizationAttributeBody" + $ref: "#/components/schemas/PersonAgendaLinkRequest" required: true responses: - "200": - content: - application/json: - schema: - $ref: "#/components/schemas/OrganizationAttributeWrite" - description: OK "201": content: application/json: schema: - $ref: "#/components/schemas/OrganizationAttributeWrite" + $ref: "#/components/schemas/PersonAgendaMutationResponse" description: Created "400": content: @@ -19506,6 +23789,12 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error "404": content: application/json: @@ -19518,6 +23807,12 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error + "422": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error "503": content: application/json: @@ -19532,49 +23827,32 @@ paths: description: Error security: - apiKey: [] - summary: Set an organization typed attribute + summary: Link an existing Kata item to a person tags: - API - /api/v1/organizations/{id}/attributes/{slug}: + /api/v1/people/{id}/agenda/{ref}: delete: - operationId: clearOrganizationAttribute + operationId: unlinkPersonAgendaItem parameters: - - description: Organization ID + - description: Durable person ID in: path name: id required: true schema: format: int64 type: integer - - description: Immutable attribute definition slug + - description: Kata issue ref or canonical UID in: path - name: slug + name: ref required: true schema: type: string - - description: Ordinal for a multi-valued definition - in: query - name: ordinal - schema: - format: int64 - type: integer - - description: "Compare-and-swap: the current value ID expected to be superseded" - in: query - name: expected_value_id - schema: - format: int64 - type: integer - - description: Validate and preview without writing - in: query - name: dry_run - schema: - type: boolean responses: "200": content: application/json: schema: - $ref: "#/components/schemas/OrganizationAttributeWrite" + $ref: "#/components/schemas/PersonAgendaMutationResponse" description: OK "400": content: @@ -19582,6 +23860,12 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error "404": content: application/json: @@ -19608,43 +23892,37 @@ paths: description: Error security: - apiKey: [] - summary: Supersede an organization typed attribute + summary: Unlink a live Kata item from a person tags: - API - /api/v1/organizations/{id}/employments: - get: - operationId: listOrganizationEmployments + patch: + operationId: updatePersonAgendaItem parameters: - - description: Organization ID + - description: Durable person ID in: path name: id required: true schema: format: int64 type: integer - - description: Only current employments - in: query - name: current_only - schema: - type: boolean - - description: Maximum results - in: query - name: limit - schema: - format: int64 - type: integer - - description: Results to skip - in: query - name: offset + - description: Kata issue ref or canonical UID + in: path + name: ref + required: true schema: - format: int64 - type: integer + type: string + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/PersonAgendaUpdateRequest" + required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/EmploymentsResponse" + $ref: "#/components/schemas/PersonAgendaMutationResponse" description: OK "400": content: @@ -19652,12 +23930,30 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error "404": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "422": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error "503": content: application/json: @@ -19672,38 +23968,42 @@ paths: description: Error security: - apiKey: [] - summary: List an organization's employment records + summary: Move a linked Kata item to another list tags: - API - /api/v1/organizations/{id}/history: + /api/v1/people/{id}/attributes: get: - operationId: getOrganizationHistory + operationId: listPersonAttributes parameters: - - description: Organization ID + - description: Durable person ID in: path name: id required: true schema: format: int64 type: integer + - description: Include superseded values + in: query + name: history + schema: + type: boolean + - description: Restrict the response to one definition slug + in: query + name: slug + schema: + type: string + - description: Restrict the response to one portable definition identifier + in: query + name: universal_id + schema: + type: string responses: "200": content: application/json: schema: - $ref: "#/components/schemas/OrganizationProfile" + $ref: "#/components/schemas/PersonAttributesResponse" description: OK - headers: - ETag: - description: Strong organization revision tag for optimistic concurrency - schema: - type: string - "400": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error "404": content: application/json: @@ -19724,44 +24024,50 @@ paths: description: Error security: - apiKey: [] - summary: Get organization profile history + summary: List a person's typed attributes tags: - API - /api/v1/organizations/{id}/merge: - post: - operationId: mergeOrganization + /api/v1/people/{id}/attributes/{slug}: + delete: + operationId: clearPersonAttribute parameters: - - description: Organization ID + - description: Durable person ID in: path name: id required: true schema: format: int64 type: integer - - description: Strong ETag returned by the latest organization read - in: header - name: If-Match - required: true + - description: Immutable attribute definition slug + in: path + name: slug + required: true + schema: + type: string + - description: Ordinal for a multi-valued definition + in: query + name: ordinal + schema: + format: int64 + type: integer + - description: "Compare-and-swap: the current value ID expected to be superseded" + in: query + name: expected_value_id schema: - type: string - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/MergeOrganizationBody" - required: true + format: int64 + type: integer + - description: Validate and preview without writing + in: query + name: dry_run + schema: + type: boolean responses: "200": content: application/json: schema: - $ref: "#/components/schemas/Organization" + $ref: "#/components/schemas/PersonAttributeWrite" description: OK - headers: - ETag: - description: Strong organization revision tag for optimistic concurrency - schema: - type: string "400": content: application/json: @@ -19778,14 +24084,8 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "428": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error + $ref: "#/components/schemas/PersonAttributeConflictResponse" + description: Conflict "503": content: application/json: @@ -19800,45 +24100,43 @@ paths: description: Error security: - apiKey: [] - summary: Merge another organization into this organization + summary: Supersede a person's attribute value tags: - API - /api/v1/organizations/{id}/profile: put: - description: Replaces all structured organization profile collections with at most 200 total values. - operationId: putOrganizationProfile + operationId: setPersonAttribute parameters: - - description: Organization ID + - description: Durable person ID in: path name: id required: true schema: format: int64 type: integer - - description: Strong ETag returned by the latest organization read - in: header - name: If-Match + - description: Immutable attribute definition slug + in: path + name: slug required: true schema: type: string + - description: Validate and preview without writing + in: query + name: dry_run + schema: + type: boolean requestBody: content: application/json: schema: - $ref: "#/components/schemas/OrganizationProfileBody" + $ref: "#/components/schemas/SetPersonAttributeRequest" required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/OrganizationProfile" + $ref: "#/components/schemas/PersonAttributeWrite" description: OK - headers: - ETag: - description: Strong organization revision tag for optimistic concurrency - schema: - type: string "400": content: application/json: @@ -19855,20 +24153,8 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "413": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "428": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error + $ref: "#/components/schemas/PersonAttributeConflictResponse" + description: Conflict "503": content: application/json: @@ -19883,36 +24169,26 @@ paths: description: Error security: - apiKey: [] - summary: Replace organization profile collections + summary: Set a person's attribute value tags: - API - /api/v1/organizations/{id}/profile/media/{media_id}/content: + /api/v1/people/{id}/brief: get: - description: Returns the exact inline bytes stored for one media value. URI-only values have no local content and return 404. - operationId: getOrganizationProfileMediaContent + operationId: getPersonBrief parameters: - - description: Organization ID + - description: Durable person ID in: path name: id required: true schema: format: int64 type: integer - - description: Structured organization profile media value ID - in: path - name: media_id - required: true - schema: - format: int64 - type: integer responses: "200": content: - "*/*": + application/json: schema: - contentMediaType: application/octet-stream - format: binary - type: string + $ref: "#/components/schemas/PersonBrief" description: OK "400": content: @@ -19920,24 +24196,12 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "401": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error "404": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "500": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error "503": content: application/json: @@ -19952,25 +24216,26 @@ paths: description: Error security: - apiKey: [] - summary: Download stored inline content for one organization profile media value + summary: Get a person's current brief version tags: - API - /api/v1/participants/completions: - post: - description: Returns a bounded, typed completion set from the committed observed-person index and current curated profile primitives. The private query stays in the JSON body. - operationId: completeParticipants - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/ParticipantCompletionHTTPRequest" - required: true + /api/v1/people/{id}/brief-enrollment: + get: + operationId: getPersonBriefEnrollment + parameters: + - description: Durable person ID + in: path + name: id + required: true + schema: + format: int64 + type: integer responses: "200": content: application/json: schema: - $ref: "#/components/schemas/ParticipantCompletionHTTPResponse" + $ref: "#/components/schemas/PersonBriefEnrollment" description: OK "400": content: @@ -19978,7 +24243,7 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "500": + "404": content: application/json: schema: @@ -19998,24 +24263,31 @@ paths: description: Error security: - apiKey: [] - summary: Complete observed people by typed contact primitives + summary: Get a person's brief enrollment tags: - API - /api/v1/participants/search: - post: - operationId: searchParticipants + put: + operationId: setPersonBriefEnrollment + parameters: + - description: Durable person ID + in: path + name: id + required: true + schema: + format: int64 + type: integer requestBody: content: application/json: schema: - $ref: "#/components/schemas/IdentitySearchHTTPRequest" + $ref: "#/components/schemas/PutPersonBriefEnrollmentRequest" required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/ParticipantSearchHTTPResponse" + $ref: "#/components/schemas/PersonBriefEnrollment" description: OK "400": content: @@ -20023,57 +24295,24 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "409": + "404": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "503": - content: - application/json: - schema: - anyOf: - - $ref: "#/components/schemas/ExploreCacheUnavailableResponse" - - $ref: "#/components/schemas/ErrorResponse" - description: Service Unavailable - default: + "409": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - security: - - apiKey: [] - summary: Search observed participant clusters - tags: - - Exploration - /api/v1/participants/{id}: - get: - operationId: getParticipant - parameters: - - description: Observed participant cluster member ID - in: path - name: id - required: true - schema: - format: int64 - type: integer - responses: - "200": - content: - application/json: - schema: - $ref: "#/components/schemas/PersonSummary" - description: OK "503": content: application/json: schema: - anyOf: - - $ref: "#/components/schemas/ExploreCacheUnavailableResponse" - - $ref: "#/components/schemas/ErrorResponse" - description: Service Unavailable + $ref: "#/components/schemas/ErrorResponse" + description: Error default: content: application/json: @@ -20082,32 +24321,26 @@ paths: description: Error security: - apiKey: [] - summary: Get one observed participant cluster + summary: Replace a person's brief enrollment tags: - API - /api/v1/participants/{id}/files/search: + /api/v1/people/{id}/brief/generate: post: - operationId: searchParticipantFiles + operationId: generatePersonBrief parameters: - - description: Observed participant cluster member ID + - description: Durable person ID in: path name: id required: true schema: format: int64 type: integer - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/PersonFileSearchHTTPRequest" - required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/PersonFileSearchHTTPResponse" + $ref: "#/components/schemas/PersonBriefRun" description: OK "400": content: @@ -20127,20 +24360,12 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "422": + "503": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "503": - content: - application/json: - schema: - anyOf: - - $ref: "#/components/schemas/ExploreCacheUnavailableResponse" - - $ref: "#/components/schemas/ErrorResponse" - description: Service Unavailable default: content: application/json: @@ -20149,35 +24374,51 @@ paths: description: Error security: - apiKey: [] - summary: Search one participant cluster's analytical files + summary: Generate a person's brief now tags: - - Exploration - /api/v1/participants/{id}/inboxes: - get: - operationId: listParticipantInboxes + - API + /api/v1/people/{id}/brief/reject: + post: + operationId: rejectPersonBrief parameters: - - description: Observed participant cluster member ID + - description: Durable person ID in: path name: id required: true schema: format: int64 type: integer + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/RejectPersonBriefRequest" + required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/PersonInboxResponse" - description: OK + $ref: "#/components/schemas/PersonBrief" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error "503": content: application/json: schema: - anyOf: - - $ref: "#/components/schemas/ExploreCacheUnavailableResponse" - - $ref: "#/components/schemas/ErrorResponse" - description: Service Unavailable + $ref: "#/components/schemas/ErrorResponse" + description: Error default: content: application/json: @@ -20186,32 +24427,32 @@ paths: description: Error security: - apiKey: [] - summary: List one participant cluster's messaging inboxes + summary: Reject a person's current brief version tags: - API - /api/v1/participants/{id}/summary: - post: - operationId: getParticipantContextSummary + /api/v1/people/{id}/brief/versions: + get: + operationId: listPersonBriefVersions parameters: - - description: Observed participant cluster member ID + - description: Durable person ID in: path name: id required: true schema: format: int64 type: integer - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/ExploreHTTPRequest" - required: true + - description: Maximum versions to return (default 20, max 200) + in: query + name: limit + schema: + format: int64 + type: integer responses: "200": content: application/json: schema: - $ref: "#/components/schemas/ParticipantContextSummaryHTTPResponse" + $ref: "#/components/schemas/PersonBriefVersionsResponse" description: OK "400": content: @@ -20219,7 +24460,7 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "409": + "404": content: application/json: schema: @@ -20229,10 +24470,8 @@ paths: content: application/json: schema: - anyOf: - - $ref: "#/components/schemas/ExploreCacheUnavailableResponse" - - $ref: "#/components/schemas/ErrorResponse" - description: Service Unavailable + $ref: "#/components/schemas/ErrorResponse" + description: Error default: content: application/json: @@ -20241,32 +24480,27 @@ paths: description: Error security: - apiKey: [] - summary: Get one participant cluster's contextual analytical summary + summary: List a person's brief version history tags: - - Exploration - /api/v1/participants/{id}/timeline: - post: - operationId: getParticipantTimeline + - API + /api/v1/people/{id}/contact-state: + get: + operationId: getPersonContactState parameters: - - description: Observed participant cluster member ID + - description: Positive durable identifier in: path name: id required: true schema: format: int64 + minimum: 1 type: integer - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/ExploreHTTPRequest" - required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/ExploreHTTPResponse" + $ref: "#/components/schemas/ContactState" description: OK "400": content: @@ -20274,42 +24508,30 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "409": + "401": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "503": + "403": content: application/json: schema: - anyOf: - - $ref: "#/components/schemas/ExploreCacheUnavailableResponse" - - $ref: "#/components/schemas/ErrorResponse" - description: Service Unavailable - default: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "404": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - security: - - apiKey: [] - summary: Get one participant cluster's canonical activity timeline - tags: - - Exploration - /api/v1/people: - get: - description: "Durable people are curated profiles; /api/v1/participants exposes observed analytical groupings. The listing is deliberately unpaginated: persons exist only through explicit promotion, so the set stays small." - operationId: listPeople - responses: - "200": + "500": content: application/json: schema: - $ref: "#/components/schemas/PeopleResponse" - description: OK + $ref: "#/components/schemas/ErrorResponse" + description: Error "503": content: application/json: @@ -20324,42 +24546,82 @@ paths: description: Error security: - apiKey: [] - summary: List durable person profiles + summary: Get computed contact state for a person tags: - API - post: - description: Returns 201 when a new person is created, or 200 when the cluster is already represented by a person (idempotent re-promotion, which also binds any unbound cluster members). - operationId: createPerson - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/CreatePersonRequest" - required: true + /api/v1/people/{id}/days: + get: + operationId: listPersonActivityDays + parameters: + - description: Positive durable identifier + in: path + name: id + required: true + schema: + format: int64 + minimum: 1 + type: integer + - description: Inclusive first local calendar date + in: query + name: from + schema: + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ + type: string + - description: Inclusive last local calendar date + in: query + name: to + schema: + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ + type: string + - description: Maximum primary rows to return + in: query + name: limit + schema: + default: 100 + format: int64 + maximum: 500 + minimum: 1 + type: integer + - description: Zero-based primary row offset + in: query + name: offset + schema: + default: 0 + format: int64 + minimum: 0 + type: integer responses: "200": content: application/json: schema: - $ref: "#/components/schemas/Person" + $ref: "#/components/schemas/PersonDaysPage" description: OK - headers: - ETag: - description: Strong person profile revision tag for optimistic concurrency + "400": + content: + application/json: schema: - type: string - "201": + $ref: "#/components/schemas/ErrorResponse" + description: Error + "401": content: application/json: schema: - $ref: "#/components/schemas/Person" - description: Created - headers: - ETag: - description: Strong person profile revision tag for optimistic concurrency + $ref: "#/components/schemas/ErrorResponse" + description: Error + "403": + content: + application/json: schema: - type: string - "409": + $ref: "#/components/schemas/ErrorResponse" + description: Error + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "500": content: application/json: schema: @@ -20379,77 +24641,68 @@ paths: description: Error security: - apiKey: [] - summary: Promote a participant cluster to a durable person + summary: List calendar days intersecting a person tags: - API - /api/v1/people/directory: + /api/v1/people/{id}/days/{date}: get: - description: Returns one stable, non-sensitive page of promoted durable people. - operationId: listDirectoryPeople + operationId: getPersonActivityDay parameters: - - description: Lexical query over person names, contact points, and organizations - in: query - name: q + - description: Positive durable identifier + in: path + name: id + required: true schema: - type: string - - description: Opaque cursor returned by the previous Directory page - in: query - name: cursor + format: int64 + minimum: 1 + type: integer + - description: Exact local calendar date + in: path + name: date + required: true schema: + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ type: string - - description: Maximum rows to return (default 50, max 100) + - description: Maximum primary rows to return in: query name: limit schema: + default: 100 format: int64 + maximum: 500 + minimum: 1 type: integer - - description: "Current contact state: active or inactive" - in: query - name: contact_state - schema: - type: string - - description: Current person category - in: query - name: category - schema: - type: string - - description: Current organization - in: query - name: organization - schema: - type: string - - description: Primary communication channel - in: query - name: primary_channel - schema: - type: string - - description: Return people contacted at or after this RFC3339 timestamp + - description: Zero-based primary row offset in: query - name: last_contact_after + name: offset schema: - format: date-time - type: string - - description: Return people contacted at or before this RFC3339 timestamp + default: 0 + format: int64 + minimum: 0 + type: integer + - description: Maximum authored entries to return independently in: query - name: last_contact_before + name: entry_limit schema: - format: date-time - type: string - - description: "Directory order: name, last_contact_desc, or last_contact_asc" + default: 100 + format: int64 + maximum: 500 + minimum: 1 + type: integer + - description: Zero-based authored-entry offset in: query - name: sort + name: entry_offset schema: - enum: - - name - - last_contact_desc - - last_contact_asc - type: string + default: 0 + format: int64 + minimum: 0 + type: integer responses: "200": content: application/json: schema: - $ref: "#/components/schemas/DirectoryPeopleResponse" + $ref: "#/components/schemas/PersonDayPage" description: OK "400": content: @@ -20457,40 +24710,30 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "503": + "401": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - default: + "403": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - security: - - apiKey: [] - summary: Query durable people for the Directory - tags: - - API - /api/v1/people/search: - post: - description: Searches only the curated person vector corpus and returns durable person roots in relevance order. - operationId: searchPeople - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/PersonSearchRequest" - required: true - responses: - "200": + "404": content: application/json: schema: - $ref: "#/components/schemas/PersonSearchResponse" - description: OK + $ref: "#/components/schemas/ErrorResponse" + description: Error + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error "503": content: application/json: @@ -20505,61 +24748,115 @@ paths: description: Error security: - apiKey: [] - summary: Search durable people semantically + summary: Get one person's activity and notes for a day tags: - API - /api/v1/people/{id}: - delete: - description: "Deletion is permanent: the person's participant bindings are removed and its vCard UID is retired forever. Re-promoting the same cluster afterwards creates a new person with a new UID." - operationId: deletePerson + /api/v1/people/{id}/employments: + get: + operationId: listPersonEmployments parameters: - - description: Durable person ID + - description: Person ID in: path name: id required: true schema: format: int64 type: integer - - description: Strong ETag returned by the latest person profile read. Must be the exact single tag from that read; the RFC 7232 forms `*` and comma-separated tag lists are not supported. - in: header - name: If-Match - required: true + - description: Only current employments + in: query + name: current_only schema: - type: string + type: boolean + - description: Maximum results + in: query + name: limit + schema: + format: int64 + type: integer + - description: Results to skip + in: query + name: offset + schema: + format: int64 + type: integer responses: - "204": - description: No Content + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/EmploymentsResponse" + description: OK "400": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "401": + "404": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "404": + "503": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "409": + default: content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "428": + security: + - apiKey: [] + summary: List a person's employment history + tags: + - API + /api/v1/people/{id}/fact-claims: + get: + operationId: listPersonFactClaims + parameters: + - description: Durable person ID + in: path + name: id + required: true + schema: + format: int64 + type: integer + - description: Exact target as kind:key:sha256:<64 lowercase hex characters> + in: query + name: target + schema: + type: string + - description: Maximum rows to return (default 50, max 200) + in: query + name: limit + schema: + format: int64 + type: integer + - description: Zero-based row offset + in: query + name: offset + schema: + format: int64 + type: integer + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/PersonFactClaimsResponse" + description: OK + "400": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "500": + "404": content: application/json: schema: @@ -20579,11 +24876,12 @@ paths: description: Error security: - apiKey: [] - summary: Delete a durable person profile + summary: List immutable person fact claims tags: - API + /api/v1/people/{id}/fact-decisions: get: - operationId: getPersonProfile + operationId: listPersonFactDecisions parameters: - description: Durable person ID in: path @@ -20592,18 +24890,36 @@ paths: schema: format: int64 type: integer + - description: Exact target as kind:key:sha256:<64 lowercase hex characters> + in: query + name: target + schema: + type: string + - description: Maximum rows to return (default 50, max 200) + in: query + name: limit + schema: + format: int64 + type: integer + - description: Zero-based row offset + in: query + name: offset + schema: + format: int64 + type: integer responses: "200": content: application/json: schema: - $ref: "#/components/schemas/Person" + $ref: "#/components/schemas/PersonFactDecisionsResponse" description: OK - headers: - ETag: - description: Strong person profile revision tag for optimistic concurrency + "400": + content: + application/json: schema: - type: string + $ref: "#/components/schemas/ErrorResponse" + description: Error "404": content: application/json: @@ -20624,11 +24940,12 @@ paths: description: Error security: - apiKey: [] - summary: Get a durable person profile + summary: List immutable person fact decisions tags: - API - patch: - operationId: patchPerson + /api/v1/people/{id}/fact-evidence: + get: + operationId: listPersonFactEvidence parameters: - description: Durable person ID in: path @@ -20637,43 +24954,37 @@ paths: schema: format: int64 type: integer - - description: Strong ETag returned by the latest person profile read. Must be the exact single tag from that read; the RFC 7232 forms `*` and comma-separated tag lists are not supported. - in: header - name: If-Match - required: true + - description: Exact target as kind:key:sha256:<64 lowercase hex characters> + in: query + name: target schema: type: string - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/PatchPersonRequest" - required: true + - description: Maximum rows to return (default 50, max 200) + in: query + name: limit + schema: + format: int64 + type: integer + - description: Zero-based row offset + in: query + name: offset + schema: + format: int64 + type: integer responses: "200": content: application/json: schema: - $ref: "#/components/schemas/Person" + $ref: "#/components/schemas/PersonFactEvidenceResponse" description: OK - headers: - ETag: - description: Strong person profile revision tag for optimistic concurrency - schema: - type: string - "404": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "409": + "400": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "428": + "404": content: application/json: schema: @@ -20693,12 +25004,12 @@ paths: description: Error security: - apiKey: [] - summary: Update a durable person's display name + summary: List immutable person fact evidence tags: - API - /api/v1/people/{id}/attributes: + /api/v1/people/{id}/fact-evidence-status-events: get: - operationId: listPersonAttributes + operationId: listPersonFactEvidenceStatusEvents parameters: - description: Durable person ID in: path @@ -20707,28 +25018,41 @@ paths: schema: format: int64 type: integer - - description: Include superseded values + - description: Restrict to one immutable evidence key in: query - name: history + name: evidence_key + schema: + type: string + - description: Restrict to supported or unsupported events + in: query + name: supported schema: type: boolean - - description: Restrict the response to one definition slug + - description: Maximum events to return (default 50, max 200) in: query - name: slug + name: limit schema: - type: string - - description: Restrict the response to one portable definition identifier + format: int64 + type: integer + - description: Zero-based event offset in: query - name: universal_id + name: offset schema: - type: string + format: int64 + type: integer responses: "200": content: application/json: schema: - $ref: "#/components/schemas/PersonAttributesResponse" + $ref: "#/components/schemas/PersonFactEvidenceStatusEventsResponse" description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error "404": content: application/json: @@ -20749,12 +25073,12 @@ paths: description: Error security: - apiKey: [] - summary: List a person's typed attributes + summary: List person fact evidence status history tags: - API - /api/v1/people/{id}/attributes/{slug}: - delete: - operationId: clearPersonAttribute + /api/v1/people/{id}/fact-pins: + get: + operationId: listPersonFactPins parameters: - description: Durable person ID in: path @@ -20763,35 +25087,12 @@ paths: schema: format: int64 type: integer - - description: Immutable attribute definition slug - in: path - name: slug - required: true - schema: - type: string - - description: Ordinal for a multi-valued definition - in: query - name: ordinal - schema: - format: int64 - type: integer - - description: "Compare-and-swap: the current value ID expected to be superseded" - in: query - name: expected_value_id - schema: - format: int64 - type: integer - - description: Validate and preview without writing - in: query - name: dry_run - schema: - type: boolean responses: "200": content: application/json: schema: - $ref: "#/components/schemas/PersonAttributeWrite" + $ref: "#/components/schemas/PersonFactPinsResponse" description: OK "400": content: @@ -20805,12 +25106,6 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "409": - content: - application/json: - schema: - $ref: "#/components/schemas/PersonAttributeConflictResponse" - description: Conflict "503": content: application/json: @@ -20825,11 +25120,12 @@ paths: description: Error security: - apiKey: [] - summary: Supersede a person's attribute value + summary: List effective person fact pins tags: - API + /api/v1/people/{id}/fact-pins/{kind}/{key}: put: - operationId: setPersonAttribute + operationId: setPersonFactPin parameters: - description: Durable person ID in: path @@ -20838,29 +25134,33 @@ paths: schema: format: int64 type: integer - - description: Immutable attribute definition slug + - description: Closed person fact target kind in: path - name: slug + name: kind required: true schema: + enum: + - attribute + - employment type: string - - description: Validate and preview without writing - in: query - name: dry_run + - description: Exact person fact target key + in: path + name: key + required: true schema: - type: boolean + type: string requestBody: content: application/json: schema: - $ref: "#/components/schemas/SetPersonAttributeRequest" + $ref: "#/components/schemas/SetPersonFactPinRequest" required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/PersonAttributeWrite" + $ref: "#/components/schemas/PersonFactPinWrite" description: OK "400": content: @@ -20878,8 +25178,8 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/PersonAttributeConflictResponse" - description: Conflict + $ref: "#/components/schemas/ErrorResponse" + description: Error "503": content: application/json: @@ -20894,12 +25194,12 @@ paths: description: Error security: - apiKey: [] - summary: Set a person's attribute value + summary: Replace an effective person fact pin tags: - API - /api/v1/people/{id}/brief: - get: - operationId: getPersonBrief + /api/v1/people/{id}/files/search: + post: + operationId: searchPersonFiles parameters: - description: Durable person ID in: path @@ -20908,12 +25208,18 @@ paths: schema: format: int64 type: integer + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/PersonFileSearchHTTPRequest" + required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/PersonBrief" + $ref: "#/components/schemas/PersonFileSearchHTTPResponse" description: OK "400": content: @@ -20927,12 +25233,26 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "503": + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "422": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error + "503": + content: + application/json: + schema: + anyOf: + - $ref: "#/components/schemas/ExploreCacheUnavailableResponse" + - $ref: "#/components/schemas/ErrorResponse" + description: Service Unavailable default: content: application/json: @@ -20941,12 +25261,12 @@ paths: description: Error security: - apiKey: [] - summary: Get a person's current brief version + summary: Search one durable person's analytical files tags: - - API - /api/v1/people/{id}/brief-enrollment: - get: - operationId: getPersonBriefEnrollment + - Exploration + /api/v1/people/{id}/merge: + post: + operationId: mergePersons parameters: - description: Durable person ID in: path @@ -20955,13 +25275,38 @@ paths: schema: format: int64 type: integer + - description: Exactly two comma-separated strong person revision tags, one for each profile + in: header + name: If-Match + required: true + schema: + type: string + - description: Opaque 1..128-byte retry key + in: header + name: Idempotency-Key + required: true + schema: + maxLength: 128 + minLength: 1 + type: string + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/MergePersonRequest" + required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/PersonBriefEnrollment" + $ref: "#/components/schemas/PersonMergeResult" description: OK + headers: + ETag: + description: Strong person profile revision tag for optimistic concurrency + schema: + type: string "400": content: application/json: @@ -20974,6 +25319,24 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "428": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error "503": content: application/json: @@ -20988,11 +25351,12 @@ paths: description: Error security: - apiKey: [] - summary: Get a person's brief enrollment + summary: Merge one durable person profile into another tags: - API - put: - operationId: setPersonBriefEnrollment + /api/v1/people/{id}/merges: + get: + operationId: listPersonMerges parameters: - description: Durable person ID in: path @@ -21001,32 +25365,32 @@ paths: schema: format: int64 type: integer - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/PutPersonBriefEnrollmentRequest" - required: true + - description: Maximum results + in: query + name: limit + schema: + format: int64 + type: integer + - description: Results to skip + in: query + name: offset + schema: + format: int64 + type: integer responses: "200": content: application/json: schema: - $ref: "#/components/schemas/PersonBriefEnrollment" + $ref: "#/components/schemas/PersonMergesResponse" description: OK - "400": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error "404": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "409": + "500": content: application/json: schema: @@ -21046,12 +25410,13 @@ paths: description: Error security: - apiKey: [] - summary: Replace a person's brief enrollment + summary: List merge history for a durable person tags: - API - /api/v1/people/{id}/brief/generate: - post: - operationId: generatePersonBrief + /api/v1/people/{id}/network: + get: + description: Returns declared person relationships and employments only; archive-derived associations are excluded. + operationId: getPersonNetwork parameters: - description: Durable person ID in: path @@ -21060,12 +25425,26 @@ paths: schema: format: int64 type: integer + - description: Breadth-first depth (default 1, minimum 1, maximum 3) + in: query + name: depth + schema: + default: 1 + format: int64 + maximum: 3 + minimum: 1 + type: integer + - description: Include ended relationships and employment records + in: query + name: include_ended + schema: + type: boolean responses: "200": content: application/json: schema: - $ref: "#/components/schemas/PersonBriefRun" + $ref: "#/components/schemas/PersonNetwork" description: OK "400": content: @@ -21079,12 +25458,6 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "409": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error "503": content: application/json: @@ -21099,12 +25472,12 @@ paths: description: Error security: - apiKey: [] - summary: Generate a person's brief now + summary: Get a bounded curated person network tags: - API - /api/v1/people/{id}/brief/reject: + /api/v1/people/{id}/notes/append: post: - operationId: rejectPersonBrief + operationId: appendPersonNote parameters: - description: Durable person ID in: path @@ -21113,18 +25486,23 @@ paths: schema: format: int64 type: integer + - description: Validate and preview without writing + in: query + name: dry_run + schema: + type: boolean requestBody: content: application/json: schema: - $ref: "#/components/schemas/RejectPersonBriefRequest" + $ref: "#/components/schemas/AppendPersonNoteRequest" required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/PersonBrief" + $ref: "#/components/schemas/PersonAttributeWrite" description: OK "400": content: @@ -21138,6 +25516,12 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error "503": content: application/json: @@ -21152,12 +25536,13 @@ paths: description: Error security: - apiKey: [] - summary: Reject a person's current brief version + summary: Append to a person's notes tags: - API - /api/v1/people/{id}/brief/versions: + /api/v1/people/{id}/profile: get: - operationId: listPersonBriefVersions + description: Returns only current structured values at one person revision. Superseded values and archive observations are available from the separate history endpoint. + operationId: getPersonStructuredProfile parameters: - description: Durable person ID in: path @@ -21166,19 +25551,18 @@ paths: schema: format: int64 type: integer - - description: Maximum versions to return (default 20, max 200) - in: query - name: limit - schema: - format: int64 - type: integer responses: "200": content: application/json: schema: - $ref: "#/components/schemas/PersonBriefVersionsResponse" + $ref: "#/components/schemas/StructuredPersonProfile" description: OK + headers: + ETag: + description: Strong person profile revision tag for optimistic concurrency + schema: + type: string "400": content: application/json: @@ -21205,53 +25589,69 @@ paths: description: Error security: - apiKey: [] - summary: List a person's brief version history + summary: Get a person's current structured profile tags: - API - /api/v1/people/{id}/contact-state: - get: - operationId: getPersonContactState + patch: + description: Applies up to 200 explicit adds and supersedes atomically under If-Match. One patch advances the person revision once. Superseding closes world and transaction time without deletion. + operationId: patchPersonStructuredProfile parameters: - - description: Positive durable identifier + - description: Durable person ID in: path name: id required: true schema: format: int64 - minimum: 1 type: integer + - description: Strong ETag returned by the latest person profile read. Must be the exact single tag from that read; the RFC 7232 forms `*` and comma-separated tag lists are not supported. + in: header + name: If-Match + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/PersonProfilePatchRequest" + required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/ContactState" + $ref: "#/components/schemas/StructuredPersonProfile" description: OK + headers: + ETag: + description: Strong person profile revision tag for optimistic concurrency + schema: + type: string "400": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "401": + "404": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "403": + "409": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "404": + "413": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "500": + "428": content: application/json: schema: @@ -21271,56 +25671,27 @@ paths: description: Error security: - apiKey: [] - summary: Get computed contact state for a person + summary: Atomically patch a person's structured profile tags: - API - /api/v1/people/{id}/days: + /api/v1/people/{id}/profile/history: get: - operationId: listPersonActivityDays + description: Returns current and superseded structured values plus source-linked observations for every participant bound to the person. + operationId: getPersonProfileHistory parameters: - - description: Positive durable identifier + - description: Durable person ID in: path name: id required: true schema: format: int64 - minimum: 1 - type: integer - - description: Inclusive first local calendar date - in: query - name: from - schema: - pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ - type: string - - description: Inclusive last local calendar date - in: query - name: to - schema: - pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ - type: string - - description: Maximum primary rows to return - in: query - name: limit - schema: - default: 100 - format: int64 - maximum: 500 - minimum: 1 - type: integer - - description: Zero-based primary row offset - in: query - name: offset - schema: - default: 0 - format: int64 - minimum: 0 type: integer responses: "200": content: application/json: schema: - $ref: "#/components/schemas/PersonDaysPage" + $ref: "#/components/schemas/PersonProfileHistory" description: OK "400": content: @@ -21328,30 +25699,12 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "401": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "403": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error "404": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "500": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error "503": content: application/json: @@ -21366,68 +25719,36 @@ paths: description: Error security: - apiKey: [] - summary: List calendar days intersecting a person + summary: Get a person's structured profile history tags: - API - /api/v1/people/{id}/days/{date}: + /api/v1/people/{id}/profile/media/{media_id}/content: get: - operationId: getPersonActivityDay + description: Returns the exact inline bytes stored for one media value. URI-only values have no local content and return 404. + operationId: getPersonProfileMediaContent parameters: - - description: Positive durable identifier + - description: Durable person ID in: path name: id required: true schema: format: int64 - minimum: 1 type: integer - - description: Exact local calendar date + - description: Structured person profile media value ID in: path - name: date + name: media_id required: true schema: - pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ - type: string - - description: Maximum primary rows to return - in: query - name: limit - schema: - default: 100 - format: int64 - maximum: 500 - minimum: 1 - type: integer - - description: Zero-based primary row offset - in: query - name: offset - schema: - default: 0 - format: int64 - minimum: 0 - type: integer - - description: Maximum authored entries to return independently - in: query - name: entry_limit - schema: - default: 100 - format: int64 - maximum: 500 - minimum: 1 - type: integer - - description: Zero-based authored-entry offset - in: query - name: entry_offset - schema: - default: 0 format: int64 - minimum: 0 type: integer responses: "200": content: - application/json: + "*/*": schema: - $ref: "#/components/schemas/PersonDayPage" + contentMediaType: application/octet-stream + format: binary + type: string description: OK "400": content: @@ -21441,12 +25762,6 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "403": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error "404": content: application/json: @@ -21473,43 +25788,30 @@ paths: description: Error security: - apiKey: [] - summary: Get one person's activity and notes for a day + summary: Download stored inline content for one person profile media value tags: - API - /api/v1/people/{id}/employments: + /api/v1/people/{id}/relationships: get: - operationId: listPersonEmployments + operationId: listPersonRelationships parameters: - - description: Person ID + - description: Durable person ID in: path name: id required: true schema: format: int64 type: integer - - description: Only current employments - in: query - name: current_only + - in: query + name: include_ended schema: type: boolean - - description: Maximum results - in: query - name: limit - schema: - format: int64 - type: integer - - description: Results to skip - in: query - name: offset - schema: - format: int64 - type: integer responses: "200": content: application/json: schema: - $ref: "#/components/schemas/EmploymentsResponse" + $ref: "#/components/schemas/PersonRelationshipsResponse" description: OK "400": content: @@ -21517,12 +25819,6 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "404": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error "503": content: application/json: @@ -21537,12 +25833,12 @@ paths: description: Error security: - apiKey: [] - summary: List a person's employment history + summary: List one person's relationships tags: - API - /api/v1/people/{id}/fact-claims: - get: - operationId: listPersonFactClaims + /api/v1/people/{id}/split: + post: + operationId: splitPersonMerge parameters: - description: Durable person ID in: path @@ -21551,101 +25847,67 @@ paths: schema: format: int64 type: integer - - description: Exact target as kind:key:sha256:<64 lowercase hex characters> - in: query - name: target + - description: Strong ETag returned by the latest person profile read. Must be the exact single tag from that read; the RFC 7232 forms `*` and comma-separated tag lists are not supported. + in: header + name: If-Match + required: true schema: type: string - - description: Maximum rows to return (default 50, max 200) - in: query - name: limit - schema: - format: int64 - type: integer - - description: Zero-based row offset - in: query - name: offset + - description: Opaque 1..128-byte retry key + in: header + name: Idempotency-Key + required: true schema: - format: int64 - type: integer + maxLength: 128 + minLength: 1 + type: string + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/SplitPersonRequest" + required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/PersonFactClaimsResponse" + $ref: "#/components/schemas/PersonSplitResult" description: OK - "400": - content: - application/json: + headers: + ETag: + description: Strong person profile revision tag for optimistic concurrency schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "404": - content: - application/json: + type: string + X-New-Person-ETag: + description: Strong revision tag for the new person created by a split schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "503": + type: string + "400": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - default: + "404": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - security: - - apiKey: [] - summary: List immutable person fact claims - tags: - - API - /api/v1/people/{id}/fact-decisions: - get: - operationId: listPersonFactDecisions - parameters: - - description: Durable person ID - in: path - name: id - required: true - schema: - format: int64 - type: integer - - description: Exact target as kind:key:sha256:<64 lowercase hex characters> - in: query - name: target - schema: - type: string - - description: Maximum rows to return (default 50, max 200) - in: query - name: limit - schema: - format: int64 - type: integer - - description: Zero-based row offset - in: query - name: offset - schema: - format: int64 - type: integer - responses: - "200": + "409": content: application/json: schema: - $ref: "#/components/schemas/PersonFactDecisionsResponse" - description: OK - "400": + $ref: "#/components/schemas/ErrorResponse" + description: Error + "428": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "404": + "500": content: application/json: schema: @@ -21665,12 +25927,12 @@ paths: description: Error security: - apiKey: [] - summary: List immutable person fact decisions + summary: Split absorbed participant lineage into a new person tags: - API - /api/v1/people/{id}/fact-evidence: + /api/v1/people/{id}/tracking: get: - operationId: listPersonFactEvidence + operationId: getPersonTracking parameters: - description: Durable person ID in: path @@ -21679,36 +25941,13 @@ paths: schema: format: int64 type: integer - - description: Exact target as kind:key:sha256:<64 lowercase hex characters> - in: query - name: target - schema: - type: string - - description: Maximum rows to return (default 50, max 200) - in: query - name: limit - schema: - format: int64 - type: integer - - description: Zero-based row offset - in: query - name: offset - schema: - format: int64 - type: integer responses: "200": content: application/json: schema: - $ref: "#/components/schemas/PersonFactEvidenceResponse" + $ref: "#/components/schemas/PersonTracking" description: OK - "400": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error "404": content: application/json: @@ -21729,12 +25968,11 @@ paths: description: Error security: - apiKey: [] - summary: List immutable person fact evidence + summary: Get a person's tracking state tags: - API - /api/v1/people/{id}/fact-evidence-status-events: - get: - operationId: listPersonFactEvidenceStatusEvents + put: + operationId: setPersonTracking parameters: - description: Durable person ID in: path @@ -21743,34 +25981,18 @@ paths: schema: format: int64 type: integer - - description: Restrict to one immutable evidence key - in: query - name: evidence_key - schema: - type: string - - description: Restrict to supported or unsupported events - in: query - name: supported - schema: - type: boolean - - description: Maximum events to return (default 50, max 200) - in: query - name: limit - schema: - format: int64 - type: integer - - description: Zero-based event offset - in: query - name: offset - schema: - format: int64 - type: integer + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/PutPersonTrackingRequest" + required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/PersonFactEvidenceStatusEventsResponse" + $ref: "#/components/schemas/PersonTracking" description: OK "400": content: @@ -21798,26 +26020,24 @@ paths: description: Error security: - apiKey: [] - summary: List person fact evidence status history + summary: Replace a person's tracking state tags: - API - /api/v1/people/{id}/fact-pins: + /api/v1/person-fact-targets: get: - operationId: listPersonFactPins + operationId: listPersonFactTargets parameters: - - description: Durable person ID - in: path - name: id - required: true + - description: Include sensitive targets + in: query + name: include_sensitive schema: - format: int64 - type: integer + type: boolean responses: "200": content: application/json: schema: - $ref: "#/components/schemas/PersonFactPinsResponse" + $ref: "#/components/schemas/Catalog" description: OK "400": content: @@ -21825,12 +26045,6 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "404": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error "503": content: application/json: @@ -21845,32 +26059,24 @@ paths: description: Error security: - apiKey: [] - summary: List effective person fact pins + summary: List eligible automatic person fact targets tags: - API - /api/v1/people/{id}/fact-pins/{kind}/{key}: - put: - operationId: setPersonFactPin + /api/v1/person-merge-candidates/{candidate_id}/decision: + post: + operationId: decidePersonMergeCandidate parameters: - - description: Durable person ID + - description: Person merge review candidate ID in: path - name: id + name: candidate_id required: true schema: format: int64 + minimum: 1 type: integer - - description: Closed person fact target kind - in: path - name: kind - required: true - schema: - enum: - - attribute - - employment - type: string - - description: Exact person fact target key - in: path - name: key + - description: Strong ETag returned by the latest person profile read. Must be the exact single tag from that read; the RFC 7232 forms `*` and comma-separated tag lists are not supported. + in: header + name: If-Match required: true schema: type: string @@ -21878,15 +26084,20 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/SetPersonFactPinRequest" + $ref: "#/components/schemas/DecidePersonMergeCandidateRequest" required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/PersonFactPinWrite" + $ref: "#/components/schemas/PersonMergeReviewCandidate" description: OK + headers: + ETag: + description: Strong person profile revision tag for optimistic concurrency + schema: + type: string "400": content: application/json: @@ -21905,6 +26116,18 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error + "428": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error "503": content: application/json: @@ -21919,32 +26142,27 @@ paths: description: Error security: - apiKey: [] - summary: Replace an effective person fact pin + summary: Accept or reject a person merge attribute candidate tags: - API - /api/v1/people/{id}/files/search: - post: - operationId: searchPersonFiles + /api/v1/person-merges/{merge_id}: + get: + operationId: getPersonMerge parameters: - - description: Durable person ID + - description: Durable person merge ID in: path - name: id + name: merge_id required: true schema: format: int64 + minimum: 1 type: integer - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/PersonFileSearchHTTPRequest" - required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/PersonFileSearchHTTPResponse" + $ref: "#/components/schemas/PersonMergeDetail" description: OK "400": content: @@ -21958,26 +26176,18 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "409": + "500": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "422": + "503": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "503": - content: - application/json: - schema: - anyOf: - - $ref: "#/components/schemas/ExploreCacheUnavailableResponse" - - $ref: "#/components/schemas/ErrorResponse" - description: Service Unavailable default: content: application/json: @@ -21986,50 +26196,31 @@ paths: description: Error security: - apiKey: [] - summary: Search one durable person's analytical files + summary: Inspect one durable person merge tags: - - Exploration - /api/v1/people/{id}/merge: - post: - operationId: mergePersons + - API + /api/v1/person-merges/{merge_id}/snapshot: + get: + operationId: getPersonMergeSnapshot parameters: - - description: Durable person ID + - description: Durable person merge ID in: path - name: id + name: merge_id required: true schema: format: int64 - type: integer - - description: Exactly two comma-separated strong person revision tags, one for each profile - in: header - name: If-Match - required: true - schema: - type: string - - description: Opaque 1..128-byte retry key - in: header - name: Idempotency-Key - required: true - schema: - maxLength: 128 - minLength: 1 - type: string - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/MergePersonRequest" - required: true + minimum: 1 + type: integer responses: "200": content: application/json: schema: - $ref: "#/components/schemas/PersonMergeResult" + $ref: "#/components/schemas/PersonMergeSnapshotResponse" description: OK headers: - ETag: - description: Strong person profile revision tag for optimistic concurrency + Cache-Control: + description: Always no-store because the response contains merge provenance schema: type: string "400": @@ -22044,18 +26235,6 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "409": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "428": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error "500": content: application/json: @@ -22076,29 +26255,23 @@ paths: description: Error security: - apiKey: [] - summary: Merge one durable person profile into another + summary: Read and verify one person merge snapshot tags: - API - /api/v1/people/{id}/merges: + /api/v1/person-relationship-reviews: get: - operationId: listPersonMerges + operationId: listPersonRelationshipReviews parameters: - - description: Durable person ID - in: path - name: id - required: true - schema: - format: int64 - type: integer - - description: Maximum results - in: query - name: limit + - in: query + name: status schema: - format: int64 - type: integer - - description: Results to skip - in: query - name: offset + enum: + - pending + - accepted + - rejected + type: string + - in: query + name: person_id schema: format: int64 type: integer @@ -22107,15 +26280,9 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/PersonMergesResponse" + $ref: "#/components/schemas/RelationshipReviewsResponse" description: OK - "404": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "500": + "400": content: application/json: schema: @@ -22135,42 +26302,34 @@ paths: description: Error security: - apiKey: [] - summary: List merge history for a durable person + summary: List imported RELATED values awaiting review tags: - API - /api/v1/people/{id}/network: - get: - description: Returns declared person relationships and employments only; archive-derived associations are excluded. - operationId: getPersonNetwork - parameters: - - description: Durable person ID - in: path - name: id - required: true - schema: - format: int64 - type: integer - - description: Breadth-first depth (default 1, minimum 1, maximum 3) - in: query - name: depth - schema: - default: 1 - format: int64 - maximum: 3 - minimum: 1 - type: integer - - description: Include ended relationships and employment records - in: query - name: include_ended - schema: - type: boolean + /api/v1/person-relationships: + post: + operationId: createPersonRelationship + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/CreatePersonRelationshipRequest" + required: true responses: - "200": + "201": content: application/json: schema: - $ref: "#/components/schemas/PersonNetwork" - description: OK + $ref: "#/components/schemas/PersonRelationship" + description: Created + headers: + ETag: + description: Strong person relationship revision tag for optimistic concurrency + schema: + type: string + Location: + description: Created person relationship + schema: + type: string "400": content: application/json: @@ -22183,6 +26342,12 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error "503": content: application/json: @@ -22197,38 +26362,29 @@ paths: description: Error security: - apiKey: [] - summary: Get a bounded curated person network + summary: Declare a relationship between two persons tags: - API - /api/v1/people/{id}/notes/append: - post: - operationId: appendPersonNote + /api/v1/person-relationships/{id}: + delete: + operationId: deletePersonRelationship parameters: - - description: Durable person ID + - description: Person relationship ID in: path name: id required: true schema: format: int64 type: integer - - description: Validate and preview without writing - in: query - name: dry_run + - description: Strong ETag returned by the latest person relationship read + in: header + name: If-Match + required: true schema: - type: boolean - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/AppendPersonNoteRequest" - required: true + type: string responses: - "200": - content: - application/json: - schema: - $ref: "#/components/schemas/PersonAttributeWrite" - description: OK + "204": + description: No Content "400": content: application/json: @@ -22247,6 +26403,12 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error + "428": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error "503": content: application/json: @@ -22261,15 +26423,13 @@ paths: description: Error security: - apiKey: [] - summary: Append to a person's notes + summary: Delete a person relationship tags: - API - /api/v1/people/{id}/profile: get: - description: Returns only current structured values at one person revision. Superseded values and archive observations are available from the separate history endpoint. - operationId: getPersonStructuredProfile + operationId: getPersonRelationship parameters: - - description: Durable person ID + - description: Person relationship ID in: path name: id required: true @@ -22281,19 +26441,13 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/StructuredPersonProfile" + $ref: "#/components/schemas/PersonRelationship" description: OK headers: ETag: - description: Strong person profile revision tag for optimistic concurrency + description: Strong person relationship revision tag for optimistic concurrency schema: type: string - "400": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error "404": content: application/json: @@ -22314,21 +26468,20 @@ paths: description: Error security: - apiKey: [] - summary: Get a person's current structured profile + summary: Get one person relationship tags: - API patch: - description: Applies up to 200 explicit adds and supersedes atomically under If-Match. One patch advances the person revision once. Superseding closes world and transaction time without deletion. - operationId: patchPersonStructuredProfile + operationId: patchPersonRelationship parameters: - - description: Durable person ID + - description: Person relationship ID in: path name: id required: true schema: format: int64 type: integer - - description: Strong ETag returned by the latest person profile read. Must be the exact single tag from that read; the RFC 7232 forms `*` and comma-separated tag lists are not supported. + - description: Strong ETag returned by the latest person relationship read in: header name: If-Match required: true @@ -22338,18 +26491,18 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/PersonProfilePatchRequest" + $ref: "#/components/schemas/PatchPersonRelationshipRequest" required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/StructuredPersonProfile" + $ref: "#/components/schemas/PersonRelationship" description: OK headers: ETag: - description: Strong person profile revision tag for optimistic concurrency + description: Strong person relationship revision tag for optimistic concurrency schema: type: string "400": @@ -22370,12 +26523,6 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "413": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error "428": content: application/json: @@ -22396,46 +26543,37 @@ paths: description: Error security: - apiKey: [] - summary: Atomically patch a person's structured profile + summary: End a relationship or replace its notes tags: - API - /api/v1/people/{id}/profile/history: - get: - description: Returns current and superseded structured values plus source-linked observations for every participant bound to the person. - operationId: getPersonProfileHistory + /api/v1/query: + post: + operationId: runQuery parameters: - - description: Durable person ID - in: path - name: id - required: true + - description: Request a background cache check including writes committed before this request + in: query + name: fresh schema: - format: int64 - type: integer + type: boolean + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/QueryRequest" + required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/PersonProfileHistory" + $ref: "#/components/schemas/QueryResult" description: OK - "400": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "404": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "503": + "202": content: application/json: schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error + $ref: "#/components/schemas/CacheBuildAccepted" + description: Accepted default: content: application/json: @@ -22444,61 +26582,58 @@ paths: description: Error security: - apiKey: [] - summary: Get a person's structured profile history + summary: Run an aggregate query tags: - API - /api/v1/people/{id}/profile/media/{media_id}/content: - get: - description: Returns the exact inline bytes stored for one media value. URI-only values have no local content and return 404. - operationId: getPersonProfileMediaContent - parameters: - - description: Durable person ID - in: path - name: id - required: true - schema: - format: int64 - type: integer - - description: Structured person profile media value ID - in: path - name: media_id - required: true + /api/v1/query/archive: + post: + operationId: runArchiveQuery + parameters: + - description: Request a background cache check including writes committed before this request + in: query + name: fresh schema: - format: int64 - type: integer + type: boolean + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/QueryRequest" + required: true responses: "200": - content: - "*/*": - schema: - contentMediaType: application/octet-stream - format: binary - type: string - description: OK - "400": content: application/json: schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "401": + $ref: "#/components/schemas/QueryResult" + description: OK + "202": content: application/json: schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "404": + $ref: "#/components/schemas/CacheBuildAccepted" + description: Accepted + default: content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "500": + security: + - apiKey: [] + summary: Run SQL restricted to archive analytics files + tags: + - API + /api/v1/relationship-types: + get: + operationId: listRelationshipTypes + responses: + "200": content: application/json: schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error + $ref: "#/components/schemas/RelationshipTypesResponse" + description: OK "503": content: application/json: @@ -22513,37 +26648,45 @@ paths: description: Error security: - apiKey: [] - summary: Download stored inline content for one person profile media value + summary: List person relationship types tags: - API - /api/v1/people/{id}/relationships: - get: - operationId: listPersonRelationships - parameters: - - description: Durable person ID - in: path - name: id - required: true - schema: - format: int64 - type: integer - - in: query - name: include_ended - schema: - type: boolean + post: + operationId: createRelationshipType + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/CreateRelationshipTypeRequest" + required: true responses: - "200": + "201": content: application/json: schema: - $ref: "#/components/schemas/PersonRelationshipsResponse" - description: OK + $ref: "#/components/schemas/RelationshipType" + description: Created + headers: + ETag: + description: Strong relationship type revision tag for optimistic concurrency + schema: + type: string + Location: + description: Created relationship type + schema: + type: string "400": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error "503": content: application/json: @@ -22558,56 +26701,29 @@ paths: description: Error security: - apiKey: [] - summary: List one person's relationships + summary: Create a user-owned relationship type tags: - API - /api/v1/people/{id}/split: - post: - operationId: splitPersonMerge + /api/v1/relationship-types/{id}: + delete: + operationId: deleteRelationshipType parameters: - - description: Durable person ID + - description: Relationship type ID in: path name: id required: true schema: format: int64 type: integer - - description: Strong ETag returned by the latest person profile read. Must be the exact single tag from that read; the RFC 7232 forms `*` and comma-separated tag lists are not supported. + - description: Strong ETag returned by the latest relationship type read in: header name: If-Match required: true schema: type: string - - description: Opaque 1..128-byte retry key - in: header - name: Idempotency-Key - required: true - schema: - maxLength: 128 - minLength: 1 - type: string - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/SplitPersonRequest" - required: true responses: - "200": - content: - application/json: - schema: - $ref: "#/components/schemas/PersonSplitResult" - description: OK - headers: - ETag: - description: Strong person profile revision tag for optimistic concurrency - schema: - type: string - X-New-Person-ETag: - description: Strong revision tag for the new person created by a split - schema: - type: string + "204": + description: No Content "400": content: application/json: @@ -22632,12 +26748,6 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "500": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error "503": content: application/json: @@ -22652,14 +26762,13 @@ paths: description: Error security: - apiKey: [] - summary: Split absorbed participant lineage into a new person + summary: Delete an unused relationship type tags: - API - /api/v1/people/{id}/tracking: get: - operationId: getPersonTracking + operationId: getRelationshipType parameters: - - description: Durable person ID + - description: Relationship type ID in: path name: id required: true @@ -22671,8 +26780,13 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/PersonTracking" + $ref: "#/components/schemas/RelationshipType" description: OK + headers: + ETag: + description: Strong relationship type revision tag for optimistic concurrency + schema: + type: string "404": content: application/json: @@ -22693,32 +26807,43 @@ paths: description: Error security: - apiKey: [] - summary: Get a person's tracking state + summary: Get a relationship type tags: - API - put: - operationId: setPersonTracking + patch: + operationId: patchRelationshipType parameters: - - description: Durable person ID + - description: Relationship type ID in: path name: id required: true schema: format: int64 type: integer + - description: Strong ETag returned by the latest relationship type read + in: header + name: If-Match + required: true + schema: + type: string requestBody: content: application/json: schema: - $ref: "#/components/schemas/PutPersonTrackingRequest" + $ref: "#/components/schemas/PatchRelationshipTypeRequest" required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/PersonTracking" + $ref: "#/components/schemas/RelationshipType" description: OK + headers: + ETag: + description: Strong relationship type revision tag for optimistic concurrency + schema: + type: string "400": content: application/json: @@ -22731,40 +26856,13 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "503": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - default: + "409": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - security: - - apiKey: [] - summary: Replace a person's tracking state - tags: - - API - /api/v1/person-fact-targets: - get: - operationId: listPersonFactTargets - parameters: - - description: Include sensitive targets - in: query - name: include_sensitive - schema: - type: boolean - responses: - "200": - content: - application/json: - schema: - $ref: "#/components/schemas/Catalog" - description: OK - "400": + "428": content: application/json: schema: @@ -22784,70 +26882,32 @@ paths: description: Error security: - apiKey: [] - summary: List eligible automatic person fact targets + summary: Update a relationship type tags: - API - /api/v1/person-merge-candidates/{candidate_id}/decision: + /api/v1/relationships: post: - operationId: decidePersonMergeCandidate - parameters: - - description: Person merge review candidate ID - in: path - name: candidate_id - required: true - schema: - format: int64 - minimum: 1 - type: integer - - description: Strong ETag returned by the latest person profile read. Must be the exact single tag from that read; the RFC 7232 forms `*` and comma-separated tag lists are not supported. - in: header - name: If-Match - required: true - schema: - type: string + operationId: listRelationships requestBody: content: application/json: schema: - $ref: "#/components/schemas/DecidePersonMergeCandidateRequest" + $ref: "#/components/schemas/RelationshipsHTTPRequest" required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/PersonMergeReviewCandidate" + $ref: "#/components/schemas/RelationshipsHTTPResponse" description: OK - headers: - ETag: - description: Strong person profile revision tag for optimistic concurrency - schema: - type: string "400": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "404": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "409": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "428": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "500": + "409": content: application/json: schema: @@ -22857,8 +26917,10 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error + anyOf: + - $ref: "#/components/schemas/ExploreCacheUnavailableResponse" + - $ref: "#/components/schemas/ErrorResponse" + description: Service Unavailable default: content: application/json: @@ -22867,27 +26929,32 @@ paths: description: Error security: - apiKey: [] - summary: Accept or reject a person merge attribute candidate + summary: Rank counterparts by reciprocity-weighted interaction tags: - - API - /api/v1/person-merges/{merge_id}: - get: - operationId: getPersonMerge + - Exploration + /api/v1/relationships/{id}/calendar: + post: + operationId: getRelationshipCalendar parameters: - - description: Durable person merge ID + - description: Any member participant ID of the counterpart's identity cluster in: path - name: merge_id + name: id required: true schema: format: int64 - minimum: 1 type: integer + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/RelationshipCalendarHTTPRequest" + required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/PersonMergeDetail" + $ref: "#/components/schemas/RelationshipCalendarHTTPResponse" description: OK "400": content: @@ -22901,18 +26968,14 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "500": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error "503": content: application/json: schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error + anyOf: + - $ref: "#/components/schemas/ExploreCacheUnavailableResponse" + - $ref: "#/components/schemas/ErrorResponse" + description: Service Unavailable default: content: application/json: @@ -22921,46 +26984,40 @@ paths: description: Error security: - apiKey: [] - summary: Inspect one durable person merge + summary: Get one counterpart's timezone-aware relationship calendar tags: - - API - /api/v1/person-merges/{merge_id}/snapshot: - get: - operationId: getPersonMergeSnapshot + - Exploration + /api/v1/relationships/{id}/timeline: + post: + operationId: getRelationshipTimeline parameters: - - description: Durable person merge ID + - description: Any member participant ID of the counterpart's identity cluster in: path - name: merge_id + name: id required: true schema: format: int64 - minimum: 1 type: integer + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/RelationshipTimelineHTTPRequest" + required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/PersonMergeSnapshotResponse" + $ref: "#/components/schemas/RelationshipTimelineHTTPResponse" description: OK - headers: - Cache-Control: - description: Always no-store because the response contains merge provenance - schema: - type: string "400": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "404": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "500": + "409": content: application/json: schema: @@ -22970,8 +27027,10 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error + anyOf: + - $ref: "#/components/schemas/ExploreCacheUnavailableResponse" + - $ref: "#/components/schemas/ErrorResponse" + description: Service Unavailable default: content: application/json: @@ -22980,45 +27039,19 @@ paths: description: Error security: - apiKey: [] - summary: Read and verify one person merge snapshot + summary: Get one counterpart's interaction timeline, with chat grouped into local-day bursts tags: - - API - /api/v1/person-relationship-reviews: + - Exploration + /api/v1/saved-views: get: - operationId: listPersonRelationshipReviews - parameters: - - in: query - name: status - schema: - enum: - - pending - - accepted - - rejected - type: string - - in: query - name: person_id - schema: - format: int64 - type: integer + operationId: listSavedViews responses: "200": content: application/json: schema: - $ref: "#/components/schemas/RelationshipReviewsResponse" + $ref: "#/components/schemas/SavedViewsResponse" description: OK - "400": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "503": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error default: content: application/json: @@ -23027,32 +27060,27 @@ paths: description: Error security: - apiKey: [] - summary: List imported RELATED values awaiting review + summary: List shared analytical Saved Views tags: - API - /api/v1/person-relationships: post: - operationId: createPersonRelationship + operationId: createSavedView requestBody: content: application/json: schema: - $ref: "#/components/schemas/CreatePersonRelationshipRequest" + $ref: "#/components/schemas/CreateSavedViewRequest" required: true responses: "201": content: application/json: schema: - $ref: "#/components/schemas/PersonRelationship" + $ref: "#/components/schemas/SavedView" description: Created headers: ETag: - description: Strong person relationship revision tag for optimistic concurrency - schema: - type: string - Location: - description: Created person relationship + description: Strong Saved View revision tag for optimistic concurrency schema: type: string "400": @@ -23061,12 +27089,6 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "404": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error "409": content: application/json: @@ -23087,21 +27109,21 @@ paths: description: Error security: - apiKey: [] - summary: Declare a relationship between two persons + summary: Create a shared analytical Saved View tags: - API - /api/v1/person-relationships/{id}: + /api/v1/saved-views/{id}: delete: - operationId: deletePersonRelationship + operationId: deleteSavedView parameters: - - description: Person relationship ID + - description: Saved View ID in: path name: id required: true schema: format: int64 type: integer - - description: Strong ETag returned by the latest person relationship read + - description: Strong ETag returned by the latest Saved View read in: header name: If-Match required: true @@ -23116,6 +27138,12 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error "404": content: application/json: @@ -23134,6 +27162,12 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error "503": content: application/json: @@ -23148,13 +27182,13 @@ paths: description: Error security: - apiKey: [] - summary: Delete a person relationship + summary: Delete a shared analytical Saved View tags: - API get: - operationId: getPersonRelationship + operationId: getSavedView parameters: - - description: Person relationship ID + - description: Saved View ID in: path name: id required: true @@ -23166,11 +27200,11 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/PersonRelationship" + $ref: "#/components/schemas/SavedView" description: OK headers: ETag: - description: Strong person relationship revision tag for optimistic concurrency + description: Strong Saved View revision tag for optimistic concurrency schema: type: string "404": @@ -23193,43 +27227,108 @@ paths: description: Error security: - apiKey: [] - summary: Get one person relationship + summary: Get a shared analytical Saved View tags: - API patch: - operationId: patchPersonRelationship + operationId: patchSavedView parameters: - - description: Person relationship ID + - description: Saved View ID + in: path + name: id + required: true + schema: + format: int64 + type: integer + - description: Strong ETag returned by the latest Saved View read + in: header + name: If-Match + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/PatchSavedViewRequest" + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/SavedView" + description: OK + headers: + ETag: + description: Strong Saved View revision tag for optimistic concurrency + schema: + type: string + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "428": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Update a shared analytical Saved View + tags: + - API + /api/v1/saved-views/{id}/run: + post: + operationId: runSavedView + parameters: + - description: Saved View ID in: path name: id required: true schema: format: int64 type: integer - - description: Strong ETag returned by the latest person relationship read - in: header - name: If-Match - required: true - schema: - type: string requestBody: content: application/json: schema: - $ref: "#/components/schemas/PatchPersonRelationshipRequest" + $ref: "#/components/schemas/RunSavedViewRequest" required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/PersonRelationship" + $ref: "#/components/schemas/RunSavedViewResponse" description: OK - headers: - ETag: - description: Strong person relationship revision tag for optimistic concurrency - schema: - type: string "400": content: application/json: @@ -23248,18 +27347,14 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "428": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error "503": content: application/json: schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error + anyOf: + - $ref: "#/components/schemas/ExploreCacheUnavailableResponse" + - $ref: "#/components/schemas/ErrorResponse" + description: Service Unavailable default: content: application/json: @@ -23268,24 +27363,18 @@ paths: description: Error security: - apiKey: [] - summary: End a relationship or replace its notes + summary: Run a shared analytical Saved View through its canonical Explore definition tags: - API - /api/v1/query: - post: - operationId: runQuery - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/QueryRequest" - required: true + /api/v1/scheduler/status: + get: + operationId: getSchedulerStatus responses: "200": content: application/json: schema: - $ref: "#/components/schemas/QueryResult" + $ref: "#/components/schemas/SchedulerStatusResponse" description: OK default: content: @@ -23295,25 +27384,143 @@ paths: description: Error security: - apiKey: [] - summary: Run an aggregate query + summary: Get scheduler status tags: - API - /api/v1/relationship-types: + /api/v1/search: get: - operationId: listRelationshipTypes + operationId: searchMessages + parameters: + - description: Search query + in: query + name: q + required: true + schema: + type: string + - description: "Search mode: fts, vector, or hybrid. conversation_id applies in every mode; other structured filter parameters require vector or hybrid" + in: query + name: mode + schema: + type: string + - description: One-based page number (default 1; values below 1 are clamped to 1). Non-numeric values are rejected with 400. + in: query + name: page + schema: + format: int64 + type: integer + - description: Page size (default 20, max 100; out-of-range values are clamped). Non-numeric values are rejected with 400. + in: query + name: page_size + schema: + format: int64 + type: integer + - description: Zero-based ranking offset for vector or hybrid search (default 0) + in: query + name: offset + schema: + format: int64 + type: integer + - description: Include score explanation when mode is vector or hybrid + in: query + name: explain + schema: + type: boolean + - description: Include scored semantic chunk excerpts for vector or hybrid results + in: query + name: include_matches + schema: + type: boolean + - description: Minimum chunk score for included excerpts; does not filter ranked messages + in: query + name: min_score + schema: + type: number + - description: Message type filter; repeat or comma-separate for multiple values + in: query + name: message_type + schema: + type: string + - description: Restrict to one account/source + in: query + name: account + schema: + type: string + - description: Restrict to one collection + in: query + name: collection + schema: + type: string + - description: Exact conversation ID (all search modes) + in: query + name: conversation_id + schema: + format: int64 + type: integer + - description: Exact sender email/address filter (vector or hybrid mode only) + in: query + name: sender + schema: + type: string + - description: Exact recipient email filter across to, cc, and bcc (vector or hybrid mode only) + in: query + name: recipient + schema: + type: string + - description: Exact sender domain filter (vector or hybrid mode only) + in: query + name: domain + schema: + type: string + - description: Exact case-insensitive label filter (vector or hybrid mode only) + in: query + name: label + schema: + type: string + - description: Exact case-insensitive RFC 2919 List-Id filter (vector or hybrid mode only) + in: query + name: list_id + schema: + type: string + - description: Calendar period in YYYY, YYYY-MM, or YYYY-MM-DD format (vector or hybrid mode only) + in: query + name: time_period + schema: + type: string + - description: Time bucket granularity (vector or hybrid mode only) + in: query + name: time_granularity + schema: + type: string + - description: Exact source ID (vector or hybrid mode only) + in: query + name: source_id + schema: + format: int64 + type: integer + - description: Only include messages with attachments (vector or hybrid mode only) + in: query + name: attachments_only + schema: + type: boolean + - description: Lower date/time bound (RFC3339 or YYYY-MM-DD; vector or hybrid mode only) + in: query + name: after + schema: + type: string + - description: Upper date/time bound (RFC3339 or YYYY-MM-DD; vector or hybrid mode only) + in: query + name: before + schema: + type: string responses: "200": content: application/json: schema: - $ref: "#/components/schemas/RelationshipTypesResponse" + oneOf: + - $ref: "#/components/schemas/SearchResult" + - $ref: "#/components/schemas/HybridSearchResponse" description: OK - "503": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error default: content: application/json: @@ -23322,82 +27529,66 @@ paths: description: Error security: - apiKey: [] - summary: List person relationship types + summary: Search messages tags: - API + /api/v1/search/attachments/visual: post: - operationId: createRelationshipType + operationId: searchVisualAttachments requestBody: content: application/json: schema: - $ref: "#/components/schemas/CreateRelationshipTypeRequest" - required: true - responses: - "201": - content: - application/json: - schema: - $ref: "#/components/schemas/RelationshipType" - description: Created - headers: - ETag: - description: Strong relationship type revision tag for optimistic concurrency - schema: - type: string - Location: - description: Created relationship type - schema: - type: string - "400": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "409": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "503": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - default: + $ref: "#/components/schemas/VisualTextSearchRequest" + multipart/form-data: + schema: + properties: + after: + type: string + before: + type: string + cursor: + type: string + direction: + items: + enum: + - from_person + - to_person + - group + type: string + type: array + filename: + type: string + image: + contentMediaType: application/octet-stream + description: Query image (JPEG, PNG, WebP, or still GIF) + format: binary + type: string + limit: + type: string + message_id: + type: string + mime_prefix: + type: string + participant_id: + type: string + person_id: + type: string + sender_person_id: + type: string + source_id: + type: string + required: + - image + type: object + required: true + responses: + "200": content: application/json: schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - security: - - apiKey: [] - summary: Create a user-owned relationship type - tags: - - API - /api/v1/relationship-types/{id}: - delete: - operationId: deleteRelationshipType - parameters: - - description: Relationship type ID - in: path - name: id - required: true - schema: - format: int64 - type: integer - - description: Strong ETag returned by the latest relationship type read - in: header - name: If-Match - required: true - schema: - type: string - responses: - "204": - description: No Content + $ref: "#/components/schemas/SearchResponse" + description: OK "400": content: application/json: @@ -23416,12 +27607,6 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "428": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error "503": content: application/json: @@ -23436,32 +27621,26 @@ paths: description: Error security: - apiKey: [] - summary: Delete an unused relationship type + summary: Search visual attachment content tags: - - API - get: - operationId: getRelationshipType - parameters: - - description: Relationship type ID - in: path - name: id - required: true - schema: - format: int64 - type: integer + - Exploration + /api/v1/search/coverage: + post: + operationId: getSearchCoverage + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/SearchCoverageRequest" + required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/RelationshipType" + $ref: "#/components/schemas/SearchCoverageResponse" description: OK - headers: - ETag: - description: Strong relationship type revision tag for optimistic concurrency - schema: - type: string - "404": + "400": content: application/json: schema: @@ -23471,8 +27650,10 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error + anyOf: + - $ref: "#/components/schemas/ExploreCacheUnavailableResponse" + - $ref: "#/components/schemas/ErrorResponse" + description: Service Unavailable default: content: application/json: @@ -23481,73 +27662,148 @@ paths: description: Error security: - apiKey: [] - summary: Get a relationship type + summary: Get semantic index coverage for an analytical context tags: - - API - patch: - operationId: patchRelationshipType + - Exploration + /api/v1/search/deep: + get: + operationId: deepSearch parameters: - - description: Relationship type ID - in: path - name: id + - description: Search query + in: query + name: q required: true + schema: + type: string + - description: "Exact search scope: body; omit for composite full-text search" + in: query + name: scope + schema: + type: string + - description: Sender email/address filter; not supported when scope=body + in: query + name: sender + schema: + type: string + - description: Sender display-name filter; not supported when scope=body + in: query + name: sender_name + schema: + type: string + - description: Recipient email/address filter; not supported when scope=body + in: query + name: recipient + schema: + type: string + - description: Recipient display-name filter; not supported when scope=body + in: query + name: recipient_name + schema: + type: string + - description: Domain filter; not supported when scope=body + in: query + name: domain + schema: + type: string + - description: Label filter; not supported when scope=body + in: query + name: label + schema: + type: string + - description: Exact case-insensitive RFC 2919 List-Id filter; not supported when scope=body + in: query + name: list_id + schema: + type: string + - description: Message type filter; not supported when scope=body + in: query + name: message_type + schema: + type: string + - description: Named time period; not supported when scope=body + in: query + name: time_period + schema: + type: string + - description: Time bucket granularity + in: query + name: time_granularity + schema: + type: string + - description: Conversation ID; not supported when scope=body + in: query + name: conversation_id schema: format: int64 type: integer - - description: Strong ETag returned by the latest relationship type read - in: header - name: If-Match - required: true + - description: Source ID + in: query + name: source_id + schema: + format: int64 + type: integer + - description: Source IDs; repeat or comma-separate values; not supported by deep search + in: query + name: source_ids + schema: + items: + format: int64 + type: integer + type: array + - description: Only include messages with attachments + in: query + name: attachments_only + schema: + type: boolean + - description: Exclude deleted messages + in: query + name: hide_deleted + schema: + type: boolean + - description: Lower date/time bound (RFC3339 or YYYY-MM-DD) + in: query + name: after schema: type: string - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/PatchRelationshipTypeRequest" - required: true - responses: - "200": - content: - application/json: - schema: - $ref: "#/components/schemas/RelationshipType" - description: OK - headers: - ETag: - description: Strong relationship type revision tag for optimistic concurrency - schema: - type: string - "400": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "404": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "409": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "428": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "503": + - description: Upper date/time bound (RFC3339 or YYYY-MM-DD) + in: query + name: before + schema: + type: string + - description: Comma-separated aggregate view names to match empty values; not supported when scope=body + in: query + name: empty_targets + schema: + type: string + - description: Zero-based row offset + in: query + name: offset + schema: + format: int64 + type: integer + - description: Maximum number of rows to return (default and max 500; larger values are clamped) + in: query + name: limit + schema: + format: int64 + type: integer + - description: "Sort field: date, size, or subject" + in: query + name: sort + schema: + type: string + - description: "Sort direction: asc or desc" + in: query + name: direction + schema: + type: string + responses: + "200": content: application/json: schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error + $ref: "#/components/schemas/DeepSearchResponse" + description: OK default: content: application/json: @@ -23556,45 +27812,198 @@ paths: description: Error security: - apiKey: [] - summary: Update a relationship type + summary: Run full-text message search tags: - API - /api/v1/relationships: - post: - operationId: listRelationships - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/RelationshipsHTTPRequest" - required: true + /api/v1/search/domains: + get: + operationId: searchMessagesByDomains + parameters: + - description: Comma-separated participant domains + in: query + name: domains + required: true + schema: + type: string + - description: Lower date/time bound (RFC3339 or YYYY-MM-DD) + in: query + name: after + schema: + type: string + - description: Upper date/time bound (RFC3339 or YYYY-MM-DD) + in: query + name: before + schema: + type: string + - description: Zero-based row offset + in: query + name: offset + schema: + format: int64 + type: integer + - description: Maximum number of rows to return + in: query + name: limit + schema: + format: int64 + type: integer responses: "200": content: application/json: schema: - $ref: "#/components/schemas/RelationshipsHTTPResponse" + $ref: "#/components/schemas/FilteredMessagesResponse" description: OK - "400": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "409": + default: content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "503": + security: + - apiKey: [] + summary: Search messages by participant domains + tags: + - API + /api/v1/search/fast: + get: + operationId: fastSearch + parameters: + - description: Search query + in: query + name: q + required: true + schema: + type: string + - description: Stats grouping view type + in: query + name: view_type + schema: + type: string + - description: Sender email/address filter + in: query + name: sender + schema: + type: string + - description: Sender display-name filter + in: query + name: sender_name + schema: + type: string + - description: Recipient email/address filter + in: query + name: recipient + schema: + type: string + - description: Recipient display-name filter + in: query + name: recipient_name + schema: + type: string + - description: Domain filter + in: query + name: domain + schema: + type: string + - description: Label filter + in: query + name: label + schema: + type: string + - description: Exact case-insensitive RFC 2919 List-Id filter + in: query + name: list_id + schema: + type: string + - description: Message type filter + in: query + name: message_type + schema: + type: string + - description: Named time period + in: query + name: time_period + schema: + type: string + - description: Time bucket granularity + in: query + name: time_granularity + schema: + type: string + - description: Conversation ID + in: query + name: conversation_id + schema: + format: int64 + type: integer + - description: Source ID + in: query + name: source_id + schema: + format: int64 + type: integer + - description: Source IDs; repeat or comma-separate values + in: query + name: source_ids + schema: + items: + format: int64 + type: integer + type: array + - description: Only include messages with attachments + in: query + name: attachments_only + schema: + type: boolean + - description: Exclude deleted messages + in: query + name: hide_deleted + schema: + type: boolean + - description: Lower date/time bound (RFC3339 or YYYY-MM-DD) + in: query + name: after + schema: + type: string + - description: Upper date/time bound (RFC3339 or YYYY-MM-DD) + in: query + name: before + schema: + type: string + - description: Comma-separated aggregate view names to match empty values + in: query + name: empty_targets + schema: + type: string + - description: Zero-based row offset + in: query + name: offset + schema: + format: int64 + type: integer + - description: Maximum number of rows to return (default and max 500; larger values are clamped) + in: query + name: limit + schema: + format: int64 + type: integer + - description: "Sort field: date, size, or subject" + in: query + name: sort + schema: + type: string + - description: "Sort direction: asc or desc" + in: query + name: direction + schema: + type: string + responses: + "200": content: application/json: schema: - anyOf: - - $ref: "#/components/schemas/ExploreCacheUnavailableResponse" - - $ref: "#/components/schemas/ErrorResponse" - description: Service Unavailable + $ref: "#/components/schemas/SearchFastResponse" + description: OK default: content: application/json: @@ -23603,53 +28012,88 @@ paths: description: Error security: - apiKey: [] - summary: Rank counterparts by reciprocity-weighted interaction + summary: Run fast aggregate search tags: - - Exploration - /api/v1/relationships/{id}/calendar: - post: - operationId: getRelationshipCalendar + - API + /api/v1/search/similar: + get: + operationId: findSimilarMessages parameters: - - description: Any member participant ID of the counterpart's identity cluster - in: path - name: id + - description: Seed message ID + in: query + name: message_id required: true schema: - format: int64 - type: integer - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/RelationshipCalendarHTTPRequest" - required: true + format: int64 + type: integer + - description: Maximum number of rows to return + in: query + name: limit + schema: + format: int64 + type: integer + - description: Account email or configured source identifier + in: query + name: account + schema: + type: string + - description: Message type filter + in: query + name: message_type + schema: + type: string + - description: Lower date/time bound (RFC3339 or YYYY-MM-DD) + in: query + name: after + schema: + type: string + - description: Upper date/time bound (RFC3339 or YYYY-MM-DD) + in: query + name: before + schema: + type: string + - description: Only include messages with attachments + in: query + name: has_attachment + schema: + type: boolean responses: "200": content: application/json: schema: - $ref: "#/components/schemas/RelationshipCalendarHTTPResponse" + $ref: "#/components/schemas/SimilarSearchResponse" description: OK - "400": + default: content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "404": + security: + - apiKey: [] + summary: Find messages similar to a seed message + tags: + - API + /api/v1/settings: + get: + operationId: getSettings + responses: + "200": content: application/json: schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "503": - content: - application/json: + $ref: "#/components/schemas/SettingsResponse" + description: OK + headers: + Credential-ETag: + description: Strong content hash for the independent provider credential store schema: - anyOf: - - $ref: "#/components/schemas/ExploreCacheUnavailableResponse" - - $ref: "#/components/schemas/ErrorResponse" - description: Service Unavailable + type: string + ETag: + description: Strong content hash for optimistic concurrency + schema: + type: string default: content: application/json: @@ -23658,33 +28102,40 @@ paths: description: Error security: - apiKey: [] - summary: Get one counterpart's timezone-aware relationship calendar + summary: Get browser-managed settings tags: - - Exploration - /api/v1/relationships/{id}/timeline: - post: - operationId: getRelationshipTimeline + - API + patch: + operationId: patchSettings parameters: - - description: Any member participant ID of the counterpart's identity cluster - in: path - name: id + - description: Strong ETag returned by the latest settings read + in: header + name: If-Match required: true schema: - format: int64 - type: integer + type: string requestBody: content: application/json: schema: - $ref: "#/components/schemas/RelationshipTimelineHTTPRequest" + $ref: "#/components/schemas/SettingsPatchRequest" required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/RelationshipTimelineHTTPResponse" + $ref: "#/components/schemas/SettingsResponse" description: OK + headers: + Credential-ETag: + description: Strong content hash for the independent provider credential store + schema: + type: string + ETag: + description: Strong content hash for optimistic concurrency + schema: + type: string "400": content: application/json: @@ -23697,14 +28148,24 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "503": + "412": content: application/json: schema: - anyOf: - - $ref: "#/components/schemas/ExploreCacheUnavailableResponse" - - $ref: "#/components/schemas/ErrorResponse" - description: Service Unavailable + $ref: "#/components/schemas/ErrorResponse" + description: Error + "422": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "428": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error default: content: application/json: @@ -23713,19 +28174,24 @@ paths: description: Error security: - apiKey: [] - summary: Get one counterpart's interaction timeline, with chat grouped into local-day bursts + summary: Update browser-managed settings tags: - - Exploration - /api/v1/saved-views: + - API + /api/v1/settings/people-inference: get: - operationId: listSavedViews + operationId: getSettingsPeopleInference responses: "200": content: application/json: schema: - $ref: "#/components/schemas/SavedViewsResponse" + $ref: "#/components/schemas/PeopleInferenceSettingsResponse" description: OK + headers: + ETag: + description: Strong content hash for optimistic concurrency + schema: + type: string default: content: application/json: @@ -23734,47 +28200,61 @@ paths: description: Error security: - apiKey: [] - summary: List shared analytical Saved Views + summary: Get people inference provider status tags: - API + /api/v1/settings/people-inference/codex/login: post: - operationId: createSavedView + operationId: startSettingsPeopleCodexLogin requestBody: content: application/json: schema: - $ref: "#/components/schemas/CreateSavedViewRequest" + $ref: "#/components/schemas/PeopleCodexLoginRequest" required: true responses: - "201": + "200": content: application/json: schema: - $ref: "#/components/schemas/SavedView" - description: Created - headers: - ETag: - description: Strong Saved View revision tag for optimistic concurrency - schema: - type: string + $ref: "#/components/schemas/PeopleCodexLoginResponse" + description: OK "400": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error "409": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error + "502": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error "503": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error + "504": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error default: content: application/json: @@ -23783,36 +28263,26 @@ paths: description: Error security: - apiKey: [] - summary: Create a shared analytical Saved View + summary: Start a private Codex device login tags: - API - /api/v1/saved-views/{id}: + /api/v1/settings/people-inference/codex/login/{id}: delete: - operationId: deleteSavedView + operationId: cancelSettingsPeopleCodexLogin parameters: - - description: Saved View ID - in: path + - in: path name: id required: true - schema: - format: int64 - type: integer - - description: Strong ETag returned by the latest Saved View read - in: header - name: If-Match - required: true schema: type: string responses: - "204": - description: No Content - "400": + "200": content: application/json: schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "401": + $ref: "#/components/schemas/PeopleCodexLoginStatusResponse" + description: OK + "403": content: application/json: schema: @@ -23830,13 +28300,7 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "428": - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - description: Error - "500": + "502": content: application/json: schema: @@ -23856,37 +28320,48 @@ paths: description: Error security: - apiKey: [] - summary: Delete a shared analytical Saved View + summary: Cancel Codex device login tags: - API get: - operationId: getSavedView + operationId: getSettingsPeopleCodexLogin parameters: - - description: Saved View ID - in: path + - in: path name: id required: true schema: - format: int64 - type: integer + type: string responses: "200": content: application/json: schema: - $ref: "#/components/schemas/SavedView" + $ref: "#/components/schemas/PeopleCodexLoginStatusResponse" description: OK - headers: - ETag: - description: Strong Saved View revision tag for optimistic concurrency + "403": + content: + application/json: schema: - type: string + $ref: "#/components/schemas/ErrorResponse" + description: Error "404": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "502": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error "503": content: application/json: @@ -23901,44 +28376,26 @@ paths: description: Error security: - apiKey: [] - summary: Get a shared analytical Saved View + summary: Get Codex device login status tags: - API - patch: - operationId: patchSavedView + /api/v1/settings/people-inference/codex/login/{id}/models: + get: + operationId: getSettingsPeopleCodexModels parameters: - - description: Saved View ID - in: path + - in: path name: id required: true - schema: - format: int64 - type: integer - - description: Strong ETag returned by the latest Saved View read - in: header - name: If-Match - required: true schema: type: string - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/PatchSavedViewRequest" - required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/SavedView" + $ref: "#/components/schemas/PeopleCodexModelsResponse" description: OK - headers: - ETag: - description: Strong Saved View revision tag for optimistic concurrency - schema: - type: string - "400": + "403": content: application/json: schema: @@ -23956,7 +28413,7 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "428": + "502": content: application/json: schema: @@ -23976,34 +28433,49 @@ paths: description: Error security: - apiKey: [] - summary: Update a shared analytical Saved View + summary: List models for completed Codex login tags: - API - /api/v1/saved-views/{id}/run: - post: - operationId: runSavedView + /api/v1/settings/people-inference/codex/login/{id}/profile: + put: + operationId: putSettingsPeopleCodexProfile parameters: - - description: Saved View ID - in: path + - in: path name: id required: true schema: - format: int64 - type: integer + type: string + - description: Strong config ETag returned by the latest settings read + in: header + name: If-Match + required: true + schema: + type: string requestBody: content: application/json: schema: - $ref: "#/components/schemas/RunSavedViewRequest" + $ref: "#/components/schemas/PeopleCodexProfileRequest" required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/RunSavedViewResponse" - description: OK - "400": + $ref: "#/components/schemas/PeopleInferenceSettingsResponse" + description: OK + headers: + ETag: + description: Strong content hash for optimistic concurrency + schema: + type: string + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "403": content: application/json: schema: @@ -24021,35 +28493,42 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "503": + "412": content: application/json: schema: - anyOf: - - $ref: "#/components/schemas/ExploreCacheUnavailableResponse" - - $ref: "#/components/schemas/ErrorResponse" - description: Service Unavailable - default: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "422": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - security: - - apiKey: [] - summary: Run a shared analytical Saved View through its canonical Explore definition - tags: - - API - /api/v1/scheduler/status: - get: - operationId: getSchedulerStatus - responses: - "200": + "428": content: application/json: schema: - $ref: "#/components/schemas/SchedulerStatusResponse" - description: OK + $ref: "#/components/schemas/ErrorResponse" + description: Error + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "502": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error default: content: application/json: @@ -24058,211 +28537,105 @@ paths: description: Error security: - apiKey: [] - summary: Get scheduler status + summary: Create a Codex profile from completed device login tags: - API - /api/v1/search: - get: - operationId: searchMessages + /api/v1/settings/people-inference/disable: + post: + operationId: disableSettingsPeopleInference parameters: - - description: Search query - in: query - name: q + - description: Strong config ETag returned by the latest settings read + in: header + name: If-Match required: true schema: type: string - - description: "Search mode: fts, vector, or hybrid. conversation_id applies in every mode; other structured filter parameters require vector or hybrid" - in: query - name: mode - schema: - type: string - - description: One-based page number (default 1; values below 1 are clamped to 1). Non-numeric values are rejected with 400. - in: query - name: page - schema: - format: int64 - type: integer - - description: Page size (default 20, max 100; out-of-range values are clamped). Non-numeric values are rejected with 400. - in: query - name: page_size - schema: - format: int64 - type: integer - - description: Zero-based ranking offset for vector or hybrid search (default 0) - in: query - name: offset - schema: - format: int64 - type: integer - - description: Include score explanation when mode is vector or hybrid - in: query - name: explain - schema: - type: boolean - - description: Include scored semantic chunk excerpts for vector or hybrid results - in: query - name: include_matches - schema: - type: boolean - - description: Minimum chunk score for included excerpts; does not filter ranked messages - in: query - name: min_score - schema: - type: number - - description: Message type filter; repeat or comma-separate for multiple values - in: query - name: message_type - schema: - type: string - - description: Restrict to one account/source - in: query - name: account - schema: - type: string - - description: Restrict to one collection - in: query - name: collection - schema: - type: string - - description: Exact conversation ID (all search modes) - in: query - name: conversation_id - schema: - format: int64 - type: integer - - description: Exact sender email/address filter (vector or hybrid mode only) - in: query - name: sender - schema: - type: string - - description: Exact recipient email filter across to, cc, and bcc (vector or hybrid mode only) - in: query - name: recipient - schema: - type: string - - description: Exact sender domain filter (vector or hybrid mode only) - in: query - name: domain - schema: - type: string - - description: Exact case-insensitive label filter (vector or hybrid mode only) - in: query - name: label - schema: - type: string - - description: Exact case-insensitive RFC 2919 List-Id filter (vector or hybrid mode only) - in: query - name: list_id - schema: - type: string - - description: Calendar period in YYYY, YYYY-MM, or YYYY-MM-DD format (vector or hybrid mode only) - in: query - name: time_period - schema: - type: string - - description: Time bucket granularity (vector or hybrid mode only) - in: query - name: time_granularity - schema: - type: string - - description: Exact source ID (vector or hybrid mode only) - in: query - name: source_id - schema: - format: int64 - type: integer - - description: Only include messages with attachments (vector or hybrid mode only) - in: query - name: attachments_only - schema: - type: boolean - - description: Lower date/time bound (RFC3339 or YYYY-MM-DD; vector or hybrid mode only) - in: query - name: after - schema: - type: string - - description: Upper date/time bound (RFC3339 or YYYY-MM-DD; vector or hybrid mode only) - in: query - name: before - schema: - type: string responses: "200": content: application/json: schema: - oneOf: - - $ref: "#/components/schemas/SearchResult" - - $ref: "#/components/schemas/HybridSearchResponse" + $ref: "#/components/schemas/PeopleInferenceSettingsResponse" description: OK - default: + headers: + ETag: + description: Strong content hash for optimistic concurrency + schema: + type: string + "400": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - security: - - apiKey: [] - summary: Search messages - tags: - - API - /api/v1/search/attachments/visual: - post: - operationId: searchVisualAttachments - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/VisualTextSearchRequest" - multipart/form-data: - schema: - properties: - after: - type: string - before: - type: string - cursor: - type: string - direction: - items: - enum: - - from_person - - to_person - - group - type: string - type: array - filename: - type: string - image: - contentMediaType: application/octet-stream - description: Query image (JPEG, PNG, WebP, or still GIF) - format: binary - type: string - limit: - type: string - message_id: - type: string - mime_prefix: - type: string - participant_id: - type: string - person_id: - type: string - sender_person_id: - type: string - source_id: - type: string - required: - - image - type: object - required: true + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "412": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "428": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Disable people inference and revoke active consent + tags: + - API + /api/v1/settings/people-inference/providers/{name}: + delete: + operationId: deleteSettingsPeopleInferenceProvider + parameters: + - in: path + name: name + required: true + schema: + type: string + - description: Strong config ETag returned by the latest settings read + in: header + name: If-Match + required: true + schema: + type: string responses: "200": content: application/json: schema: - $ref: "#/components/schemas/SearchResponse" + $ref: "#/components/schemas/PeopleInferenceSettingsResponse" description: OK + headers: + ETag: + description: Strong content hash for optimistic concurrency + schema: + type: string "400": content: application/json: @@ -24281,6 +28654,24 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error + "412": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "428": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error "503": content: application/json: @@ -24295,189 +28686,83 @@ paths: description: Error security: - apiKey: [] - summary: Search visual attachment content + summary: Remove a people inference provider profile tags: - - Exploration - /api/v1/search/coverage: - post: - operationId: getSearchCoverage + - API + put: + operationId: putSettingsPeopleInferencePreset + parameters: + - in: path + name: name + required: true + schema: + type: string + - description: Strong config ETag returned by the latest settings read + in: header + name: If-Match + required: true + schema: + type: string requestBody: content: application/json: schema: - $ref: "#/components/schemas/SearchCoverageRequest" + $ref: "#/components/schemas/PeopleInferencePresetCreateRequest" required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/SearchCoverageResponse" + $ref: "#/components/schemas/PeopleInferenceSettingsResponse" description: OK + headers: + ETag: + description: Strong content hash for optimistic concurrency + schema: + type: string "400": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - "503": + "409": content: application/json: schema: - anyOf: - - $ref: "#/components/schemas/ExploreCacheUnavailableResponse" - - $ref: "#/components/schemas/ErrorResponse" - description: Service Unavailable - default: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "412": content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" description: Error - security: - - apiKey: [] - summary: Get semantic index coverage for an analytical context - tags: - - Exploration - /api/v1/search/deep: - get: - operationId: deepSearch - parameters: - - description: Search query - in: query - name: q - required: true - schema: - type: string - - description: "Exact search scope: body; omit for composite full-text search" - in: query - name: scope - schema: - type: string - - description: Sender email/address filter; not supported when scope=body - in: query - name: sender - schema: - type: string - - description: Sender display-name filter; not supported when scope=body - in: query - name: sender_name - schema: - type: string - - description: Recipient email/address filter; not supported when scope=body - in: query - name: recipient - schema: - type: string - - description: Recipient display-name filter; not supported when scope=body - in: query - name: recipient_name - schema: - type: string - - description: Domain filter; not supported when scope=body - in: query - name: domain - schema: - type: string - - description: Label filter; not supported when scope=body - in: query - name: label - schema: - type: string - - description: Exact case-insensitive RFC 2919 List-Id filter; not supported when scope=body - in: query - name: list_id - schema: - type: string - - description: Message type filter; not supported when scope=body - in: query - name: message_type - schema: - type: string - - description: Named time period; not supported when scope=body - in: query - name: time_period - schema: - type: string - - description: Time bucket granularity - in: query - name: time_granularity - schema: - type: string - - description: Conversation ID; not supported when scope=body - in: query - name: conversation_id - schema: - format: int64 - type: integer - - description: Source ID - in: query - name: source_id - schema: - format: int64 - type: integer - - description: Source IDs; repeat or comma-separate values; not supported by deep search - in: query - name: source_ids - schema: - items: - format: int64 - type: integer - type: array - - description: Only include messages with attachments - in: query - name: attachments_only - schema: - type: boolean - - description: Exclude deleted messages - in: query - name: hide_deleted - schema: - type: boolean - - description: Lower date/time bound (RFC3339 or YYYY-MM-DD) - in: query - name: after - schema: - type: string - - description: Upper date/time bound (RFC3339 or YYYY-MM-DD) - in: query - name: before - schema: - type: string - - description: Comma-separated aggregate view names to match empty values; not supported when scope=body - in: query - name: empty_targets - schema: - type: string - - description: Zero-based row offset - in: query - name: offset - schema: - format: int64 - type: integer - - description: Maximum number of rows to return (default and max 500; larger values are clamped) - in: query - name: limit - schema: - format: int64 - type: integer - - description: "Sort field: date, size, or subject" - in: query - name: sort - schema: - type: string - - description: "Sort direction: asc or desc" - in: query - name: direction - schema: - type: string - responses: - "200": + "422": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "428": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": content: application/json: schema: - $ref: "#/components/schemas/DeepSearchResponse" - description: OK + $ref: "#/components/schemas/ErrorResponse" + description: Error default: content: application/json: @@ -24486,48 +28771,73 @@ paths: description: Error security: - apiKey: [] - summary: Run full-text message search + summary: Create a vendor-bound people inference provider tags: - API - /api/v1/search/domains: - get: - operationId: searchMessagesByDomains + /api/v1/settings/people-inference/providers/{name}/check: + post: + operationId: checkSettingsPeopleInferenceProvider parameters: - - description: Comma-separated participant domains - in: query - name: domains + - in: path + name: name required: true schema: type: string - - description: Lower date/time bound (RFC3339 or YYYY-MM-DD) - in: query - name: after - schema: - type: string - - description: Upper date/time bound (RFC3339 or YYYY-MM-DD) - in: query - name: before + - description: Strong config ETag returned by the latest settings read + in: header + name: If-Match + required: true schema: type: string - - description: Zero-based row offset - in: query - name: offset - schema: - format: int64 - type: integer - - description: Maximum number of rows to return - in: query - name: limit - schema: - format: int64 - type: integer responses: "200": content: application/json: schema: - $ref: "#/components/schemas/FilteredMessagesResponse" + $ref: "#/components/schemas/PeopleInferenceCheckResponse" description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "412": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "428": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "502": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error default: content: application/json: @@ -24536,139 +28846,102 @@ paths: description: Error security: - apiKey: [] - summary: Search messages by participant domains + summary: Run a synthetic people inference provider check tags: - API - /api/v1/search/fast: - get: - operationId: fastSearch + /api/v1/settings/people-inference/providers/{name}/consent: + post: + operationId: consentSettingsPeopleInferenceProvider parameters: - - description: Search query - in: query - name: q + - in: path + name: name required: true schema: type: string - - description: Stats grouping view type - in: query - name: view_type - schema: - type: string - - description: Sender email/address filter - in: query - name: sender - schema: - type: string - - description: Sender display-name filter - in: query - name: sender_name - schema: - type: string - - description: Recipient email/address filter - in: query - name: recipient - schema: - type: string - - description: Recipient display-name filter - in: query - name: recipient_name - schema: - type: string - - description: Domain filter - in: query - name: domain - schema: - type: string - - description: Label filter - in: query - name: label - schema: - type: string - - description: Exact case-insensitive RFC 2919 List-Id filter - in: query - name: list_id - schema: - type: string - - description: Message type filter - in: query - name: message_type - schema: - type: string - - description: Named time period - in: query - name: time_period - schema: - type: string - - description: Time bucket granularity - in: query - name: time_granularity - schema: - type: string - - description: Conversation ID - in: query - name: conversation_id - schema: - format: int64 - type: integer - - description: Source ID - in: query - name: source_id - schema: - format: int64 - type: integer - - description: Source IDs; repeat or comma-separate values - in: query - name: source_ids - schema: - items: - format: int64 - type: integer - type: array - - description: Only include messages with attachments - in: query - name: attachments_only - schema: - type: boolean - - description: Exclude deleted messages - in: query - name: hide_deleted - schema: - type: boolean - - description: Lower date/time bound (RFC3339 or YYYY-MM-DD) - in: query - name: after - schema: - type: string - - description: Upper date/time bound (RFC3339 or YYYY-MM-DD) - in: query - name: before - schema: - type: string - - description: Comma-separated aggregate view names to match empty values - in: query - name: empty_targets + - description: Strong config ETag returned by the latest settings read + in: header + name: If-Match + required: true schema: type: string - - description: Zero-based row offset - in: query - name: offset - schema: - format: int64 - type: integer - - description: Maximum number of rows to return (default and max 500; larger values are clamped) - in: query - name: limit - schema: - format: int64 - type: integer - - description: "Sort field: date, size, or subject" - in: query - name: sort + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/PeopleInferenceConsentRequest" + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/PeopleInferenceSettingsResponse" + description: OK + headers: + ETag: + description: Strong content hash for optimistic concurrency + schema: + type: string + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "412": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "428": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + default: + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + security: + - apiKey: [] + summary: Grant exact people inference consent + tags: + - API + /api/v1/settings/people-inference/providers/{name}/key: + delete: + operationId: deleteSettingsPeopleInferenceKey + parameters: + - in: path + name: name + required: true schema: type: string - - description: "Sort direction: asc or desc" - in: query - name: direction + - description: Opaque revision for this people provider credential + in: header + name: If-Match + required: true schema: type: string responses: @@ -24676,8 +28949,55 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/SearchFastResponse" + $ref: "#/components/schemas/PeopleInferenceSettingsResponse" description: OK + headers: + ETag: + description: Strong content hash for optimistic concurrency + schema: + type: string + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "412": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "428": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error default: content: application/json: @@ -24686,58 +29006,83 @@ paths: description: Error security: - apiKey: [] - summary: Run fast aggregate search + summary: Clear a stored people inference API key tags: - API - /api/v1/search/similar: - get: - operationId: findSimilarMessages + put: + operationId: putSettingsPeopleInferenceKey parameters: - - description: Seed message ID - in: query - name: message_id + - in: path + name: name required: true - schema: - format: int64 - type: integer - - description: Maximum number of rows to return - in: query - name: limit - schema: - format: int64 - type: integer - - description: Account email or configured source identifier - in: query - name: account - schema: - type: string - - description: Message type filter - in: query - name: message_type schema: type: string - - description: Lower date/time bound (RFC3339 or YYYY-MM-DD) - in: query - name: after - schema: - type: string - - description: Upper date/time bound (RFC3339 or YYYY-MM-DD) - in: query - name: before + - description: Opaque revision for this people provider credential + in: header + name: If-Match + required: true schema: type: string - - description: Only include messages with attachments - in: query - name: has_attachment - schema: - type: boolean + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/PeopleInferenceKeyWriteRequest" + required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/SimilarSearchResponse" + $ref: "#/components/schemas/PeopleInferenceSettingsResponse" description: OK + headers: + ETag: + description: Strong content hash for optimistic concurrency + schema: + type: string + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "412": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "428": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error default: content: application/json: @@ -24746,28 +29091,72 @@ paths: description: Error security: - apiKey: [] - summary: Find messages similar to a seed message + summary: Set a write-only people inference API key tags: - API - /api/v1/settings: - get: - operationId: getSettings + /api/v1/settings/people-inference/providers/{name}/revoke: + post: + operationId: revokeSettingsPeopleInferenceProvider + parameters: + - in: path + name: name + required: true + schema: + type: string + - description: Strong config ETag returned by the latest settings read + in: header + name: If-Match + required: true + schema: + type: string responses: "200": content: application/json: schema: - $ref: "#/components/schemas/SettingsResponse" + $ref: "#/components/schemas/PeopleInferenceSettingsResponse" description: OK headers: - Credential-ETag: - description: Strong content hash for the independent provider credential store - schema: - type: string ETag: description: Strong content hash for optimistic concurrency schema: type: string + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "412": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "428": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error default: content: application/json: @@ -24776,13 +29165,14 @@ paths: description: Error security: - apiKey: [] - summary: Get browser-managed settings + summary: Revoke exact people inference consent tags: - API - patch: - operationId: patchSettings + /api/v1/settings/people-inference/select: + post: + operationId: selectSettingsPeopleInference parameters: - - description: Strong ETag returned by the latest settings read + - description: Strong config ETag returned by the latest settings read in: header name: If-Match required: true @@ -24792,20 +29182,16 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/SettingsPatchRequest" + $ref: "#/components/schemas/PeopleInferenceSelectionRequest" required: true responses: "200": content: application/json: schema: - $ref: "#/components/schemas/SettingsResponse" + $ref: "#/components/schemas/PeopleInferenceSettingsResponse" description: OK headers: - Credential-ETag: - description: Strong content hash for the independent provider credential store - schema: - type: string ETag: description: Strong content hash for optimistic concurrency schema: @@ -24840,6 +29226,18 @@ paths: schema: $ref: "#/components/schemas/ErrorResponse" description: Error + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error + "503": + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + description: Error default: content: application/json: @@ -24848,7 +29246,7 @@ paths: description: Error security: - apiKey: [] - summary: Update browser-managed settings + summary: Select a checked and consented people inference provider tags: - API /api/v1/settings/person-enrichment/providers/{name}: diff --git a/cmd/msgvault-codex-bridge/main.go b/cmd/msgvault-codex-bridge/main.go new file mode 100644 index 000000000..ba616f9d4 --- /dev/null +++ b/cmd/msgvault-codex-bridge/main.go @@ -0,0 +1,52 @@ +//go:build linux + +// msgvault-codex-bridge provides a loopback HTTP proxy endpoint inside the +// Codex network namespace. It forwards bytes to the daemon's Unix CONNECT +// policy service; only that service can open an upstream connection. +package main + +import ( + "io" + "net" + "os" + "os/exec" +) + +func main() { + listener, err := net.Listen("tcp", "127.0.0.1:3128") + if err != nil { + os.Exit(1) + } + go func() { + for { + client, err := listener.Accept() + if err != nil { + return + } + go relay(client) + } + }() + command := exec.Command("/codex", os.Args[1:]...) //nolint:gosec // The launcher fixes /codex and validates the app-server arguments. + command.Stdin, command.Stdout, command.Stderr = os.Stdin, os.Stdout, os.Stderr + if err := command.Run(); err != nil { + os.Exit(1) + } +} + +func relay(client net.Conn) { + defer func() { _ = client.Close() }() + upstream, err := net.Dial("unix", "/work/.proxy.sock") + if err != nil { + return + } + defer func() { _ = upstream.Close() }() + done := make(chan struct{}) + go func() { + _, _ = io.Copy(upstream, client) + close(done) + }() + _, _ = io.Copy(client, upstream) + _ = client.Close() + _ = upstream.Close() + <-done +} diff --git a/cmd/msgvault/cmd/account_identity.go b/cmd/msgvault/cmd/account_identity.go index 56cb66506..bb8d876df 100644 --- a/cmd/msgvault/cmd/account_identity.go +++ b/cmd/msgvault/cmd/account_identity.go @@ -1,32 +1,86 @@ package cmd import ( + "encoding/json/jsontext" + "encoding/json/v2" "fmt" "io" + "log/slog" "strings" + "github.com/spf13/cobra" "go.kenn.io/msgvault/internal/store" ) // noDefaultIdentityHelp is the flag help text for --no-default-identity. // Each ingest command registers its own bool variable and reuses this constant. -const noDefaultIdentityHelp = "Suppress auto-default-identity at account creation. " + +const noDefaultIdentityHelp = "Suppress automatic default identity confirmation. " + "Note: a one-time legacy [identity] config migration may still write confirmed " + "identifiers to the account on first post-upgrade startup." -// confirmDefaultIdentity writes one confirmed identifier to a freshly -// created source's identity. Best-effort: any error is logged and swallowed -// so a partially failed identity write never breaks ingest. Empty identifiers -// are a silent no-op. +const savedDefaultIdentityHelp = noDefaultIdentityHelp + + " Saved for later syncs and re-authorization; omit the flag to keep the choice, " + + "or use --no-default-identity=false to re-enable defaults." + +// setDefaultIdentityOptOut changes the saved choice only for an explicit flag. +// source must contain the config from before registration: IMAP add commands +// replace provider settings, so an omitted flag must carry the old choice forward. +func setDefaultIdentityOptOut(cmd *cobra.Command, s *store.Store, source *store.Source, optOut bool) error { + src, err := s.GetSourceByID(source.ID) + if err != nil { + return fmt.Errorf("read identity preference: %w", err) + } + if !cmd.Flags().Changed("no-default-identity") { + if src.SyncConfig == source.SyncConfig { + return nil + } + var previous struct { + NoDefaultIdentity bool `json:"no_default_identity"` + } + if source.SyncConfig.Valid { + if err := json.Unmarshal([]byte(source.SyncConfig.String), &previous); err != nil { + return fmt.Errorf("parse saved identity preference: %w", err) + } + } + optOut = previous.NoDefaultIdentity + } + cfg := make(map[string]jsontext.Value) + if src.SyncConfig.Valid { + if err := json.Unmarshal([]byte(src.SyncConfig.String), &cfg); err != nil { + return fmt.Errorf("parse identity preference: %w", err) + } + } + if optOut { + if cfg == nil { + cfg = make(map[string]jsontext.Value) + } + cfg["no_default_identity"] = jsontext.Value("true") + } else { + if _, exists := cfg["no_default_identity"]; !exists { + return nil + } + delete(cfg, "no_default_identity") + } + encoded, err := json.Marshal(cfg, json.Deterministic(true)) + if err != nil { + return fmt.Errorf("encode identity preference: %w", err) + } + if err := s.UpdateSourceSyncConfig(source.ID, string(encoded)); err != nil { + return fmt.Errorf("save identity preference: %w", err) + } + return nil +} + +// confirmDefaultIdentity writes one confirmed identifier to a source that has +// no saved opt-out or existing confirmed identity. Best-effort: any error is +// logged and swallowed so a partially failed identity write never breaks +// ingest. Empty identifiers are a silent no-op. // -// Skips the write when the source already has at least one identity row. -// add-account / add-imap / add-o365 / import-* commands all call this on -// every invocation (including reruns and rebinds), so without this guard -// an identity the user explicitly removed via `identity remove` would be -// re-added on the next ingest re-run, silently affecting dedup sent-copy -// detection. The guard preserves the documented "freshly created source" -// intent while degrading gracefully if the user has removed every -// identity (in which case the default is restored, which is desirable). +// Skips the write when the source opted out or already has an identity row. +// Removing a source's last confirmed identity saves the default-identity +// opt-out, so scheduled sync and later add-command reruns cannot restore the +// removed identifier. A source can explicitly clear that choice with +// --no-default-identity=false. // // **Ordering note:** ingest commands MUST call confirmDefaultIdentity // BEFORE runPostSourceCreateMigrations on the same invocation. The @@ -40,11 +94,26 @@ const noDefaultIdentityHelp = "Suppress auto-default-identity at account creatio // // account is the user-facing account name shown in the confirmation message. // Callers should gate this behind the per-command --no-default-identity flag. -func confirmDefaultIdentity(out io.Writer, s *store.Store, sourceID int64, account, identifier, signal string) { +func confirmDefaultIdentity(out io.Writer, s *store.Store, sourceID int64, account, identifier, signal string, logger *slog.Logger) { id := strings.TrimSpace(identifier) if id == "" { return } + src, err := s.GetSourceByID(sourceID) + var cfg struct { + NoDefaultIdentity bool `json:"no_default_identity"` + } + if err == nil && src.SyncConfig.Valid { + err = json.Unmarshal([]byte(src.SyncConfig.String), &cfg) + } + if err != nil { + logger.Warn("auto-default-identity preference check failed", + "source_id", sourceID, "account", account, "error", err.Error()) + return + } + if cfg.NoDefaultIdentity { + return + } existing, err := s.ListAccountIdentities(sourceID) if err != nil { logger.Warn("auto-default-identity precheck failed", diff --git a/cmd/msgvault/cmd/account_identity_sync_test.go b/cmd/msgvault/cmd/account_identity_sync_test.go new file mode 100644 index 000000000..d4f9482e5 --- /dev/null +++ b/cmd/msgvault/cmd/account_identity_sync_test.go @@ -0,0 +1,230 @@ +package cmd + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/json/v2" + "encoding/pem" + "fmt" + "io" + "net" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/microsoft" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil" +) + +type testTransport func(*http.Request) (*http.Response, error) + +func (f testTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +func TestAddServiceAccountDefaultIdentityScheduledSync(t *testing.T) { + assert, require := assert.New(t), require.New(t) + saveAddAccountFlags(t) + // Only Google's token and profile responses are simulated. Registration, + // service-account token creation, scheduled sync, and database writes are real. + // Scheduled Gmail sync constructs its HTTP client with context.Background(), + // so oauth2.HTTPClient in the invocation context cannot intercept its requests. + // Keep this test nonparallel and restore the transport after it runs. + savedTransport := http.DefaultTransport + t.Cleanup(func() { http.DefaultTransport = savedTransport }) + http.DefaultTransport = testTransport(func(req *http.Request) (*http.Response, error) { + var body string + switch req.Method + " " + req.URL.String() { + case "POST https://token.example.com/oauth2": + body = `{"access_token":"synthetic-token","token_type":"Bearer","expires_in":3600}` + case "GET https://gmail.googleapis.com/gmail/v1/users/me/profile": + body = `{"emailAddress":"user@example.com","historyId":"100"}` + default: + return nil, fmt.Errorf("unexpected request: %s %s", req.Method, req.URL) + } + return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(body)), Request: req}, nil + }) + + home := t.TempDir() + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(err) + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + require.NoError(err) + keyJSON, err := json.Marshal(map[string]string{ + "type": "service_account", + "client_email": "service@example.com", + "private_key": string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER})), + "token_uri": "https://token.example.com/oauth2", + }) + require.NoError(err) + keyPath := filepath.Join(home, "service-account.json") + require.NoError(os.WriteFile(keyPath, keyJSON, 0600)) + cfg := &config.Config{HomeDir: home, Data: config.DataConfig{DataDir: home}, + OAuth: config.OAuthConfig{ServiceAccountKey: keyPath}} + ctx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + + for _, flag := range []string{"--no-default-identity", "", "--no-default-identity=false"} { + optOut := flag != "--no-default-identity=false" + cmd := &cobra.Command{Use: addAccountUse, RunE: runAddAccountLocal} + registerAddAccountFlags(cmd) + args := []string{"user@example.com"} + if flag != "" { + args = append(args, flag) + } + cmd.SetArgs(args) + require.NoError(cmd.ExecuteContext(ctx)) + st, err := store.Open(cfg.DatabaseDSN()) + require.NoError(err) + t.Cleanup(func() { _ = st.Close() }) + src, err := findGmailSource(st, "user@example.com") + require.NoError(err) + // Seed the cursor of an already-synced mailbox so this scheduled run + // completes with no new messages. + require.NoError(st.UpdateSourceSyncCursor(src.ID, "100")) + summary, err := runScheduledGmailSync(ctx, "user@example.com", src, st, nil, invocationFromContext(ctx)) + require.NoError(err) + assert.Zero(summary.Errors) + ids, err := st.ListAccountIdentities(src.ID) + require.NoError(err) + if optOut { + assert.Empty(ids, "scheduled sync must preserve the service-account opt-out") + } else { + require.Len(ids, 1) + assert.Equal("user@example.com", ids[0].Address) + if flag == "--no-default-identity=false" { + removed, err := st.RemoveAccountIdentity(src.ID, "user@example.com") + require.NoError(err) + require.EqualValues(1, removed) + summary, err = runScheduledGmailSync(ctx, "user@example.com", src, st, nil, invocationFromContext(ctx)) + require.NoError(err) + assert.Zero(summary.Errors) + ids, err = st.ListAccountIdentities(src.ID) + require.NoError(err) + assert.Empty(ids, "scheduled Gmail sync must not restore an explicitly removed last identity") + } + } + } +} + +func TestAddIMAPDefaultIdentityScheduledSync(t *testing.T) { + assert, require := assert.New(t), require.New(t) + savedHost, savedPort, savedUsername := imapHost, imapPort, imapUsername + savedNoTLS, savedSTARTTLS, savedNoDefault := imapNoTLS, imapSTARTTLS, noDefaultIdentityAddImap + t.Cleanup(func() { + imapHost, imapPort, imapUsername = savedHost, savedPort, savedUsername + imapNoTLS, imapSTARTTLS, noDefaultIdentityAddImap = savedNoTLS, savedSTARTTLS, savedNoDefault + }) + t.Setenv(daemonCLISubprocessEnv, strconv.Itoa(os.Getppid())) + t.Setenv("MSGVAULT_IMAP_PASSWORD", testutil.IMAPTestPassword) + addr, _ := testutil.StartIMAPMemServerWithSpecialUse(t, map[string]int{"INBOX": 1}, nil) + host, port, err := net.SplitHostPort(addr) + require.NoError(err) + home := t.TempDir() + cfg := &config.Config{HomeDir: home, Data: config.DataConfig{DataDir: home}} + ctx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + + // Re-registering preserves the choice unless the flag is explicit. + for _, flag := range []string{"--no-default-identity", "", "--no-default-identity=false"} { + optOut := flag != "--no-default-identity=false" + cmd := newAddIMAPCmd() + args := []string{"--host", host, "--port", port, "--username", testutil.IMAPTestUsername, "--no-tls"} + if flag != "" { + args = append(args, flag) + } + cmd.SetArgs(args) + require.NoError(cmd.ExecuteContext(ctx)) + st, err := store.Open(cfg.DatabaseDSN()) + require.NoError(err) + t.Cleanup(func() { _ = st.Close() }) + sources, err := st.ListSources(sourceTypeIMAP) + require.NoError(err) + require.Len(sources, 1) + src := sources[0] + // Re-enabling defaults creates the identity before this explicit removal. + // The next scheduled sync must preserve the removal. + if !optOut { + removed, err := st.RemoveAccountIdentity(src.ID, testutil.IMAPTestUsername) + require.NoError(err) + require.EqualValues(1, removed) + } + summary, err := runScheduledIMAPSync(ctx, src, st, invocationFromContext(ctx)) + require.NoError(err) + assert.Zero(summary.Errors) + ids, err := st.ListAccountIdentities(src.ID) + require.NoError(err) + assert.Empty(ids, "scheduled sync must preserve the saved opt-out or last-identity removal") + } +} + +func TestAddMicrosoftDefaultIdentityOptOut(t *testing.T) { + savedGraph := o365Graph + savedO365, savedTeams := noDefaultIdentityAddO365, noDefaultIdentityAddTeams + savedO365Headless, savedTeamsHeadless := o365Headless, teamsHeadless + savedO365Tenant, savedTeamsTenant := o365TenantID, teamsTenantID + t.Cleanup(func() { + o365Graph = savedGraph + noDefaultIdentityAddO365, noDefaultIdentityAddTeams = savedO365, savedTeams + o365Headless, teamsHeadless = savedO365Headless, savedTeamsHeadless + o365TenantID, teamsTenantID = savedO365Tenant, savedTeamsTenant + }) + for _, tc := range []struct { + name string + newCommand func() *cobra.Command + args []string + }{ + {"o365", newAddO365LocalCmd, nil}, + {"graph", newAddO365LocalCmd, []string{"--graph"}}, + {"teams", newAddTeamsLocalCmd, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + assert, require := assert.New(t), require.New(t) + home := t.TempDir() + cfg := &config.Config{HomeDir: home, Data: config.DataConfig{DataDir: home}, + Microsoft: config.MicrosoftConfig{ClientID: "synthetic-client"}} + ctx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + const email = "user@example.com" + mgr := microsoft.NewManager(cfg.Microsoft.ClientID, "common", cfg.Microsoft.EffectiveRedirectURI(), cfg.TokensDir(), testDiscardLogger()) + require.NoError(os.MkdirAll(cfg.TokensDir(), 0700)) + require.NoError(os.WriteFile(mgr.TokenPath(email), []byte(`{"access_token":"synthetic-token"}`), 0600)) + for _, flag := range []string{"--no-default-identity", "", "--no-default-identity=false"} { + cmd := tc.newCommand() + args := append([]string{email, "--" + oauthPreflightedFlag}, tc.args...) + if flag != "" { + args = append(args, flag) + } + cmd.SetArgs(args) + require.NoError(cmd.ExecuteContext(ctx)) + st, err := store.Open(cfg.DatabaseDSN()) + require.NoError(err) + t.Cleanup(func() { _ = st.Close() }) + sources, err := st.ListSources("") + require.NoError(err) + require.Len(sources, 1) + if tc.name == "graph" { + // Identity setup precedes token loading. With no Graph token, + // the real scheduled path stops before making network requests. + err := runScheduledMSMailSync(ctx, sources[0], st, invocationFromContext(ctx)) + require.ErrorContains(err, "no valid token") + } else { + confirmDefaultIdentity(io.Discard, st, sources[0].ID, email, email, "account-identifier", testDiscardLogger()) + } + ids, err := st.ListAccountIdentities(sources[0].ID) + require.NoError(err) + if flag == "--no-default-identity=false" { + require.Len(ids, 1, "explicit false restores the default") + assert.Equal(email, ids[0].Address) + } else { + assert.Empty(ids, "default identity confirmation must honor the saved opt-out (flag %q)", flag) + } + } + }) + } +} diff --git a/cmd/msgvault/cmd/account_identity_test.go b/cmd/msgvault/cmd/account_identity_test.go index 79cd739f5..352b52654 100644 --- a/cmd/msgvault/cmd/account_identity_test.go +++ b/cmd/msgvault/cmd/account_identity_test.go @@ -21,7 +21,7 @@ func TestConfirmDefaultIdentity_HappyPath(t *testing.T) { src, err := s.GetOrCreateSource("gmail", "alice@example.com") require.NoError(err) - confirmDefaultIdentity(io.Discard, s, src.ID, "alice@example.com", "alice@example.com", "account-identifier") + confirmDefaultIdentity(io.Discard, s, src.ID, "alice@example.com", "alice@example.com", "account-identifier", slog.New(slog.DiscardHandler)) rows, err := s.ListAccountIdentities(src.ID) require.NoError(err) require.Len(rows, 1, "got %+v", rows) @@ -39,12 +39,14 @@ func TestConfirmDefaultIdentity_EmptyIdentifierIsNoOp(t *testing.T) { src, err := s.GetOrCreateSource("gmail", "alice@example.com") require.NoError(err) - confirmDefaultIdentity(io.Discard, s, src.ID, "alice@example.com", "", "account-identifier") + confirmDefaultIdentity(io.Discard, s, src.ID, "alice@example.com", "", "account-identifier", slog.New(slog.DiscardHandler)) rows, _ := s.ListAccountIdentities(src.ID) assert.Empty(t, rows, "want empty, got %+v", rows) } func TestConfirmDefaultIdentity_StoreErrorDoesNotPanic(t *testing.T) { + logger := testLoggerValue() + tmpDir := t.TempDir() s, err := store.Open(filepath.Join(tmpDir, "msgvault.db")) require.NoError(t, err) @@ -61,7 +63,7 @@ func TestConfirmDefaultIdentity_StoreErrorDoesNotPanic(t *testing.T) { // sourceID 99999 does not exist; FK violation returns an error // from AddAccountIdentity. The helper must swallow it. - confirmDefaultIdentity(io.Discard, s, 99999, "ghost@example.com", "ghost@example.com", "account-identifier") + confirmDefaultIdentity(io.Discard, s, 99999, "ghost@example.com", "ghost@example.com", "account-identifier", slog.New(slog.DiscardHandler)) } // TestConfirmDefaultIdentity_LegacyMigrationOverridesNoDefault pins the diff --git a/cmd/msgvault/cmd/activity.go b/cmd/msgvault/cmd/activity.go index cd031c8f7..ca07642ea 100644 --- a/cmd/msgvault/cmd/activity.go +++ b/cmd/msgvault/cmd/activity.go @@ -2,8 +2,10 @@ package cmd import ( "context" + "errors" "fmt" "log/slog" + "time" "github.com/spf13/cobra" activitypkg "go.kenn.io/msgvault/internal/activity" @@ -13,8 +15,13 @@ import ( ) const ( - activityBuildSubcommand = "build" - activityProjectionJob = "activity-projection" + activityBuildSubcommand = "build" + activityProjectionJob = "activity-projection" + activityProjectionMaxBatches = 10 + // The scheduler asks a preemptible job to stop after it has held the gate + // for a minute while other work waits. Projection stops at its next batch + // boundary; the longer hard limit leaves room for that batch to commit. + activityProjectionMaxRuntime = 2 * time.Minute ) func newActivityCommand() *cobra.Command { @@ -45,7 +52,13 @@ func newActivityCommand() *cobra.Command { } func runActivityBuildLocal(cmd *cobra.Command, backstop bool) error { - st, cleanup, err := openWritableStoreAndInit() + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger + st, cleanup, err := openWritableStoreAndInitForInvocation(state) if err != nil { return err } @@ -98,16 +111,26 @@ func registerActivityProjectionJob( Timezone: activityConfig.Timezone, MaxDirectCounterparts: activityConfig.MaxDirectCounterparts, BatchSize: activityConfig.BatchSize, + MaxBatches: activityProjectionMaxBatches, Log: log, }) if err != nil { return err } return sched.AddJob(scheduler.Job{ - Name: activityProjectionJob, - Schedule: activityConfig.Schedule, + Name: activityProjectionJob, + Preemptible: true, + MaxRuntime: activityProjectionMaxRuntime, + Schedule: activityConfig.Schedule, Run: func(ctx context.Context) error { result, runErr := projector.RunOnce(ctx) + if errors.Is(runErr, activitypkg.ErrWorkRemaining) { + log.Info("activity projection pass complete; continuing behind queued work", "processed", result.Processed, "batches", result.Batches) + return scheduler.ErrReschedule + } + if runErr != nil && (errors.Is(runErr, context.Canceled) || errors.Is(runErr, context.DeadlineExceeded)) { + return runErr + } if runErr != nil { log.Error("activity projection failed", "error", runErr, diff --git a/cmd/msgvault/cmd/activity_test.go b/cmd/msgvault/cmd/activity_test.go index a68379ff3..90f105e9d 100644 --- a/cmd/msgvault/cmd/activity_test.go +++ b/cmd/msgvault/cmd/activity_test.go @@ -2,15 +2,19 @@ package cmd import ( "bytes" + "context" "database/sql" + "fmt" "log/slog" "path/filepath" + "sync" "testing" "time" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/api" "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/scheduler" "go.kenn.io/msgvault/internal/store" @@ -18,12 +22,16 @@ import ( ) func TestRunActivityBuildLocalUsesConfigAndBackstop(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) markDaemonCLISubprocessForTest(t) savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Data.DataDir = filepath.Join(t.TempDir(), "data") cfg.Activity.Timezone = "Pacific/Kiritimati" cfg.Activity.BatchSize = 1 @@ -86,7 +94,8 @@ func TestRunActivityBuildLocalUsesConfigAndBackstop(t *testing.T) { var output bytes.Buffer command := newActivityCommand() - command.SetContext(t.Context()) + command.SetContext(testCtx) + command.SetContext(testCtx) command.SetOut(&output) command.SetArgs([]string{"build"}) require.NoError(command.Execute()) @@ -95,7 +104,7 @@ func TestRunActivityBuildLocalUsesConfigAndBackstop(t *testing.T) { st, err = store.OpenForTest(cfg.DatabaseDSN()) require.NoError(err) var timezone, localDate string - require.NoError(st.DB().QueryRowContext(t.Context(), st.Rebind(` + require.NoError(st.DB().QueryRowContext(testCtx, st.Rebind(` SELECT timezone, local_date FROM activity_events WHERE message_id = ? @@ -122,21 +131,22 @@ func TestRunActivityBuildLocalUsesConfigAndBackstop(t *testing.T) { assert.Equal([]string{"co_presence", "co_presence"}, evidence, "configured max_direct_counterparts=1 must classify two recipients as broadcast") }() - _, err = st.DB().ExecContext(t.Context(), st.Rebind( + _, err = st.DB().ExecContext(testCtx, st.Rebind( `DELETE FROM activity_events WHERE message_id = ?`), messageID) require.NoError(err) require.NoError(st.Close()) output.Reset() command = newActivityCommand() - command.SetContext(t.Context()) + command.SetContext(testCtx) + command.SetContext(testCtx) command.SetOut(&output) command.SetArgs([]string{"build"}) require.NoError(command.Execute()) st, err = store.OpenForTest(cfg.DatabaseDSN()) require.NoError(err) var count int - require.NoError(st.DB().QueryRowContext(t.Context(), st.Rebind( + require.NoError(st.DB().QueryRowContext(testCtx, st.Rebind( `SELECT COUNT(*) FROM activity_events WHERE message_id = ?`), messageID).Scan(&count)) assert.Zero(count, "ordinary build must not force-scan below the watermark") @@ -144,7 +154,8 @@ func TestRunActivityBuildLocalUsesConfigAndBackstop(t *testing.T) { output.Reset() command = newActivityCommand() - command.SetContext(t.Context()) + command.SetContext(testCtx) + command.SetContext(testCtx) command.SetOut(&output) command.SetArgs([]string{"build", "--backstop"}) require.NoError(command.Execute()) @@ -152,13 +163,15 @@ func TestRunActivityBuildLocalUsesConfigAndBackstop(t *testing.T) { st, err = store.OpenForTest(cfg.DatabaseDSN()) require.NoError(err) t.Cleanup(func() { _ = st.Close() }) - require.NoError(st.DB().QueryRowContext(t.Context(), st.Rebind( + require.NoError(st.DB().QueryRowContext(testCtx, st.Rebind( `SELECT COUNT(*) FROM activity_events WHERE message_id = ?`), messageID).Scan(&count)) assert.Equal(1, count) } func TestActivityBuildProxiesThroughDaemonCLIRunner(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) @@ -169,11 +182,13 @@ func TestActivityBuildProxiesThroughDaemonCLIRunner(t *testing.T) { `{"type":"stdout","data":"Projected 2 event(s) in 1 batch(es); touched 1 and recomputed 1 person(s); watermark 2.\n"}`, `{"type":"complete"}`, ) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx cfg.Data.DataDir = t.TempDir() var stdout bytes.Buffer root := &cobra.Command{Use: "msgvault"} + root.SetContext(testCtx) root.AddCommand(newActivityCommand()) root.SetOut(&stdout) root.SetArgs([]string{"activity", "build", "--backstop"}) @@ -230,3 +245,67 @@ func TestRegisterActivityProjectionJobHonorsDisabledSchedule(t *testing.T) { sched, f.Store, activityConfig, slog.Default())) assert.False(t, sched.IsJobScheduled(activityProjectionJob)) } + +func TestScheduledProjectionYieldsToSyncBetweenPasses(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := storetest.New(t) + for i := range 23 { + f.NewMessage().WithSourceMessageID(fmt.Sprintf("scheduled-projection-%d", i)). + WithSentAt(time.Date(2026, 10, 1, 12, i, 0, 0, time.UTC)).Create(t, f.Store) + } + gate := &projectionAdmissionTracker{SerialOperationGate: api.NewSerialOperationGate(), acquired: make(chan struct{}), release: make(chan struct{})} + syncObserved := make(chan int, 1) + sched := scheduler.New(func(ctx context.Context, _ string) error { + var count int + err := f.Store.DB().QueryRowContext(ctx, `SELECT COUNT(*) FROM activity_events`).Scan(&count) + syncObserved <- count + return err + }).WithWorkTracker(gate).WithLogger(testDiscardLogger()) + var releaseOnce sync.Once + release := func() { releaseOnce.Do(func() { close(gate.release) }) } + t.Cleanup(func() { release(); <-sched.Stop().Done() }) + require.NoError(sched.AddAccount("short@example.com", "0 0 1 1 *")) + require.NoError(registerActivityProjectionJob(sched, f.Store, config.ActivityConfig{ + Timezone: "UTC", BatchSize: 1, MaxDirectCounterparts: 25, Schedule: "0 0 1 1 *", + }, testDiscardLogger())) + require.NoError(sched.StartJob(activityProjectionJob)) + select { + case <-gate.acquired: + case <-time.After(10 * time.Second): + require.FailNow("projection did not acquire the gate") + } + require.NoError(sched.TriggerSync("short@example.com")) + require.Eventually(func() bool { return sched.Status()[0].Queued }, 10*time.Second, 10*time.Millisecond) + release() + select { + case count := <-syncObserved: + assert.Equal(activityProjectionMaxBatches, count, "sync runs before the next projection pass") + case <-time.After(10 * time.Second): + require.FailNow("sync was starved behind projection") + } + require.Eventually(func() bool { + status := sched.JobStatus()[0] + return !status.Running && !status.Queued && !status.Pending && !status.LastRun.IsZero() + }, 10*time.Second, 10*time.Millisecond) + var count int + require.NoError(f.Store.DB().QueryRowContext(t.Context(), `SELECT COUNT(*) FROM activity_events`).Scan(&count)) + assert.Equal(23, count, "follow-up passes finish without another cron tick") + assert.Empty(sched.JobStatus()[0].LastError) +} + +type projectionAdmissionTracker struct { + *api.SerialOperationGate + + acquired chan struct{} + release chan struct{} + first sync.Once +} + +func (g *projectionAdmissionTracker) BeginLabeledWorkContext(ctx context.Context, label string) (func(), bool) { + done, ok := g.SerialOperationGate.BeginLabeledWorkContext(ctx, label) + if ok && label == activityProjectionJob { + g.first.Do(func() { close(g.acquired); <-g.release }) + } + return done, ok +} diff --git a/cmd/msgvault/cmd/add_beeper.go b/cmd/msgvault/cmd/add_beeper.go index 007d292a8..50223bc56 100644 --- a/cmd/msgvault/cmd/add_beeper.go +++ b/cmd/msgvault/cmd/add_beeper.go @@ -11,6 +11,7 @@ import ( "github.com/spf13/cobra" "go.kenn.io/msgvault/internal/beeper" "go.kenn.io/msgvault/internal/clirun" + "go.kenn.io/msgvault/internal/config" ) var ( @@ -47,6 +48,11 @@ Examples: MSGVAULT_BEEPER_TOKEN="..." msgvault add-beeper`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg if !isDaemonCLISubprocess() { token, err := readAddBeeperToken(cmd) if err != nil { @@ -56,13 +62,13 @@ Examples: // when the daemon shares this machine so problems fail fast; // with a remote daemon, validation happens daemon-side against // the Beeper Desktop running there. - if IsRemoteMode() { + if IsRemoteMode(state) { _, _ = fmt.Fprintln(cmd.OutOrStdout(), "Remote daemon configured: it must run beside its own Beeper Desktop, which will validate the token.") } else { // A token check, not discovery: the daemon subprocess below // does the real enumeration, so this stays the single // cheapest authenticated request. - accounts, err := beeperClient(token).ListAccounts(cmd.Context()) + accounts, err := beeperClient(cfg, token).ListAccounts(cmd.Context()) if err != nil { return err } @@ -77,7 +83,7 @@ Examples: if token == "" { return errors.New("missing Beeper token in daemon subprocess (set MSGVAULT_BEEPER_TOKEN)") } - accounts, err := beeperClient(token).DiscoverAccounts(cmd.Context()) + accounts, err := beeperClient(cfg, token).DiscoverAccounts(cmd.Context()) if err != nil { return err } @@ -85,7 +91,7 @@ Examples: return fmt.Errorf("save beeper token: %w", err) } - s, cleanup, err := openWritableStoreAndInitForIngest() + s, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) if err != nil { return err } @@ -106,13 +112,13 @@ Examples: } if !noDefaultIdentityAddBeeper { confirmDefaultIdentity(cmd.OutOrStdout(), s, source.ID, - acct.AccountID, beeperSelfIdentity(acct), "account-identifier") + acct.AccountID, beeperSelfIdentity(acct), "account-identifier", state.logger) } _, _ = fmt.Fprintf(cmd.OutOrStdout(), " Added %s (%s)%s\n", acct.AccountID, beeperSourceDisplayName(acct), beeperAccountOrigin(acct)) added++ } - if err := runPostSourceCreateMigrations(s); err != nil { + if err := runPostSourceCreateMigrationsForInvocation(s, state); err != nil { return fmt.Errorf("post-source-create migrations: %w", err) } @@ -129,7 +135,7 @@ Examples: // beeperClient builds a Beeper Desktop API client from the configured URL and // rate limit with a static token. -func beeperClient(token string) *beeper.Client { +func beeperClient(cfg *config.Config, token string) *beeper.Client { // An empty URL selects the client's loopback default. return beeper.NewClient(cfg.Beeper.URL, func(context.Context) (string, error) { return token, nil }, diff --git a/cmd/msgvault/cmd/add_discord.go b/cmd/msgvault/cmd/add_discord.go index 8f20bfafb..97180d014 100644 --- a/cmd/msgvault/cmd/add_discord.go +++ b/cmd/msgvault/cmd/add_discord.go @@ -60,6 +60,9 @@ accepted as a command-line flag. When the bot can access one guild, that guild is selected automatically; otherwise repeat --guild with the desired guild IDs.`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { + if invocationFromContext(cmd.Context()) != nil && deps.bind != nil { + deps = deps.bind(cmd.Context()) + } return runAddDiscord(cmd, deps, opts) }, } diff --git a/cmd/msgvault/cmd/add_oauth_routing_test.go b/cmd/msgvault/cmd/add_oauth_routing_test.go index 6b6f6a2c7..7e2090e2c 100644 --- a/cmd/msgvault/cmd/add_oauth_routing_test.go +++ b/cmd/msgvault/cmd/add_oauth_routing_test.go @@ -22,10 +22,12 @@ func TestAddAccountUsesDaemonRunner(t *testing.T) { "alice@example.com", }, req.Args, "args") }, `{"type":"stdout","data":"Account authorized\n"}`, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx var stdout bytes.Buffer cmd := newAddAccountCmd() + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetArgs([]string{ "alice@example.com", @@ -52,10 +54,12 @@ func TestAddO365UsesDaemonRunner(t *testing.T) { "alice@example.com", }, req.Args, "args") }, `{"type":"stdout","data":"Microsoft 365 account added\n"}`, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx var stdout bytes.Buffer cmd := newAddO365Cmd() + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetArgs([]string{"alice@example.com", "--tenant", "acme", "--no-default-identity"}) @@ -76,10 +80,12 @@ func TestAddTeamsUsesDaemonRunner(t *testing.T) { "alice@example.com", }, req.Args, "args") }, `{"type":"stdout","data":"Microsoft Teams account authorized\n"}`, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx var stdout bytes.Buffer cmd := newAddTeamsCmd() + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetArgs([]string{"alice@example.com", "--tenant", "acme", "--no-default-identity"}) @@ -111,10 +117,12 @@ func TestAddCalendarUsesDaemonRunner(t *testing.T) { "alice@example.com", }, req.Args, "args") }, `{"type":"stdout","data":"Registered 2 calendars\n"}`, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx var stdout bytes.Buffer cmd := newAddCalendarCmd() + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetArgs([]string{ "alice@example.com", @@ -153,10 +161,12 @@ func TestSyncCalendarUsesDaemonRunner(t *testing.T) { "alice@example.com", }, req.Args, "args") }, `{"type":"stdout","data":"Calendar sync complete\n"}`, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx var stdout bytes.Buffer cmd := newSyncCalendarCmd() + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetArgs([]string{ "alice@example.com", @@ -201,10 +211,12 @@ func TestAddCalendarPromptsScopeEscalationBeforeDaemonRunner(t *testing.T) { "alice@example.com", }, req.Args, "args") }, `{"type":"stdout","data":"Registered 1 calendar\n"}`, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx var stdout bytes.Buffer cmd := newAddCalendarCmd() + cmd.SetContext(testCtx) cmd.SetIn(bytes.NewBufferString("y\n")) cmd.SetOut(&stdout) cmd.SetArgs([]string{"alice@example.com"}) diff --git a/cmd/msgvault/cmd/add_slack.go b/cmd/msgvault/cmd/add_slack.go index c83fa83db..a9e609ca2 100644 --- a/cmd/msgvault/cmd/add_slack.go +++ b/cmd/msgvault/cmd/add_slack.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "io" + "log/slog" "os" "strings" @@ -26,18 +27,19 @@ func newAddSlackCmd() *cobra.Command { Short: "Add a Slack workspace as an archive source", Long: `Add a Slack workspace as an archive source. -Archives your own view of the workspace — public/private channels you are a -member of, group DMs, and 1:1 DMs — via the Slack Web API. +Archives conversations allowed by your Slack user token. A token limited to +public-channel scopes archives all public channels, including ones you have +not joined. Broader tokens archive your channel memberships and DMs. Requires a user token (xoxp-...) from an internal Slack app you create: 1. https://api.slack.com/apps > Create New App > From scratch, in your workspace. - 2. OAuth & Permissions > User Token Scopes, add: - channels:history groups:history im:history mpim:history - channels:read groups:read im:read mpim:read - users:read users:read.email files:read reactions:read team:read - search:read + 2. OAuth & Permissions > User Token Scopes, for public channels only: + channels:history channels:read users:read users:read.email + To also archive private channels and DMs you belong to, add: + groups:history groups:read im:history im:read mpim:history mpim:read + Optional: files:read for downloads; search:read for faster reply discovery. 3. Install to Workspace, then copy the "User OAuth Token". Internal apps you create yourself are not subject to Slack's non-Marketplace @@ -46,12 +48,23 @@ rate limits, so backfills run at full speed. Provide the token via --token-file, the MSGVAULT_SLACK_TOKEN environment variable, or the interactive prompt. +Without search:read, each sync revisits channel history to find replies on old +threads. search:read can read private conversations through search; files:read +and reactions:read also grant access beyond public-channel messages. Omit them +for a public-channel-only token. Use a fresh app when reducing permissions; +reinstalling an existing app can retain previously granted scopes. + Examples: msgvault add-slack msgvault add-slack --token-file ~/slack-token.txt MSGVAULT_SLACK_TOKEN="xoxp-..." msgvault add-slack`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg if !isDaemonCLISubprocess() { token, err := readAddSlackToken(cmd) if err != nil { @@ -71,17 +84,12 @@ Examples: if err != nil { return err } - // The reply sweep needs search:read; catch an under-scoped token - // here, where the fix is cheap, not on every future sync. - if err := client.ValidateSearchScope(cmd.Context()); err != nil { - return err - } teamDomain := strings.TrimSuffix(strings.TrimPrefix(auth.URL, "https://"), ".slack.com/") if err := slack.SaveToken(cfg.TokensDir(), auth.TeamID, teamDomain, auth.UserID, token); err != nil { return fmt.Errorf("save slack token: %w", err) } - s, cleanup, err := openWritableStoreAndInitForIngest() + s, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) if err != nil { return err } @@ -97,9 +105,9 @@ Examples: return fmt.Errorf("set display name for %s: %w", identifier, err) } if !noDefaultIdentityAddSlack { - confirmDefaultSlackIdentity(cmd.OutOrStdout(), s, source.ID, auth.TeamID, auth.UserID) + confirmDefaultSlackIdentity(cmd.OutOrStdout(), s, source.ID, auth.TeamID, auth.UserID, state.logger) } - if err := runPostSourceCreateMigrations(s); err != nil { + if err := runPostSourceCreateMigrationsForInvocation(s, state); err != nil { return fmt.Errorf("post-source-create migrations: %w", err) } @@ -119,9 +127,9 @@ func writeAddedSlackWorkspace(out io.Writer, team, teamID, identifier string) { textutil.SanitizeTerminal(team), teamID, identifier) } -func confirmDefaultSlackIdentity(out io.Writer, s *store.Store, sourceID int64, teamID, userID string) { +func confirmDefaultSlackIdentity(out io.Writer, s *store.Store, sourceID int64, teamID, userID string, logger *slog.Logger) { account := teamID + ":" + userID - confirmDefaultIdentity(out, s, sourceID, account, account, "account-identifier") + confirmDefaultIdentity(out, s, sourceID, account, account, "account-identifier", logger) } // readAddSlackToken resolves the user token: env var, then --token-file, diff --git a/cmd/msgvault/cmd/add_synctech_sms_drive.go b/cmd/msgvault/cmd/add_synctech_sms_drive.go index 690385d7b..2d489377d 100644 --- a/cmd/msgvault/cmd/add_synctech_sms_drive.go +++ b/cmd/msgvault/cmd/add_synctech_sms_drive.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "log/slog" "os" "path/filepath" "time" @@ -33,6 +34,11 @@ func newAddSynctechSMSDriveCmd() *cobra.Command { Short: "Configure a Google Drive SMS Backup & Restore source", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg if opts.OwnerPhone == "" { return errors.New("--owner-phone is required") } @@ -46,7 +52,7 @@ func newAddSynctechSMSDriveCmd() *cobra.Command { // Complete OAuth in this process — which owns the user's // browser — before proxying; the daemon subprocess's // idempotent token check then skips the browser flow. - if !opts.SkipAuthForTest && !IsRemoteMode() { + if !opts.SkipAuthForTest && !IsRemoteMode(state) { if err := ensureSynctechSMSDriveToken(cmd.Context(), opts.GoogleAccount, opts.OAuthApp); err != nil { return err } @@ -105,6 +111,11 @@ func newSyncSynctechSMSCmd() *cobra.Command { Short: "Run one configured synctech-sms source now", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg if !isDaemonCLISubprocess() { return runDaemonCLICommandHTTPFromCobra(cmd, args) } @@ -118,7 +129,11 @@ func newSyncSynctechSMSCmd() *cobra.Command { } func runConfiguredSynctechSMSSource(ctx context.Context, src config.SynctechSMSSource) error { - st, cleanup, err := openWritableStoreAndInitForIngest() + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + st, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) if err != nil { return err } @@ -132,7 +147,7 @@ func runConfiguredSynctechSMSSourceWithStore(ctx context.Context, st *store.Stor } func runConfiguredSynctechSMSSourceWithStoreDriveClient(ctx context.Context, st *store.Store, src config.SynctechSMSSource, driveClient synctechsms.DriveClient) error { - opts := synctechImportOptions(src) + opts := synctechImportOptions(src, invocationFromContext(ctx).cfg) if opts.OwnerPhone == "" { return fmt.Errorf("synctech-sms source %q owner_phone is required", src.Name) } @@ -142,7 +157,7 @@ func runConfiguredSynctechSMSSourceWithStoreDriveClient(ctx context.Context, st if src.Path == "" { return fmt.Errorf("synctech-sms source %q path is required for local backend", src.Name) } - if _, err := ensureConfiguredSynctechSMSSource(st, src, opts); err != nil { + if _, err := ensureConfiguredSynctechSMSSource(st, src, opts, invocationFromContext(ctx)); err != nil { return err } _, err = synctechsms.NewImporter(st, opts).ImportPath(src.Path) @@ -155,12 +170,15 @@ func runConfiguredSynctechSMSSourceWithStoreDriveClient(ctx context.Context, st default: return fmt.Errorf("unsupported synctech-sms backend %q", src.Backend) } - refreshErr := rebuildCacheAfterScheduledSync( - context.WithoutCancel(ctx), "synctech-sms:"+src.Name) + var refreshErr error + if !isDaemonCLISubprocess() { + refreshErr = rebuildCacheAfterScheduledSync( + context.WithoutCancel(ctx), "synctech-sms:"+src.Name) + } return errors.Join(err, refreshErr) } -func ensureConfiguredSynctechSMSSource(st *store.Store, src config.SynctechSMSSource, opts synctechsms.ImportOptions) (*store.Source, error) { +func ensureConfiguredSynctechSMSSource(st *store.Store, src config.SynctechSMSSource, opts synctechsms.ImportOptions, state *invocation) (*store.Source, error) { if opts.OwnerPhone == "" { return nil, fmt.Errorf("synctech-sms source %q owner_phone is required", src.Name) } @@ -168,8 +186,11 @@ func ensureConfiguredSynctechSMSSource(st *store.Store, src config.SynctechSMSSo if err != nil { return nil, fmt.Errorf("get source: %w", err) } - confirmDefaultIdentity(io.Discard, st, source.ID, src.Name, opts.OwnerPhone, "account-identifier") - if err := runPostSourceCreateMigrations(st); err != nil { + if state == nil || state.logger == nil { + return nil, errors.New("invocation state is unavailable") + } + confirmDefaultIdentity(io.Discard, st, source.ID, src.Name, opts.OwnerPhone, "account-identifier", state.logger) + if err := runPostSourceCreateMigrationsForInvocation(st, state); err != nil { return nil, fmt.Errorf("post-source-create migrations: %w", err) } return source, nil @@ -203,7 +224,13 @@ func runSynctechSMSDriveSourceWithClient(ctx context.Context, st *store.Store, s if err := validateSynctechSMSDriveSource(src); err != nil { return summary, err } - source, err := ensureConfiguredSynctechSMSSource(st, src, opts) + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil { + return summary, errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger + source, err := ensureConfiguredSynctechSMSSource(st, src, opts, state) if err != nil { return summary, err } @@ -322,11 +349,16 @@ func importOneDriveBackup(ctx context.Context, st *store.Store, imp *synctechsms } func newSynctechSMSDriveClient(ctx context.Context, src config.SynctechSMSSource) (synctechsms.DriveClient, error) { + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil { + return nil, errors.New("configuration is unavailable") + } + cfg := state.cfg clientSecrets, err := cfg.OAuth.ClientSecretsFor(src.OAuthApp) if err != nil { return nil, err } - mgr, err := newSynctechSMSDriveOAuthManager(clientSecrets) + mgr, err := newSynctechSMSDriveOAuthManager(cfg, state.logger, clientSecrets) if err != nil { return nil, err } @@ -345,11 +377,16 @@ func newSynctechSMSDriveClient(ctx context.Context, src config.SynctechSMSSource } func ensureSynctechSMSDriveToken(ctx context.Context, googleAccount, oauthApp string) error { + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg clientSecrets, err := cfg.OAuth.ClientSecretsFor(oauthApp) if err != nil { return err } - mgr, err := newSynctechSMSDriveOAuthManager(clientSecrets) + mgr, err := newSynctechSMSDriveOAuthManager(cfg, state.logger, clientSecrets) if err != nil { return err } @@ -359,7 +396,7 @@ func ensureSynctechSMSDriveToken(ctx context.Context, googleAccount, oauthApp st return mgr.Authorize(ctx, googleAccount) } -func newSynctechSMSDriveOAuthManager(clientSecrets string) (*oauth.Manager, error) { +func newSynctechSMSDriveOAuthManager(cfg *config.Config, logger *slog.Logger, clientSecrets string) (*oauth.Manager, error) { // The current OAuth manager validates account identity through Gmail's // profile endpoint, so request a read-only Gmail scope alongside Drive. return oauth.NewManagerWithScopes(clientSecrets, cfg.TokensDir(), logger, []string{ @@ -368,10 +405,14 @@ func newSynctechSMSDriveOAuthManager(clientSecrets string) (*oauth.Manager, erro }) } -func synctechImportOptions(src config.SynctechSMSSource) synctechsms.ImportOptions { +func synctechImportOptions(src config.SynctechSMSSource, cfg *config.Config) synctechsms.ImportOptions { + attachmentsDir := "" + if cfg != nil { + attachmentsDir = cfg.AttachmentsDir() + } return synctechsms.ImportOptions{ OwnerPhone: src.OwnerPhone, - AttachmentsDir: cfg.AttachmentsDir(), + AttachmentsDir: attachmentsDir, IncludeSMS: src.IncludeSMS, IncludeMMS: src.IncludeMMS, IncludeCalls: src.IncludeCalls, @@ -381,5 +422,5 @@ func synctechImportOptions(src config.SynctechSMSSource) synctechsms.ImportOptio func init() { rootCmd.AddCommand(newAddSynctechSMSDriveCmd()) - rootCmd.AddCommand(newSyncSynctechSMSCmd()) + rootCmd.AddCommand(addManualSyncCacheFlags(newSyncSynctechSMSCmd())) } diff --git a/cmd/msgvault/cmd/add_synctech_sms_drive_test.go b/cmd/msgvault/cmd/add_synctech_sms_drive_test.go index 04b08d4dc..4c471c92a 100644 --- a/cmd/msgvault/cmd/add_synctech_sms_drive_test.go +++ b/cmd/msgvault/cmd/add_synctech_sms_drive_test.go @@ -24,10 +24,12 @@ func TestAddSynctechSMSDriveWritesConfigWithoutSecrets(t *testing.T) { require := require.New(t) assert := assert.New(t) home := t.TempDir() - cfg = config.NewDefaultConfig() + cfg := config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) cfg.HomeDir = home cfg.Data.DataDir = home cmd := newTestRootCmd() + cmd.SetContext(testCtx) cmd.AddCommand(newAddSynctechSMSDriveCmd()) cmd.SetArgs([]string{ "add-synctech-sms-drive", "pixel", @@ -56,7 +58,8 @@ func TestSynctechSMSDriveRunUsesSingleOuterSyncRun(t *testing.T) { require := require.New(t) assert := assert.New(t) home := t.TempDir() - cfg = config.NewDefaultConfig() + cfg := config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) cfg.HomeDir = home cfg.Data.DataDir = home f := storetest.New(t) @@ -77,7 +80,7 @@ func TestSynctechSMSDriveRunUsesSingleOuterSyncRun(t *testing.T) { }, } - summary, err := runSynctechSMSDriveSourceWithClient(context.Background(), f.Store, src, synctechImportOptions(src), client) + summary, err := runSynctechSMSDriveSourceWithClient(testCtx, f.Store, src, synctechImportOptions(src, invocationFromContext(testCtx).cfg), client) require.NoError(err, "runSynctechSMSDriveSourceWithClient") require.Len(summary.MessageIDs, 1, "summary message IDs") @@ -101,11 +104,8 @@ func TestSynctechSMSDriveRunSetsUpIdentityAndPostSourceMigration(t *testing.T) { require := require.New(t) assert := assert.New(t) home := t.TempDir() - savedCfg := cfg - t.Cleanup(func() { - cfg = savedCfg - }) - cfg = config.NewDefaultConfig() + cfg := config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) cfg.HomeDir = home cfg.Data.DataDir = home cfg.Identity.Addresses = []string{"legacy@example.com"} @@ -116,7 +116,7 @@ func TestSynctechSMSDriveRunSetsUpIdentityAndPostSourceMigration(t *testing.T) { src := synctechDriveTestSource() client := fakeSynctechDriveClient{} - _, err = runSynctechSMSDriveSourceWithClient(context.Background(), st, src, synctechImportOptions(src), client) + _, err = runSynctechSMSDriveSourceWithClient(testCtx, st, src, synctechImportOptions(src, invocationFromContext(testCtx).cfg), client) require.NoError(err, "runSynctechSMSDriveSourceWithClient") synctechSource := getSynctechSource(t, st, src.OwnerPhone) @@ -141,7 +141,8 @@ func TestSynctechSMSDriveRunRecordsZeroSelectedPoll(t *testing.T) { require := require.New(t) assert := assert.New(t) home := t.TempDir() - cfg = config.NewDefaultConfig() + cfg := config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) cfg.HomeDir = home cfg.Data.DataDir = home f := storetest.New(t) @@ -157,7 +158,7 @@ func TestSynctechSMSDriveRunRecordsZeroSelectedPoll(t *testing.T) { }}, } - _, err := runSynctechSMSDriveSourceWithClient(context.Background(), f.Store, src, synctechImportOptions(src), client) + _, err := runSynctechSMSDriveSourceWithClient(testCtx, f.Store, src, synctechImportOptions(src, invocationFromContext(testCtx).cfg), client) require.NoError(err, "runSynctechSMSDriveSourceWithClient") source := getSynctechSource(t, f.Store, src.OwnerPhone) @@ -174,7 +175,9 @@ func TestSynctechSMSDriveRunMarksOuterSyncFailedOnDownloadError(t *testing.T) { require := require.New(t) assert := assert.New(t) home := t.TempDir() - cfg = config.NewDefaultConfig() + cfg := config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.HomeDir = home cfg.Data.DataDir = home f := storetest.New(t) @@ -191,7 +194,7 @@ func TestSynctechSMSDriveRunMarksOuterSyncFailedOnDownloadError(t *testing.T) { downloadErr: downloadErr, } - _, err := runSynctechSMSDriveSourceWithClient(context.Background(), f.Store, src, synctechImportOptions(src), client) + _, err := runSynctechSMSDriveSourceWithClient(testCtx, f.Store, src, synctechImportOptions(src, invocationFromContext(testCtx).cfg), client) require.ErrorIs(err, downloadErr, "runSynctechSMSDriveSourceWithClient") source := getSynctechSource(t, f.Store, src.OwnerPhone) @@ -208,12 +211,16 @@ func TestSynctechSMSDriveRunMarksOuterSyncFailedOnDownloadError(t *testing.T) { } func TestSynctechSMSDrivePartialFailureEnqueuesImportedMessages(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) home := t.TempDir() savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.HomeDir = home cfg.Data.DataDir = home st := testutil.NewSQLiteTestStore(t) @@ -259,7 +266,7 @@ func TestSynctechSMSDrivePartialFailureEnqueuesImportedMessages(t *testing.T) { ", Args: cobra.ExactArgs(1), @@ -130,6 +135,9 @@ func TestAddAccount_InheritedBindingValidatesToken(t *testing.T) { } func TestAddAccount_CalendarOnlyTokenRequiresGmailReauth(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + require := require.New(t) tmpDir := t.TempDir() @@ -156,9 +164,11 @@ func TestAddAccount_CalendarOnlyTokenRequiresGmailReauth(t *testing.T) { Data: config.DataConfig{DataDir: tmpDir}, OAuth: config.OAuthConfig{ClientSecrets: secretsPath}, } + testCtx := testInvocationContext(gmailProfileContext(t, "user@example.com"), cfg, invocationOptions{}) + _ = testCtx logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(testCtx) cancel() testCmd := &cobra.Command{ @@ -172,6 +182,7 @@ func TestAddAccount_CalendarOnlyTokenRequiresGmailReauth(t *testing.T) { testCmd.Flags().BoolVar(&noDefaultIdentityAddAccount, "no-default-identity", false, "") root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(testCmd) root.SetArgs([]string{"add-account", "user@example.com"}) @@ -188,6 +199,9 @@ func TestAddAccount_CalendarOnlyTokenRequiresGmailReauth(t *testing.T) { } func TestAddAccount_FullGmailScopeTokenCanBeReused(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + require := require.New(t) tmpDir := t.TempDir() dbPath := filepath.Join(tmpDir, "msgvault.db") @@ -225,9 +239,12 @@ func TestAddAccount_FullGmailScopeTokenCanBeReused(t *testing.T) { Data: config.DataConfig{DataDir: tmpDir}, OAuth: config.OAuthConfig{ClientSecrets: secretsPath}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) ctx := gmailProfileContext(t, "user@example.com") + ctx = testInvocationContext(ctx, cfg, invocationOptions{}) testCmd := &cobra.Command{ Use: "add-account ", Args: cobra.ExactArgs(1), @@ -240,6 +257,7 @@ func TestAddAccount_FullGmailScopeTokenCanBeReused(t *testing.T) { testCmd.Flags().BoolVar(&noDefaultIdentityAddAccount, "no-default-identity", false, "") root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(testCmd) root.SetArgs([]string{"add-account", "user@example.com", "--no-default-identity"}) @@ -268,6 +286,9 @@ func TestAddAccountOAuthScopesForTokenPreservesExistingCalendarGrant(t *testing. // OAuth app binding with an existing token updates the binding // without re-authorizing (headless rebind scenario). func TestAddAccount_RebindWithExistingToken(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + require := require.New(t) assert := assert.New(t) tmpDir := t.TempDir() @@ -277,7 +298,7 @@ func TestAddAccount_RebindWithExistingToken(t *testing.T) { s, err := store.Open(dbPath) require.NoError(err, "open store") require.NoError(s.InitSchema(), "init schema") - source, err := s.GetOrCreateSource("gmail", "user@acme.com") + source, err := s.GetOrCreateSource("gmail", "user-a@example.com") require.NoError(err, "create source") require.NoError(s.UpdateSourceOAuthApp(source.ID, sql.NullString{ String: "old-app", Valid: true, @@ -296,7 +317,7 @@ func TestAddAccount_RebindWithExistingToken(t *testing.T) { "client_id": "test.apps.googleusercontent.com", }) require.NoError(err, "marshal token") - tokenPath := filepath.Join(tokensDir, "user@acme.com.json") + tokenPath := filepath.Join(tokensDir, "user-a@example.com.json") require.NoError(os.WriteFile(tokenPath, tokenData, 0600), "write token") // Write fake client secrets @@ -324,6 +345,8 @@ func TestAddAccount_RebindWithExistingToken(t *testing.T) { }, }, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) testCmd := &cobra.Command{ @@ -338,13 +361,14 @@ func TestAddAccount_RebindWithExistingToken(t *testing.T) { testCmd.Flags().BoolVar(&noDefaultIdentityAddAccount, "no-default-identity", false, "") root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(testCmd) root.SetArgs([]string{ - "add-account", "user@acme.com", "--oauth-app", "new-app", + "add-account", "user-a@example.com", "--oauth-app", "new-app", }) // Should succeed without opening a browser — token exists - err = root.ExecuteContext(gmailProfileContext(t, "user@acme.com")) + err = root.ExecuteContext(testInvocationContext(gmailProfileContext(t, "user-a@example.com"), cfg, invocationOptions{})) require.NoError(err) // Token file should still exist @@ -356,7 +380,7 @@ func TestAddAccount_RebindWithExistingToken(t *testing.T) { require.NoError(err, "reopen store") defer func() { _ = s2.Close() }() - src, err := findGmailSource(s2, "user@acme.com") + src, err := findGmailSource(s2, "user-a@example.com") require.NoError(err, "find source") require.NotNil(src, "source not found after rebind") assert.True(src.OAuthApp.Valid && src.OAuthApp.String == "new-app", @@ -369,6 +393,9 @@ func TestAddAccount_RebindWithExistingToken(t *testing.T) { // add-account --oauth-app with no existing source row rejects a token // minted by a different OAuth client (forces re-auth, not silent accept). func TestAddAccount_NewRegistrationRejectsMismatchedToken(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + require := require.New(t) tmpDir := t.TempDir() @@ -410,10 +437,12 @@ func TestAddAccount_NewRegistrationRejectsMismatchedToken(t *testing.T) { }, }, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) // Pre-cancel so if it falls through to Authorize, it fails fast - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(testCtx) cancel() testCmd := &cobra.Command{ @@ -428,6 +457,7 @@ func TestAddAccount_NewRegistrationRejectsMismatchedToken(t *testing.T) { testCmd.Flags().BoolVar(&noDefaultIdentityAddAccount, "no-default-identity", false, "") root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(testCmd) root.SetArgs([]string{ "add-account", "new@acme.com", "--oauth-app", "acme", @@ -441,6 +471,9 @@ func TestAddAccount_NewRegistrationRejectsMismatchedToken(t *testing.T) { // TestAddAccount_ExplicitDefaultRejectsMismatchedToken verifies that // --oauth-app "" rejects a token minted by a different client. func TestAddAccount_ExplicitDefaultRejectsMismatchedToken(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + require := require.New(t) tmpDir := t.TempDir() @@ -478,9 +511,11 @@ func TestAddAccount_ExplicitDefaultRejectsMismatchedToken(t *testing.T) { Data: config.DataConfig{DataDir: tmpDir}, OAuth: config.OAuthConfig{ClientSecrets: secretsPath}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(testCtx) cancel() testCmd := &cobra.Command{ @@ -495,6 +530,7 @@ func TestAddAccount_ExplicitDefaultRejectsMismatchedToken(t *testing.T) { testCmd.Flags().BoolVar(&noDefaultIdentityAddAccount, "no-default-identity", false, "") root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(testCmd) root.SetArgs([]string{ "add-account", "user@example.com", "--oauth-app", "", @@ -507,6 +543,9 @@ func TestAddAccount_ExplicitDefaultRejectsMismatchedToken(t *testing.T) { // TestAddAccount_ExplicitDefaultAcceptsMatchingToken verifies that // --oauth-app "" accepts a token minted by the default client. func TestAddAccount_ExplicitDefaultAcceptsMatchingToken(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + require := require.New(t) tmpDir := t.TempDir() @@ -544,6 +583,8 @@ func TestAddAccount_ExplicitDefaultAcceptsMatchingToken(t *testing.T) { Data: config.DataConfig{DataDir: tmpDir}, OAuth: config.OAuthConfig{ClientSecrets: secretsPath}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) testCmd := &cobra.Command{ @@ -559,8 +600,10 @@ func TestAddAccount_ExplicitDefaultAcceptsMatchingToken(t *testing.T) { // Verify the cached token against a synthetic Gmail profile. ctx := gmailProfileContext(t, "user@example.com") + ctx = testInvocationContext(ctx, cfg, invocationOptions{}) root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(testCmd) root.SetArgs([]string{ "add-account", "user@example.com", "--oauth-app", "", @@ -572,6 +615,9 @@ func TestAddAccount_ExplicitDefaultAcceptsMatchingToken(t *testing.T) { } func TestAddAccount_ForceRebindPreservesBindingOnFailure(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + require := require.New(t) tmpDir := t.TempDir() dbPath := filepath.Join(tmpDir, "msgvault.db") @@ -612,10 +658,12 @@ func TestAddAccount_ForceRebindPreservesBindingOnFailure(t *testing.T) { }, }, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) // Pre-cancel context so Authorize fails immediately - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(testCtx) cancel() testCmd := &cobra.Command{ @@ -630,6 +678,7 @@ func TestAddAccount_ForceRebindPreservesBindingOnFailure(t *testing.T) { testCmd.Flags().BoolVar(&noDefaultIdentityAddAccount, "no-default-identity", false, "") root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(testCmd) root.SetArgs([]string{ "add-account", "user@acme.com", @@ -654,6 +703,9 @@ func TestAddAccount_ForceRebindPreservesBindingOnFailure(t *testing.T) { // TestAddAccount_HeadlessExplicitEmptyOAuthApp verifies that // --headless --oauth-app "" does not re-inherit the stored binding. func TestAddAccount_HeadlessExplicitEmptyOAuthApp(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + require := require.New(t) tmpDir := t.TempDir() dbPath := filepath.Join(tmpDir, "msgvault.db") @@ -687,6 +739,8 @@ func TestAddAccount_HeadlessExplicitEmptyOAuthApp(t *testing.T) { HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) // The RunE reads package-level flag vars, but uses @@ -705,6 +759,7 @@ func TestAddAccount_HeadlessExplicitEmptyOAuthApp(t *testing.T) { testCmd.Flags().BoolVar(&noDefaultIdentityAddAccount, "no-default-identity", false, "") root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(testCmd) root.SetArgs([]string{ "add-account", "user@acme.com", @@ -726,6 +781,9 @@ func TestAddAccount_HeadlessExplicitEmptyOAuthApp(t *testing.T) { // TestAddAccount_AutoDefaultIdentityFires verifies that running add-account // with a reusable token writes an account-identifier identity row. func TestAddAccount_AutoDefaultIdentityFires(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + require := require.New(t) assert := assert.New(t) tmpDir := t.TempDir() @@ -761,6 +819,8 @@ func TestAddAccount_AutoDefaultIdentityFires(t *testing.T) { Data: config.DataConfig{DataDir: tmpDir}, OAuth: config.OAuthConfig{ClientSecrets: secretsPath}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) testCmd := &cobra.Command{ @@ -775,10 +835,14 @@ func TestAddAccount_AutoDefaultIdentityFires(t *testing.T) { testCmd.Flags().BoolVar(&noDefaultIdentityAddAccount, "no-default-identity", false, "") root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(testCmd) root.SetArgs([]string{"add-account", "user@example.com"}) - require.NoError(root.ExecuteContext(gmailProfileContext(t, "user@example.com"))) + execCtx := testInvocationContext(gmailProfileContext(t, "user@example.com"), cfg, invocationOptions{}) + invocationFromContext(execCtx).logger = logger + root.SetContext(execCtx) + require.NoError(root.ExecuteContext(execCtx)) s, err := store.Open(dbPath) require.NoError(err, "reopen store") @@ -798,7 +862,10 @@ func TestAddAccount_AutoDefaultIdentityFires(t *testing.T) { // TestAddAccount_NoDefaultIdentitySuppresses verifies that --no-default-identity // prevents the auto-identity write. func TestAddAccount_NoDefaultIdentitySuppresses(t *testing.T) { - require := require.New(t) + cfg := testConfigValue() + logger := testLoggerValue() + + assert, require := assert.New(t), require.New(t) tmpDir := t.TempDir() dbPath := filepath.Join(tmpDir, "msgvault.db") @@ -832,6 +899,8 @@ func TestAddAccount_NoDefaultIdentitySuppresses(t *testing.T) { Data: config.DataConfig{DataDir: tmpDir}, OAuth: config.OAuthConfig{ClientSecrets: secretsPath}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) testCmd := &cobra.Command{ @@ -846,10 +915,11 @@ func TestAddAccount_NoDefaultIdentitySuppresses(t *testing.T) { testCmd.Flags().BoolVar(&noDefaultIdentityAddAccount, "no-default-identity", false, "") root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(testCmd) root.SetArgs([]string{"add-account", "user@example.com", "--no-default-identity"}) - require.NoError(root.ExecuteContext(gmailProfileContext(t, "user@example.com"))) + require.NoError(root.ExecuteContext(testInvocationContext(gmailProfileContext(t, "user@example.com"), cfg, invocationOptions{}))) s, err := store.Open(dbPath) require.NoError(err, "reopen store") @@ -861,7 +931,29 @@ func TestAddAccount_NoDefaultIdentitySuppresses(t *testing.T) { ids, err := s.ListAccountIdentities(src.ID) require.NoError(err, "ListAccountIdentities") - assert.Empty(t, ids, "expected 0 identity rows with --no-default-identity") + assert.Empty(ids, "expected 0 identity rows with --no-default-identity") + + // The daemon's shared helper must honor the choice after the store reopens. + confirmDefaultIdentity(io.Discard, s, src.ID, "user@example.com", "user@example.com", "account-identifier", logger) + ids, err = s.ListAccountIdentities(src.ID) + require.NoError(err) + assert.Empty(ids, "the shared identity helper must preserve the opt-out") + + // Use a new command, as a separate re-authorization invocation would. + rerun := &cobra.Command{Use: addAccountUse, RunE: runAddAccountLocal} + registerAddAccountFlags(rerun) + rerun.SetArgs([]string{"user@example.com"}) + require.NoError(rerun.ExecuteContext(testInvocationContext(gmailProfileContext(t, "user@example.com"), cfg, invocationOptions{}))) + ids, err = s.ListAccountIdentities(src.ID) + require.NoError(err) + assert.Empty(ids, "omitting the flag must preserve the saved choice") + + root.SetArgs([]string{"add-account", "user@example.com", "--no-default-identity=false"}) + require.NoError(root.ExecuteContext(testInvocationContext(gmailProfileContext(t, "user@example.com"), cfg, invocationOptions{}))) + ids, err = s.ListAccountIdentities(src.ID) + require.NoError(err) + require.Len(ids, 1, "explicit false restores the default") + assert.Equal("user@example.com", ids[0].Address) } // TestAddAccount_DeferredLegacyIdentityMigrationFires verifies that legacy @@ -872,6 +964,9 @@ func TestAddAccount_NoDefaultIdentitySuppresses(t *testing.T) { // applied on the *next* command — leaving the new source without its // configured identities until then. func TestAddAccount_DeferredLegacyIdentityMigrationFires(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + require := require.New(t) assert := assert.New(t) tmpDir := t.TempDir() @@ -910,8 +1005,10 @@ func TestAddAccount_DeferredLegacyIdentityMigrationFires(t *testing.T) { Addresses: []string{"alias@example.com", "alt@work.com"}, }, } + testCtx := testInvocationContext(gmailProfileContext(t, "user@example.com"), cfg, invocationOptions{}) var logBuf strings.Builder logger = slog.New(slog.NewTextHandler(&logBuf, nil)) + invocationFromContext(testCtx).logger = logger testCmd := &cobra.Command{ Use: "add-account ", @@ -923,14 +1020,16 @@ func TestAddAccount_DeferredLegacyIdentityMigrationFires(t *testing.T) { testCmd.Flags().BoolVar(&forceReauth, "force", false, "") testCmd.Flags().StringVar(&accountDisplayName, "display-name", "", "") testCmd.Flags().BoolVar(&noDefaultIdentityAddAccount, "no-default-identity", false, "") + testCmd.SetContext(testCtx) root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(testCmd) // --no-default-identity isolates the test to the legacy migration path: // the auto-default would otherwise add a third identity row. root.SetArgs([]string{"add-account", "user@example.com", "--no-default-identity"}) - require.NoError(root.ExecuteContext(gmailProfileContext(t, "user@example.com"))) + require.NoError(root.ExecuteContext(testCtx)) // The user-facing notice must only describe the applied path. // Emitting the "deferred — will run on the next command" notice @@ -976,6 +1075,9 @@ func TestAddAccount_DeferredLegacyIdentityMigrationFires(t *testing.T) { // account-identifier write entirely — leaving the source without its // own identifier and breaking dedup sent-copy detection. func TestAddAccount_LegacyMigrationDoesNotSuppressDefaultIdentity(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + require := require.New(t) tmpDir := t.TempDir() dbPath := filepath.Join(tmpDir, "msgvault.db") @@ -1016,6 +1118,8 @@ func TestAddAccount_LegacyMigrationDoesNotSuppressDefaultIdentity(t *testing.T) Addresses: []string{"alias@example.com", "alt@work.com"}, }, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) testCmd := &cobra.Command{ @@ -1030,12 +1134,13 @@ func TestAddAccount_LegacyMigrationDoesNotSuppressDefaultIdentity(t *testing.T) testCmd.Flags().BoolVar(&noDefaultIdentityAddAccount, "no-default-identity", false, "") root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(testCmd) // Note: NOT passing --no-default-identity. The bug only manifests // when the auto-default write is supposed to fire. root.SetArgs([]string{"add-account", "user@example.com"}) - require.NoError(root.ExecuteContext(gmailProfileContext(t, "user@example.com"))) + require.NoError(root.ExecuteContext(testInvocationContext(gmailProfileContext(t, "user@example.com"), cfg, invocationOptions{}))) s, err := store.Open(dbPath) require.NoError(err, "reopen store") @@ -1059,6 +1164,9 @@ func TestAddAccount_LegacyMigrationDoesNotSuppressDefaultIdentity(t *testing.T) } func TestAddAccount_HeadlessServiceAccountReturnsActionableError(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + tmpDir := t.TempDir() savedCfg := cfg @@ -1083,6 +1191,8 @@ func TestAddAccount_HeadlessServiceAccountReturnsActionableError(t *testing.T) { }, }, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) testCmd := &cobra.Command{ @@ -1096,6 +1206,7 @@ func TestAddAccount_HeadlessServiceAccountReturnsActionableError(t *testing.T) { testCmd.Flags().StringVar(&accountDisplayName, "display-name", "", "") root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(testCmd) root.SetArgs([]string{ "add-account", "user@company.com", @@ -1113,6 +1224,9 @@ func TestAddAccount_HeadlessServiceAccountReturnsActionableError(t *testing.T) { } func TestAddAccount_ForceServiceAccountReturnsActionableError(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + tmpDir := t.TempDir() savedCfg := cfg @@ -1137,6 +1251,8 @@ func TestAddAccount_ForceServiceAccountReturnsActionableError(t *testing.T) { }, }, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) testCmd := &cobra.Command{ @@ -1150,6 +1266,7 @@ func TestAddAccount_ForceServiceAccountReturnsActionableError(t *testing.T) { testCmd.Flags().StringVar(&accountDisplayName, "display-name", "", "") root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(testCmd) root.SetArgs([]string{ "add-account", "user@company.com", diff --git a/cmd/msgvault/cmd/addimap.go b/cmd/msgvault/cmd/addimap.go index c05ce7ac2..72dafb5cf 100644 --- a/cmd/msgvault/cmd/addimap.go +++ b/cmd/msgvault/cmd/addimap.go @@ -83,6 +83,12 @@ Examples: msgvault add-imap --host mail.example.com --username user@example.com --starttls msgvault add-imap --host mail.example.com --username user@example.com --no-tls`, RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger if imapHost == "" { return usageErr(cmd, errors.New("--host is required")) } @@ -126,7 +132,7 @@ Examples: } fmt.Printf("Connected successfully as %s\n", profile.EmailAddress) - s, cleanup, err := openWritableStoreAndInitForIngest() + s, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) if err != nil { return err } @@ -161,10 +167,13 @@ Examples: // Auto-default-identity must run BEFORE the legacy migration // retry — see comment in account_identity.go. + if err := setDefaultIdentityOptOut(cmd, s, source, noDefaultIdentityAddImap); err != nil { + return err + } if !noDefaultIdentityAddImap { - confirmDefaultIdentity(cmd.OutOrStdout(), s, source.ID, imapUsername, imapUsername, "account-identifier") + confirmDefaultIdentity(cmd.OutOrStdout(), s, source.ID, imapUsername, imapUsername, "account-identifier", state.logger) } - if err := runPostSourceCreateMigrations(s); err != nil { + if err := runPostSourceCreateMigrationsForInvocation(s, state); err != nil { return fmt.Errorf("post-source-create migrations: %w", err) } @@ -183,7 +192,7 @@ Examples: cmd.Flags().StringVar(&imapUsername, "username", "", "IMAP username / email address (required)") cmd.Flags().BoolVar(&imapNoTLS, "no-tls", false, "Disable TLS (plain connection, not recommended)") cmd.Flags().BoolVar(&imapSTARTTLS, "starttls", false, "Use STARTTLS instead of implicit TLS") - cmd.Flags().BoolVar(&noDefaultIdentityAddImap, "no-default-identity", false, noDefaultIdentityHelp) + cmd.Flags().BoolVar(&noDefaultIdentityAddImap, "no-default-identity", false, savedDefaultIdentityHelp) return cmd } diff --git a/cmd/msgvault/cmd/addimap_test.go b/cmd/msgvault/cmd/addimap_test.go index b6b92b917..2649307db 100644 --- a/cmd/msgvault/cmd/addimap_test.go +++ b/cmd/msgvault/cmd/addimap_test.go @@ -213,12 +213,14 @@ func TestAddIMAPUsesDaemonRunnerAndForwardsPasswordEnv(t *testing.T) { imapSTARTTLS = savedStartTLS noDefaultIdentityAddImap = savedNoDefaultIdentity }) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx t.Setenv("MSGVAULT_IMAP_PASSWORD", "secret") var stdout bytes.Buffer var stderr bytes.Buffer cmd := newAddIMAPCmd() + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetErr(&stderr) cmd.SetArgs([]string{ diff --git a/cmd/msgvault/cmd/addo365.go b/cmd/msgvault/cmd/addo365.go index d98417f85..22b790bf7 100644 --- a/cmd/msgvault/cmd/addo365.go +++ b/cmd/msgvault/cmd/addo365.go @@ -1,6 +1,7 @@ package cmd import ( + "errors" "fmt" "strings" @@ -11,8 +12,10 @@ import ( ) var ( + o365Headless bool o365TenantID string noDefaultIdentityAddO365 bool + o365Graph bool ) func newAddO365Cmd() *cobra.Command { @@ -33,23 +36,20 @@ func newAddO365Cmd() *cobra.Command { // before proxying, so the daemon subprocess never opens a browser or waits // on human consent while holding the operation gate. func preflightAddO365Authorize(cmd *cobra.Command, email string) error { - if IsRemoteMode() { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + if IsRemoteMode(state) { // Tokens live on the remote host; authorization must happen there. return nil } - if err := requireMicrosoftOAuthConfig(); err != nil { + if err := requireMicrosoftOAuthConfig(cfg); err != nil { return err } - msMgr := microsoft.NewManager( - cfg.Microsoft.ClientID, - microsoftTenantID(o365TenantID), - cfg.Microsoft.EffectiveRedirectURI(), - cfg.TokensDir(), - logger, - ) - fmt.Printf("Authorizing %s with Microsoft...\n", email) - if err := msMgr.Authorize(cmd.Context(), email); err != nil { - return fmt.Errorf("authorization failed: %w", err) + if err := authorizeO365(cmd, email); err != nil { + return err } if err := cmd.Flags().Set(oauthPreflightedFlag, "true"); err != nil { return fmt.Errorf("set --%s after authorization: %w", oauthPreflightedFlag, err) @@ -63,35 +63,57 @@ func newAddO365LocalCmd() *cobra.Command { Short: "Add a Microsoft 365 account via OAuth", Long: `Add a Microsoft 365 / Outlook.com email account using OAuth2 authentication. -This opens a browser for Microsoft authorization, then configures IMAP access +This opens a browser for Microsoft authorization (or, with --headless, prints a +device code to enter on any device), then configures IMAP access to outlook.office365.com automatically using the XOAUTH2 SASL mechanism. Requires a [microsoft] section in config.toml with your Azure AD app's client_id. See the docs for Azure AD app registration setup. +With --graph, the account syncs through the Microsoft Graph mail API instead +of IMAP. Use it when IMAP is turned off for the mailbox. It needs the Mail.Read +permission on the app registration. The first delete-staged for the account +asks for Mail.ReadWrite, which the app registration must also list. A Graph +account is a separate account: if the mailbox is also synced over IMAP, the +vault holds two copies, and 'msgvault dedup --collection' hides the extra ones. + Examples: msgvault add-o365 user@outlook.com - msgvault add-o365 user@company.com --tenant my-tenant-id`, + msgvault add-o365 user@outlook.com --headless + msgvault add-o365 user@company.com --tenant my-tenant-id + msgvault add-o365 user@company.com --graph`, Args: cobra.ExactArgs(1), RunE: runAddO365Local, } cmd.Flags().StringVar(&o365TenantID, "tenant", "", "Azure AD tenant ID (default: \"common\" for multi-tenant)") - cmd.Flags().BoolVar(&noDefaultIdentityAddO365, "no-default-identity", false, noDefaultIdentityHelp) + cmd.Flags().BoolVar(&noDefaultIdentityAddO365, "no-default-identity", false, savedDefaultIdentityHelp) + cmd.Flags().BoolVar(&o365Headless, "headless", false, + "Sign in with a device code instead of a local browser") + cmd.Flags().BoolVar(&o365Graph, "graph", false, "sync through the Microsoft Graph mail API instead of IMAP") registerOAuthPreflightedFlag(cmd) return cmd } func runAddO365Local(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger email := args[0] - if err := requireMicrosoftOAuthConfig(); err != nil { + if err := requireMicrosoftOAuthConfig(cfg); err != nil { return err } + if o365Graph { + return runAddO365GraphLocal(cmd, email) + } msMgr := microsoft.NewManager( cfg.Microsoft.ClientID, - microsoftTenantID(o365TenantID), + microsoftTenantID(o365TenantID, cfg), cfg.Microsoft.EffectiveRedirectURI(), cfg.TokensDir(), logger, @@ -102,9 +124,8 @@ func runAddO365Local(cmd *cobra.Command, args []string) error { return err } if !preflighted { - fmt.Printf("Authorizing %s with Microsoft...\n", email) - if err := msMgr.Authorize(cmd.Context(), email); err != nil { - return fmt.Errorf("authorization failed: %w", err) + if err := authorizeO365(cmd, email); err != nil { + return err } } @@ -124,7 +145,7 @@ func runAddO365Local(cmd *cobra.Command, args []string) error { AuthMethod: imapclient.AuthXOAuth2, } - s, cleanup, err := openWritableStoreAndInitForIngest() + s, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) if err != nil { return err } @@ -174,10 +195,13 @@ func runAddO365Local(cmd *cobra.Command, args []string) error { // Auto-default-identity must run BEFORE the legacy migration // retry — see comment in account_identity.go. + if err := setDefaultIdentityOptOut(cmd, s, source, noDefaultIdentityAddO365); err != nil { + return err + } if !noDefaultIdentityAddO365 { - confirmDefaultIdentity(cmd.OutOrStdout(), s, source.ID, email, email, "account-identifier") + confirmDefaultIdentity(cmd.OutOrStdout(), s, source.ID, email, email, "account-identifier", state.logger) } - if err := runPostSourceCreateMigrations(s); err != nil { + if err := runPostSourceCreateMigrationsForInvocation(s, state); err != nil { return fmt.Errorf("post-source-create migrations: %w", err) } @@ -191,6 +215,86 @@ func runAddO365Local(cmd *cobra.Command, args []string) error { return nil } +// authorizeO365 runs the Microsoft sign-in for the account kind: Graph mail +// with --graph, IMAP otherwise. +func authorizeO365(cmd *cobra.Command, email string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger + tenant := microsoftTenantID(o365TenantID, cfg) + redirect := cfg.Microsoft.EffectiveRedirectURI() + fmt.Printf("Authorizing %s with Microsoft...\n", email) + var err error + if o365Graph { + mgr := microsoft.NewGraphMailManager(cfg.Microsoft.ClientID, tenant, redirect, cfg.TokensDir(), logger) + if o365Headless { + mgr.UseDeviceCode() + } + err = mgr.Authorize(cmd.Context(), email) + } else { + mgr := microsoft.NewManager(cfg.Microsoft.ClientID, tenant, redirect, cfg.TokensDir(), logger) + if o365Headless { + mgr.UseDeviceCode() + } + err = mgr.Authorize(cmd.Context(), email) + } + if err != nil { + return fmt.Errorf("authorization failed: %w", err) + } + return nil +} + +// runAddO365GraphLocal creates an msmail source, the Graph mail counterpart of +// the IMAP source that add-o365 makes by default. +func runAddO365GraphLocal(cmd *cobra.Command, email string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + preflighted, err := oauthPreflighted(cmd) + if err != nil { + return err + } + if !preflighted { + if err := authorizeO365(cmd, email); err != nil { + return err + } + } + + s, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) + if err != nil { + return err + } + defer cleanup() + + source, err := s.GetOrCreateSource(sourceTypeMSMail, email) + if err != nil { + return fmt.Errorf("create source: %w", err) + } + if err := s.UpdateSourceDisplayName(source.ID, email); err != nil { + return fmt.Errorf("set display name: %w", err) + } + if err := setDefaultIdentityOptOut(cmd, s, source, noDefaultIdentityAddO365); err != nil { + return err + } + if !noDefaultIdentityAddO365 { + confirmDefaultIdentity(cmd.OutOrStdout(), s, source.ID, email, email, "account-identifier", state.logger) + } + if err := runPostSourceCreateMigrationsForInvocation(s, state); err != nil { + return fmt.Errorf("post-source-create migrations: %w", err) + } + + fmt.Printf("\nMicrosoft 365 account added for Graph mail sync!\n") + fmt.Printf(" Email: %s\n", email) + fmt.Println() + fmt.Println("You can now run:") + fmt.Printf(" msgvault sync %s\n", email) + return nil +} + // isMicrosoftIMAPSource returns true only if src is an IMAP source already // configured for Microsoft XOAUTH2 with the given username. This prevents // a non-Microsoft IMAP source (e.g. a password-auth source) that happens to diff --git a/cmd/msgvault/cmd/agent_token.go b/cmd/msgvault/cmd/agent_token.go index e49e8b7ca..5d074fd24 100644 --- a/cmd/msgvault/cmd/agent_token.go +++ b/cmd/msgvault/cmd/agent_token.go @@ -18,6 +18,7 @@ var ( agentTokenLabel string agentTokenPermissions []string agentTokenSourceIDs string // comma-separated source IDs + agentTokenSenders []string agentTokenJSON bool ) @@ -45,16 +46,15 @@ var agentTokenIssueCmd = &cobra.Command{ if err != nil { return err } + senderSelections, err := parseAgentTokenSenders(agentTokenSenders) + if err != nil { + return err + } client, _, err := OpenHTTPStore(cmd.Context()) if err != nil { return err } - result, err := client.IssueAgentToken( - cmd.Context(), - agentTokenLabel, - agentTokenPermissions, - sourceIDs, - ) + result, err := client.IssueAgentToken(cmd.Context(), agentTokenLabel, agentTokenPermissions, sourceIDs, senderSelections) if err != nil { return err } @@ -125,6 +125,25 @@ func parseAgentTokenSourceIDs(raw string) ([]int64, error) { return ids, nil } +func parseAgentTokenSenders(values []string) (map[int64][]string, error) { + if len(values) == 0 { + return nil, nil //nolint:nilnil // Omitted sender selections use the daemon's default. + } + result := make(map[int64][]string) + for _, value := range values { + sourceID, address, ok := strings.Cut(value, "=") + if !ok || strings.TrimSpace(address) == "" { + return nil, errors.New("--sender must use SOURCE_ID=ADDRESS") + } + id, err := strconv.ParseInt(strings.TrimSpace(sourceID), 10, 64) + if err != nil || id <= 0 { + return nil, errors.New("invalid sender source ID " + strconv.Quote(sourceID)) + } + result[id] = append(result[id], strings.TrimSpace(address)) + } + return result, nil +} + func printAgentTokenIssueResult(cmd *cobra.Command, r *generated.AgentTokenIssueResponse) { w := cmd.OutOrStdout() _, _ = fmt.Fprintf(w, "ID: %s\n", r.ID) @@ -133,7 +152,7 @@ func printAgentTokenIssueResult(cmd *cobra.Command, r *generated.AgentTokenIssue if len(r.Sources) > 0 { parts := make([]string, len(r.Sources)) for i, s := range r.Sources { - parts[i] = fmt.Sprintf("%d (%s)", s.ID, s.Identifier) + parts[i] = fmt.Sprintf("%d (%s; senders: %s)", s.ID, s.Identifier, strings.Join(s.SenderKeys, ",")) } _, _ = fmt.Fprintf(w, "Sources: %s\n", strings.Join(parts, ", ")) } @@ -155,7 +174,7 @@ func printAgentTokenList(cmd *cobra.Command, tokens []generated.AgentTokenView) for _, t := range tokens { sourceParts := make([]string, len(t.Sources)) for i, s := range t.Sources { - sourceParts[i] = fmt.Sprintf("%d/%s/%s", s.ID, s.Type, s.Identifier) + sourceParts[i] = fmt.Sprintf("%d/%s/%s[%s]", s.ID, s.Type, s.Identifier, strings.Join(s.SenderKeys, ",")) } _, _ = fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\n", t.ID, @@ -178,6 +197,8 @@ func init() { "Comma-separated list of permissions to grant (e.g. draft.create)") agentTokenIssueCmd.Flags().StringVar(&agentTokenSourceIDs, "source-ids", "", "Comma-separated list of source IDs the token may access") + agentTokenIssueCmd.Flags().StringArrayVar(&agentTokenSenders, "sender", nil, + "Restrict one source's sender identity (repeat as SOURCE_ID=ADDRESS)") agentTokenIssueCmd.Flags().BoolVar(&agentTokenJSON, flagJSON, false, "Output as JSON") agentTokenListCmd.Flags().BoolVar(&agentTokenJSON, flagJSON, false, "Output as JSON") } diff --git a/cmd/msgvault/cmd/agent_token_test.go b/cmd/msgvault/cmd/agent_token_test.go index 3d3c6152b..3c8a00be9 100644 --- a/cmd/msgvault/cmd/agent_token_test.go +++ b/cmd/msgvault/cmd/agent_token_test.go @@ -1,13 +1,16 @@ package cmd import ( + "bufio" "bytes" + "context" "encoding/json" "log/slog" "net/http" "net/http/httptest" "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -19,26 +22,34 @@ import ( "go.kenn.io/msgvault/internal/api" "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/daemonclient" + "go.kenn.io/msgvault/internal/gmail" + imaplib "go.kenn.io/msgvault/internal/imap" + "go.kenn.io/msgvault/internal/store" "go.kenn.io/msgvault/internal/testutil" ) // runAgentTokenCommand runs a single agent-token subcommand with the supplied // args and returns its combined stdout/stderr output. func runAgentTokenCommand( - t *testing.T, template *cobra.Command, args ...string, + ctx context.Context, t *testing.T, template *cobra.Command, args ...string, ) (string, error) { t.Helper() var output bytes.Buffer cmd := &cobra.Command{Use: template.Use, Args: template.Args, RunE: template.RunE} cmd.Flags().AddFlagSet(template.Flags()) cmd.Flags().VisitAll(func(flag *pflag.Flag) { - require.NoError(t, flag.Value.Set(flag.DefValue)) + if value, ok := flag.Value.(pflag.SliceValue); ok && flag.DefValue == "[]" { + require.NoError(t, value.Replace(nil)) + } else { + require.NoError(t, flag.Value.Set(flag.DefValue)) + } flag.Changed = false }) cmd.SetOut(&output) cmd.SetErr(&output) cmd.SetArgs(args) - err := cmd.Execute() + cmd.SetContext(ctx) + err := cmd.ExecuteContext(ctx) return output.String(), err } @@ -105,6 +116,297 @@ type agentTokenListFixture struct { Tokens []agentTokenFixtureView `json:"tokens"` } +func TestDelegatedDraftSourceScopeThroughHTTP(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + providerCalls := 0 + clientFactory := adapter.draftClientFactory + adapter.draftClientFactory = func(ctx context.Context, source *store.Source) (*imaplib.Client, error) { + providerCalls++ + return clientFactory(ctx, source) + } + server := httptest.NewServer(api.NewServerWithOptions(api.ServerOptions{ + Config: &config.Config{ + HomeDir: t.TempDir(), + Server: config.ServerConfig{APIKey: "owner-test-key", AgentAccess: true}, + }, + Store: adapter, + Logger: slog.New(slog.DiscardHandler), + }).Router()) + t.Cleanup(server.Close) + + issue := func(sourceID int64) string { + body, err := json.Marshal(map[string]any{ + "label": "test-agent", + "permissions": []string{"draft.create"}, + "source_ids": []int64{sourceID}, + }) + require.NoError(err) + req, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/agent-tokens", bytes.NewReader(body)) + require.NoError(err) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Api-Key", "owner-test-key") + resp, err := http.DefaultClient.Do(req) + require.NoError(err) + defer func() { _ = resp.Body.Close() }() + require.Equal(http.StatusCreated, resp.StatusCode) + var issued agentTokenIssueFixture + require.NoError(json.NewDecoder(resp.Body).Decode(&issued)) + return issued.Secret + } + + run := func(secret string) []api.CLIRunEvent { + args := []string{ + "draft-reply", strconv.FormatInt(fixture.parentID, 10), + "--from", testutil.IMAPTestUsername, "--body", "reply body", "--json", + } + body, err := json.Marshal(map[string]any{"args": args}) + require.NoError(err) + req, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/cli/run", bytes.NewReader(body)) + require.NoError(err) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Msgvault-Agent-Token", secret) + resp, err := http.DefaultClient.Do(req) + require.NoError(err) + defer func() { _ = resp.Body.Close() }() + require.Equal(http.StatusOK, resp.StatusCode) + var events []api.CLIRunEvent + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + var event api.CLIRunEvent + require.NoError(json.Unmarshal(scanner.Bytes(), &event)) + events = append(events, event) + } + require.NoError(scanner.Err()) + return events + } + + events := run(issue(fixture.source.ID)) + require.Len(events, 2) + assert.Equal(cliStreamStdout, events[0].Type) + var result draftReplyOutput + require.NoError(json.Unmarshal([]byte(events[0].Data), &result)) + assert.Equal(draftReplyStatusCreated, result.Status) + assert.Equal(fixture.source.ID, result.SourceID) + assert.Equal("Drafts", result.Mailbox) + assert.Equal("complete", events[1].Type) + assert.Equal(1, providerCalls) + + secondSource, err := fixture.store.GetOrCreateSource("imap", "other@example.com") + require.NoError(err) + events = run(issue(secondSource.ID)) + require.Len(events, 1) + assert.Equal("error", events[0].Type) + assert.Equal("not_permitted", events[0].Error) + assert.Equal(1, providerCalls, "an out-of-grant source must be rejected before provider work") +} + +func TestDelegatedGmailDraftLifecycleThroughHTTP(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture := newGmailDraftTestFixture(t) + draft := fixture.seedDraft(t) + providerCalls := 0 + factory := fixture.adapter.gmailDraftClientFactory + fixture.adapter.gmailDraftClientFactory = func(ctx context.Context, source *store.Source) (gmail.DraftAPI, error) { + providerCalls++ + return factory(ctx, source) + } + server := httptest.NewServer(api.NewServerWithOptions(api.ServerOptions{ + Config: &config.Config{ + HomeDir: t.TempDir(), + Server: config.ServerConfig{APIKey: "owner-test-key", AgentAccess: true}, + }, + Store: fixture.adapter, + Logger: slog.New(slog.DiscardHandler), + }).Router()) + t.Cleanup(server.Close) + + issue := func(permissions []string, sourceID int64, senderSelections ...map[string][]string) string { + request := map[string]any{ + "label": "test-agent", "permissions": permissions, "source_ids": []int64{sourceID}, + } + if len(senderSelections) > 0 { + request["sender_selections"] = senderSelections[0] + } + body, err := json.Marshal(request) + require.NoError(err) + req, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/agent-tokens", bytes.NewReader(body)) + require.NoError(err) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Api-Key", "owner-test-key") + resp, err := http.DefaultClient.Do(req) + require.NoError(err) + defer func() { _ = resp.Body.Close() }() + require.Equal(http.StatusCreated, resp.StatusCode) + var issued agentTokenIssueFixture + require.NoError(json.NewDecoder(resp.Body).Decode(&issued)) + return issued.Secret + } + run := func(secret string, args []string) (int, []api.CLIRunEvent) { + body, err := json.Marshal(map[string]any{"args": args}) + require.NoError(err) + req, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/cli/run", bytes.NewReader(body)) + require.NoError(err) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Msgvault-Agent-Token", secret) + resp, err := http.DefaultClient.Do(req) + require.NoError(err) + defer func() { _ = resp.Body.Close() }() + var events []api.CLIRunEvent + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + var event api.CLIRunEvent + require.NoError(json.Unmarshal(scanner.Bytes(), &event)) + events = append(events, event) + } + require.NoError(scanner.Err()) + return resp.StatusCode, events + } + + sourceID := strconv.FormatInt(fixture.source.ID, 10) + ownerSender := map[string][]string{sourceID: {"owner@example.test"}} + getCode, getEvents := run(issue([]string{"draft.create"}, fixture.source.ID, ownerSender), []string{ + api.CLIRunDraftGetCommand, draft.DraftID, "--json", + }) + require.Equal(http.StatusOK, getCode) + require.Len(getEvents, 2) + assert.Equal(cliStreamStdout, getEvents[0].Type) + assert.Contains(getEvents[0].Data, `"content":"original`) + assert.Equal("complete", getEvents[1].Type) + + originalRaw, err := fixture.store.GetMessageRawContext(t.Context(), draft.CurrentMessageID) + require.NoError(err) + differentSenderRaw := strings.Replace(string(originalRaw), "From: owner@example.test", "From: other@example.test", 1) + require.NotEqual(string(originalRaw), differentSenderRaw) + require.NoError(fixture.store.UpsertMessageRaw(draft.CurrentMessageID, []byte(differentSenderRaw))) + beforeSenderProviderCalls := providerCalls + beforeSenderClientCalls := fixture.client.getCalls + fixture.client.updateCalls + fixture.client.deleteCalls + restrictedCode, restrictedEvents := run(issue([]string{"draft.create"}, fixture.source.ID, ownerSender), []string{ + api.CLIRunDraftGetCommand, draft.DraftID, "--json", + }) + require.Equal(http.StatusOK, restrictedCode) + require.Len(restrictedEvents, 1) + assert.Equal("error", restrictedEvents[0].Type) + assert.Equal("not_permitted", restrictedEvents[0].Error) + assert.Equal(beforeSenderProviderCalls, providerCalls) + assert.Equal(beforeSenderClientCalls, fixture.client.getCalls+fixture.client.updateCalls+fixture.client.deleteCalls) + emptySenders := map[string][]string{sourceID: {}} + emptyCode, emptyEvents := run(issue([]string{"draft.create"}, fixture.source.ID, emptySenders), []string{ + api.CLIRunDraftGetCommand, draft.DraftID, "--json", + }) + require.Equal(http.StatusOK, emptyCode) + require.Len(emptyEvents, 1) + assert.Equal("error", emptyEvents[0].Type) + assert.Equal("not_permitted", emptyEvents[0].Error) + assert.Equal(beforeSenderProviderCalls, providerCalls) + assert.Equal(beforeSenderClientCalls, fixture.client.getCalls+fixture.client.updateCalls+fixture.client.deleteCalls) + require.NoError(fixture.store.UpsertMessageRaw(draft.CurrentMessageID, originalRaw)) + + fixture.client.updateDraft = &gmail.Draft{ + ID: "gmail-draft-managed", + Message: gmail.RawMessage{ID: "gmail-message-edited", ThreadID: "gmail-thread-1"}, + } + editCode, editEvents := run(issue([]string{"draft.edit"}, fixture.source.ID), []string{ + api.CLIRunDraftEditCommand, draft.DraftID, "--revision", "1", "--body", "edited", "--json", + }) + require.Equal(http.StatusOK, editCode) + require.Len(editEvents, 2) + assert.Equal(cliStreamStdout, editEvents[0].Type) + assert.Contains(editEvents[0].Data, `"status":"edited"`) + assert.Contains(editEvents[0].Data, `"revision":2`) + assert.Equal("complete", editEvents[1].Type) + + fixture.client.getDraft = &gmail.Draft{ + ID: "gmail-draft-managed", + Message: gmail.RawMessage{ID: "gmail-message-edited", ThreadID: "gmail-thread-1"}, + } + getAfterEditCode, getAfterEditEvents := run(issue([]string{"draft.create"}, fixture.source.ID, ownerSender), []string{ + api.CLIRunDraftGetCommand, draft.DraftID, "--json", + }) + require.Equal(http.StatusOK, getAfterEditCode) + require.Len(getAfterEditEvents, 2) + assert.Equal(cliStreamStdout, getAfterEditEvents[0].Type) + assert.Contains(getAfterEditEvents[0].Data, `"content":"edited`) + assert.Contains(getAfterEditEvents[0].Data, `"revision":2`) + assert.Equal("complete", getAfterEditEvents[1].Type) + + beforeNonDraftProviderCalls := providerCalls + beforeNonDraftClientCalls := fixture.client.getCalls + fixture.client.updateCalls + fixture.client.deleteCalls + nonDraftCode, nonDraftEvents := run(issue([]string{"draft.edit"}, fixture.source.ID), []string{ + api.CLIRunDraftGetCommand, strconv.FormatInt(fixture.parentID, 10), "--json", + }) + require.Equal(http.StatusOK, nonDraftCode) + require.Len(nonDraftEvents, 1) + assert.Equal("error", nonDraftEvents[0].Type) + assert.Equal("not_permitted", nonDraftEvents[0].Error) + assert.Equal(beforeNonDraftProviderCalls, providerCalls) + assert.Equal(beforeNonDraftClientCalls, fixture.client.getCalls+fixture.client.updateCalls+fixture.client.deleteCalls) + + deleteCode, deleteEvents := run(issue([]string{"draft.delete"}, fixture.source.ID), []string{ + api.CLIRunDraftDeleteCommand, draft.DraftID, "--revision", "2", "--json", + }) + require.Equal(http.StatusOK, deleteCode) + require.Len(deleteEvents, 2) + assert.Contains(deleteEvents[0].Data, `"status":"deleted"`) + assert.Contains(deleteEvents[0].Data, `"lifecycle":"discarded"`) + assert.Equal("complete", deleteEvents[1].Type) + + secondSource, err := fixture.store.GetOrCreateSource("gmail", "other@example.test") + require.NoError(err) + beforeProviderCalls := providerCalls + beforeClientCalls := fixture.client.getCalls + fixture.client.updateCalls + fixture.client.deleteCalls + deleteCode, deleteEvents = run(issue([]string{"draft.delete"}, secondSource.ID), []string{ + api.CLIRunDraftDeleteCommand, draft.DraftID, "--revision", "2", "--json", + }) + require.Equal(http.StatusOK, deleteCode) + require.Len(deleteEvents, 1) + assert.Equal("error", deleteEvents[0].Type) + assert.Equal("not_permitted", deleteEvents[0].Error) + assert.Equal(beforeProviderCalls, providerCalls) + assert.Equal(beforeClientCalls, fixture.client.getCalls+fixture.client.updateCalls+fixture.client.deleteCalls) +} + +func TestDelegatedGmailDraftGetThroughAgentCLI(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture := newGmailDraftTestFixture(t) + draft := fixture.seedDraft(t) + server := httptest.NewServer(api.NewServerWithOptions(api.ServerOptions{ + Config: &config.Config{ + HomeDir: t.TempDir(), + Server: config.ServerConfig{APIKey: "owner-test-key", AgentAccess: true}, + }, + Store: fixture.adapter, + Logger: slog.New(slog.DiscardHandler), + }).Router()) + t.Cleanup(server.Close) + owner, err := daemonclient.New(daemonclient.Config{URL: server.URL, APIKey: "owner-test-key", AllowInsecure: true}) + require.NoError(err) + t.Cleanup(func() { _ = owner.Close() }) + grant, err := owner.IssueAgentToken(t.Context(), "test agent", []string{"draft.create"}, []int64{fixture.source.ID}, nil) + require.NoError(err) + tokenFile := filepath.Join(t.TempDir(), "agent.token") + require.NoError(os.WriteFile(tokenFile, []byte(grant.Secret+"\n"), 0o600)) + ctx := testInvocationContext(t.Context(), config.NewDefaultConfig(), invocationOptions{ + agentURL: server.URL, agentTokenFile: tokenFile, agentAllowInsecure: true, + agentURLChanged: true, agentTokenChanged: true, + }) + root := newTestRootCmd() + root.SetContext(ctx) + root.AddCommand(newDraftGetCommand()) + root.SetArgs([]string{api.CLIRunDraftGetCommand, draft.DraftID}) + var stdout, stderr bytes.Buffer + root.SetOut(&stdout) + root.SetErr(&stderr) + err = root.ExecuteContext(ctx) + require.NoError(err) + assert.Contains(stdout.String(), "content:\noriginal") + assert.Empty(stderr.String()) +} + // TestAgentTokenIssueOutputsSecret verifies that the issue subcommand (row 6): // - sends POST /api/v1/agent-tokens with the correct JSON body // - displays the token ID, label, permissions, and one-time secret in plain @@ -128,11 +430,12 @@ func TestAgentTokenIssueOutputsSecret(t *testing.T) { _, _ = w.Write([]byte(agentTokenIssueResponseJSON(wantSecret))) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) + _ = testCtx - output, err := runAgentTokenCommand(t, agentTokenIssueCmd, + output, err := runAgentTokenCommand(testCtx, t, agentTokenIssueCmd, "--label", "Test Agent", "--permissions", "draft.create", "--source-ids", "1", @@ -146,7 +449,7 @@ func TestAgentTokenIssueOutputsSecret(t *testing.T) { assert.Contains(output, "tok_abc123") assert.Contains(output, "Test Agent") assert.Contains(output, wantSecret, "one-time secret must appear in issue output") - output, err = runAgentTokenCommand(t, agentTokenIssueCmd, + output, err = runAgentTokenCommand(testCtx, t, agentTokenIssueCmd, "--label", "Test Agent", "--permissions", "draft.create", "--source-ids", "1", "--json") require.NoError(err) assert.True(strings.HasSuffix(output, "\n"), "JSON output must end with a newline") @@ -168,11 +471,12 @@ func TestAgentTokenListFormatsTable(t *testing.T) { _, _ = w.Write([]byte(agentTokenListResponseJSON())) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) + _ = testCtx - output, err := runAgentTokenCommand(t, agentTokenListCmd) + output, err := runAgentTokenCommand(testCtx, t, agentTokenListCmd) require.NoError(err) assert.Equal(http.MethodGet, gotMethod) @@ -198,11 +502,12 @@ func TestAgentTokenRevokeCallsDelete(t *testing.T) { w.WriteHeader(http.StatusNoContent) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) + _ = testCtx - output, err := runAgentTokenCommand(t, agentTokenRevokeCmd, "tok_abc123") + output, err := runAgentTokenCommand(testCtx, t, agentTokenRevokeCmd, "tok_abc123") require.NoError(err) assert.Equal(http.MethodDelete, gotMethod) @@ -233,25 +538,22 @@ func TestOpenAgentDelegatedStore(t *testing.T) { }) require.NoError(err) t.Cleanup(func() { _ = owner.Close() }) - grant, err := owner.IssueAgentToken(t.Context(), "test agent", []string{"draft.create"}, []int64{source.ID}) + grant, err := owner.IssueAgentToken(t.Context(), "test agent", []string{"draft.create"}, []int64{source.ID}, nil) require.NoError(err) // Write the token to a temp file. tokenFile := filepath.Join(t.TempDir(), "agent.token") require.NoError(os.WriteFile(tokenFile, []byte(grant.Secret+"\n"), 0o600)) - // Set package-level flags. - oldURL, oldFile, oldInsecure := agentURL, agentTokenFile, agentAllowInsecure - agentURL = server.URL - agentTokenFile = tokenFile - agentAllowInsecure = true - t.Cleanup(func() { - agentURL = oldURL - agentTokenFile = oldFile - agentAllowInsecure = oldInsecure + ctx := testInvocationContext(t.Context(), config.NewDefaultConfig(), invocationOptions{ + agentURL: server.URL, + agentTokenFile: tokenFile, + agentAllowInsecure: true, + agentURLChanged: true, + agentTokenChanged: true, }) - client, info, err := OpenHTTPStore(t.Context()) + client, info, err := OpenHTTPStore(ctx) require.NoError(err) require.NotNil(client) t.Cleanup(func() { _ = client.Close() }) @@ -269,17 +571,15 @@ func TestOpenAgentDelegatedStoreRejectsLocalFlag(t *testing.T) { tokenFile := filepath.Join(t.TempDir(), "agent.token") require.NoError(t, os.WriteFile(tokenFile, []byte("mva1_abc"), 0o600)) - oldURL, oldFile, oldUseLocal := agentURL, agentTokenFile, useLocal - agentURL = "http://daemon:8080" - agentTokenFile = tokenFile - useLocal = true - t.Cleanup(func() { - agentURL = oldURL - agentTokenFile = oldFile - useLocal = oldUseLocal + ctx := testInvocationContext(t.Context(), config.NewDefaultConfig(), invocationOptions{ + agentURL: "http://daemon:8080", + agentTokenFile: tokenFile, + useLocal: true, + agentURLChanged: true, + agentTokenChanged: true, }) - _, _, err := OpenHTTPStore(t.Context()) + _, _, err := OpenHTTPStore(ctx) require.Error(t, err) assert.Contains(t, err.Error(), "incompatible", err.Error()) } @@ -292,23 +592,23 @@ func TestOpenAgentDelegatedStoreRequiresBothFlags(t *testing.T) { require.NoError(t, os.WriteFile(tokenFile, []byte("mva1_abc"), 0o600)) t.Run("agent-url alone errors", func(t *testing.T) { - oldURL, oldFile := agentURL, agentTokenFile - agentURL = "http://daemon:8080" - agentTokenFile = "" - t.Cleanup(func() { agentURL = oldURL; agentTokenFile = oldFile }) + ctx := testInvocationContext(t.Context(), config.NewDefaultConfig(), invocationOptions{ + agentURL: "http://daemon:8080", + agentURLChanged: true, + }) - _, _, err := openAgentDelegatedStore(t.Context()) + _, _, err := openAgentDelegatedStore(ctx, invocationFromContext(ctx)) require.Error(t, err) assert.Contains(t, err.Error(), "--agent-token-file") }) t.Run("agent-token-file alone errors", func(t *testing.T) { - oldURL, oldFile := agentURL, agentTokenFile - agentURL = "" - agentTokenFile = tokenFile - t.Cleanup(func() { agentURL = oldURL; agentTokenFile = oldFile }) + ctx := testInvocationContext(t.Context(), config.NewDefaultConfig(), invocationOptions{ + agentTokenFile: tokenFile, + agentTokenChanged: true, + }) - _, _, err := openAgentDelegatedStore(t.Context()) + _, _, err := openAgentDelegatedStore(ctx, invocationFromContext(ctx)) require.Error(t, err) assert.Contains(t, err.Error(), "--agent-url") }) @@ -322,8 +622,7 @@ func TestOpenHTTPStoreRejectsExplicitEmptyAgentFlags(t *testing.T) { } { t.Run(strings.Join(args, " "), func(t *testing.T) { require := require.New(t) - withAgentFlags(t, "", "") - withStoreResolverConfig(t, &config.Config{ + ctx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: "https://daemon.example", APIKey: "owner-test-key"}, }) for _, name := range []string{"agent-url", "agent-token-file"} { @@ -332,11 +631,15 @@ func TestOpenHTTPStoreRejectsExplicitEmptyAgentFlags(t *testing.T) { flag.Changed = false t.Cleanup(func() { flag.Changed = changed }) } + root := &cobra.Command{Use: "msgvault"} + root.PersistentFlags().AddFlagSet(rootCmd.PersistentFlags()) cmd := &cobra.Command{Use: "draft-reply"} - cmd.Flags().AddFlagSet(rootCmd.PersistentFlags()) + root.AddCommand(cmd) + cmd.SetContext(ctx) require.NoError(cmd.ParseFlags(args)) + prepareInvocation(cmd) - client, info, err := OpenHTTPStore(t.Context()) + client, info, err := OpenHTTPStore(cmd.Context()) if client != nil { t.Cleanup(func() { _ = client.Close() }) } @@ -348,18 +651,16 @@ func TestOpenHTTPStoreRejectsExplicitEmptyAgentFlags(t *testing.T) { // TestAgentModeRejectsConfigFlag verifies that --config is rejected in // agent-delegated mode for a delegated-capable command. func TestAgentModeRejectsConfigFlag(t *testing.T) { - oldURL, oldFile := agentURL, agentTokenFile - oldCfg := cfgFile - agentURL = "http://daemon:8080" - agentTokenFile = "/tmp/token" - cfgFile = "/tmp/config.toml" - t.Cleanup(func() { - agentURL = oldURL - agentTokenFile = oldFile - cfgFile = oldCfg + ctx := testInvocationContext(t.Context(), config.NewDefaultConfig(), invocationOptions{ + agentURL: "http://daemon:8080", + agentTokenFile: "/tmp/token", + agentURLChanged: true, + agentTokenChanged: true, + cfgFile: "/tmp/config.toml", }) cmd := &cobra.Command{Use: "draft-reply"} + cmd.SetContext(ctx) err := rootCmd.PersistentPreRunE(cmd, nil) require.Error(t, err) assert.Contains(t, err.Error(), "--config") @@ -368,18 +669,16 @@ func TestAgentModeRejectsConfigFlag(t *testing.T) { // TestAgentModeRejectsHomeFlag verifies that --home is rejected in // agent-delegated mode for a delegated-capable command. func TestAgentModeRejectsHomeFlag(t *testing.T) { - oldURL, oldFile := agentURL, agentTokenFile - oldHome := homeDir - agentURL = "http://daemon:8080" - agentTokenFile = "/tmp/token" - homeDir = "/tmp/home" - t.Cleanup(func() { - agentURL = oldURL - agentTokenFile = oldFile - homeDir = oldHome + ctx := testInvocationContext(t.Context(), config.NewDefaultConfig(), invocationOptions{ + agentURL: "http://daemon:8080", + agentTokenFile: "/tmp/token", + agentURLChanged: true, + agentTokenChanged: true, + homeDir: "/tmp/home", }) cmd := &cobra.Command{Use: "draft-reply"} + cmd.SetContext(ctx) err := rootCmd.PersistentPreRunE(cmd, nil) require.Error(t, err) assert.Contains(t, err.Error(), "--home") @@ -388,12 +687,14 @@ func TestAgentModeRejectsHomeFlag(t *testing.T) { // TestOpenAgentDelegatedStoreRejectsNonexistentTokenFile verifies that a missing // token file produces a clear error (P3: token-file read-error branch). func TestOpenAgentDelegatedStoreRejectsNonexistentTokenFile(t *testing.T) { - oldURL, oldFile := agentURL, agentTokenFile - agentURL = "https://daemon:8080" - agentTokenFile = filepath.Join(t.TempDir(), "does-not-exist.token") - t.Cleanup(func() { agentURL = oldURL; agentTokenFile = oldFile }) + ctx := testInvocationContext(t.Context(), config.NewDefaultConfig(), invocationOptions{ + agentURL: "https://daemon:8080", + agentTokenFile: filepath.Join(t.TempDir(), "does-not-exist.token"), + agentURLChanged: true, + agentTokenChanged: true, + }) - _, _, err := openAgentDelegatedStore(t.Context()) + _, _, err := openAgentDelegatedStore(ctx, invocationFromContext(ctx)) require.Error(t, err) assert.Contains(t, err.Error(), "read agent token file") } @@ -405,24 +706,28 @@ func TestOpenAgentDelegatedStoreRejectsEmptyTokenFile(t *testing.T) { t.Run("empty file", func(t *testing.T) { require.NoError(t, os.WriteFile(tokenFile, []byte(""), 0o600)) - oldURL, oldFile := agentURL, agentTokenFile - agentURL = "https://daemon:8080" - agentTokenFile = tokenFile - t.Cleanup(func() { agentURL = oldURL; agentTokenFile = oldFile }) + ctx := testInvocationContext(t.Context(), config.NewDefaultConfig(), invocationOptions{ + agentURL: "https://daemon:8080", + agentTokenFile: tokenFile, + agentURLChanged: true, + agentTokenChanged: true, + }) - _, _, err := openAgentDelegatedStore(t.Context()) + _, _, err := openAgentDelegatedStore(ctx, invocationFromContext(ctx)) require.Error(t, err) assert.Contains(t, err.Error(), "is empty") }) t.Run("whitespace only", func(t *testing.T) { require.NoError(t, os.WriteFile(tokenFile, []byte(" \n\t \n"), 0o600)) - oldURL, oldFile := agentURL, agentTokenFile - agentURL = "https://daemon:8080" - agentTokenFile = tokenFile - t.Cleanup(func() { agentURL = oldURL; agentTokenFile = oldFile }) + ctx := testInvocationContext(t.Context(), config.NewDefaultConfig(), invocationOptions{ + agentURL: "https://daemon:8080", + agentTokenFile: tokenFile, + agentURLChanged: true, + agentTokenChanged: true, + }) - _, _, err := openAgentDelegatedStore(t.Context()) + _, _, err := openAgentDelegatedStore(ctx, invocationFromContext(ctx)) require.Error(t, err) assert.Contains(t, err.Error(), "is empty") }) diff --git a/cmd/msgvault/cmd/archive_remote_images.go b/cmd/msgvault/cmd/archive_remote_images.go index 643c61191..462eefa50 100644 --- a/cmd/msgvault/cmd/archive_remote_images.go +++ b/cmd/msgvault/cmd/archive_remote_images.go @@ -5,10 +5,11 @@ import ( "fmt" "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/remoteimage" ) -func configuredRemoteImageFetcher() *remoteimage.Fetcher { +func configuredRemoteImageFetcher(cfg *config.Config) *remoteimage.Fetcher { if cfg == nil || !cfg.Sync.ArchiveRemoteImages { return nil } @@ -25,6 +26,12 @@ func newArchiveRemoteImagesCmd() *cobra.Command { Long: "Download remote img src images from archived email for offline viewing.\n\nDownloading can activate tracking pixels and disclose the archive server's IP\naddress to senders. This command always requires --allow-tracking. It does not\nenable automatic archiving, change original messages, or re-fetch stored images.", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger if !allowTracking { return errors.New("remote image downloads can activate tracking pixels; pass --allow-tracking to consent") } @@ -34,7 +41,7 @@ func newArchiveRemoteImagesCmd() *cobra.Command { if !isDaemonCLISubprocess() { return runDaemonCLICommandHTTPFromCobra(cmd, args) } - st, cleanup, err := openWritableStoreAndInitForIngest() + st, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) if err != nil { return err } @@ -51,7 +58,7 @@ func newArchiveRemoteImagesCmd() *cobra.Command { err = errors.Join(err, st.AdvanceDerivedDataRevision()) } // Partial success and cancellation can still leave new attachments. - return errors.Join(err, rebuildCacheAfterWrite(cfg.DatabaseDSN())) + return errors.Join(err, rebuildCacheAfterWrite(cfg.DatabaseDSN(), state)) }, } command.Flags().BoolVar(&allowTracking, "allow-tracking", false, "Consent to sender-controlled image requests and tracking pixels") diff --git a/cmd/msgvault/cmd/archive_remote_images_test.go b/cmd/msgvault/cmd/archive_remote_images_test.go index 1510a50a9..905972246 100644 --- a/cmd/msgvault/cmd/archive_remote_images_test.go +++ b/cmd/msgvault/cmd/archive_remote_images_test.go @@ -27,6 +27,7 @@ import ( func TestArchiveRemoteImagesRequiresTrackingConsentBeforeDispatch(t *testing.T) { require := require.New(t) command := newArchiveRemoteImagesCmd() + command.SetContext(testInvocationContext(t.Context(), config.NewDefaultConfig(), invocationOptions{})) require.ErrorContains(command.RunE(command, nil), "--allow-tracking") require.NoError(command.Flags().Set("allow-tracking", "true")) require.NoError(command.Flags().Set("limit", "-1")) @@ -71,20 +72,18 @@ func TestArchivedRemoteImagesThroughDaemonAdapter(t *testing.T) { func TestConfiguredRemoteImageFetcherDefaultsOff(t *testing.T) { assert := assert.New(t) - previous := cfg - t.Cleanup(func() { cfg = previous }) - cfg = config.NewDefaultConfig() - assert.Nil(configuredRemoteImageFetcher()) + cfg := config.NewDefaultConfig() + assert.Nil(configuredRemoteImageFetcher(cfg)) cfg.Sync.ArchiveRemoteImages = true - assert.NotNil(configuredRemoteImageFetcher()) + assert.NotNil(configuredRemoteImageFetcher(cfg)) cfg.Sync.ArchiveRemoteImages = false - assert.Nil(configuredRemoteImageFetcher()) + assert.Nil(configuredRemoteImageFetcher(cfg)) } func TestArchiveRemoteImagesRefreshesExistingCacheOnReuse(t *testing.T) { assert, require := assert.New(t), require.New(t) configuration := lifecycleTestConfig(t.TempDir()) - withStoreResolverConfig(t, configuration) + testCtx := withStoreResolverConfig(t, configuration) t.Setenv(daemonCLISubprocessEnv, strconv.Itoa(os.Getppid())) st, err := store.Open(configuration.DatabaseDSN()) @@ -115,7 +114,7 @@ func TestArchiveRemoteImagesRefreshesExistingCacheOnReuse(t *testing.T) { content := []byte("\x89PNG\r\n\x1a\nimage") receipt, err := export.StoreAttachmentFileDurable(configuration.AttachmentsDir(), &mime.Attachment{ContentType: "image/png", Content: content}) require.NoError(err) - require.NoError(st.UpsertRemoteImageAttachment(t.Context(), id, store.AttachmentWrite{ + require.NoError(st.UpsertRemoteImageAttachment(testCtx, id, store.AttachmentWrite{ Filename: "image.png", MIMEType: "image/png", StoragePath: receipt.StoragePath, ContentHash: receipt.ContentHash, Size: int64(len(content)), SourceAttachmentID: key, SourcePartKey: key, ContentID: key, Role: store.AttachmentRoleInline, RoleSource: store.AttachmentRoleSourceImporterSemantics, @@ -123,14 +122,17 @@ func TestArchiveRemoteImagesRefreshesExistingCacheOnReuse(t *testing.T) { require.NoError(st.Close()) command := newArchiveRemoteImagesCmd() + command.SetContext(testCtx) command.SetOut(io.Discard) command.SetErr(io.Discard) command.SetArgs([]string{"--allow-tracking"}) - require.NoError(command.Execute()) + require.NotNil(invocationFromCommand(command)) + require.Same(configuration, invocationFromCommand(command).cfg) + require.NoError(command.ExecuteContext(testCtx)) engine, err := query.NewDuckDBEngine(configuration.AnalyticsDir(), "", nil) require.NoError(err) - result, queryErr := engine.QuerySQL(t.Context(), ` + result, queryErr := engine.QuerySQL(testCtx, ` SELECT m.attachment_count, m.has_attachments, COUNT(a.attachment_id) FROM messages m LEFT JOIN attachments a ON a.message_id = m.id GROUP BY m.id, m.attachment_count, m.has_attachments`) diff --git a/cmd/msgvault/cmd/attachment_maintenance.go b/cmd/msgvault/cmd/attachment_maintenance.go index 845e26641..10359d5ad 100644 --- a/cmd/msgvault/cmd/attachment_maintenance.go +++ b/cmd/msgvault/cmd/attachment_maintenance.go @@ -5,20 +5,32 @@ import ( "errors" "fmt" "log/slog" + "sync/atomic" + "time" "go.kenn.io/kit/packstore" "go.kenn.io/msgvault/internal/api" "go.kenn.io/msgvault/internal/attachmentstore" + "go.kenn.io/msgvault/internal/export" + "go.kenn.io/msgvault/internal/jobctx" "go.kenn.io/msgvault/internal/scheduler" "go.kenn.io/msgvault/internal/store" ) const ( - automaticAttachmentBytes = int64(256 << 20) - attachmentMaintenanceJob = "attachment-maintenance" - attachmentMaintenanceCron = "17 3 * * *" - importMboxCommand = "import-mbox" + automaticAttachmentBytes = int64(256 << 20) + // Kit's Pack and Repack cannot stop at a checkpoint on request, so the + // scheduled jobs are not preemptible. This limit alone returns the gate. + automaticAttachmentMaxRuntime = time.Minute + attachmentMaintenanceJob = "attachment-maintenance" + attachmentMaintenanceCron = "17 3 * * *" + // attachmentPackJob packs blobs that scheduled syncs left loose. A pass + // re-reads the whole pack catalog, so it runs a few times a day rather + // than after every sync. Bounded follow-ups drain any remaining backlog. + attachmentPackJob = "attachment-pack" + attachmentPackCron = "41 */6 * * *" + importMboxCommand = "import-mbox" ) // attachmentMaintenance coordinates daemon-owned attachment maintenance. Its @@ -31,6 +43,10 @@ type attachmentMaintenance struct { blob *attachmentstore.Store logger *slog.Logger packCreationEnabled bool + attachmentsDir string + // packPending records that a scheduled sync wrote loose blobs that no + // pack pass has drained yet. Startup requests one scan of existing blobs. + packPending atomic.Bool } func newAttachmentMaintenance( @@ -59,6 +75,7 @@ func newAttachmentMaintenance( blob: attachmentstore.Wrap(maintainer.Store()), logger: logger, packCreationEnabled: packCreationEnabled, + attachmentsDir: attachmentsDir, }, nil } @@ -117,14 +134,24 @@ func (m *attachmentMaintenance) runAutomaticPack(ctx context.Context, emitWarnin m.log().Debug("automatic attachment packing disabled") return nil } + start := time.Now() stats, err := m.pack(ctx, automaticAttachmentBytes) + // Each counter is a committed change a later pass need not repeat. + // PacksQuarantined is not: a damaged pack stays in place and is counted + // again on every pass. + if stats.BlobsPacked > 0 || stats.PacksAdopted > 0 || stats.PacksRemoved > 0 || + stats.MappingsPruned > 0 || stats.RecordsDropped > 0 || stats.LooseSwept > 0 || + stats.LooseOrphansRemoved > 0 { + jobctx.RecordProgress(ctx) + } + duration := time.Since(start) if err != nil { - if errors.Is(err, context.Canceled) { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { m.log().Info("automatic attachment maintenance canceled") return err } - m.logAutomaticPackSummary("automatic attachment maintenance progress", stats) + m.logAutomaticPackSummary("automatic attachment maintenance progress", stats, duration) const retry = "run `msgvault pack-attachments` to retry" m.log().Warn("automatic attachment maintenance failed", "error", err, @@ -139,12 +166,16 @@ func (m *attachmentMaintenance) runAutomaticPack(ctx context.Context, emitWarnin return err } - m.logAutomaticPackSummary("automatic attachment maintenance complete", stats) + m.logAutomaticPackSummary("automatic attachment maintenance complete", stats, duration) + if stats.BudgetExhausted { + m.markPackPending() + } return nil } -func (m *attachmentMaintenance) logAutomaticPackSummary(message string, stats packstore.PackStats) { +func (m *attachmentMaintenance) logAutomaticPackSummary(message string, stats packstore.PackStats, duration time.Duration) { m.log().Info(message, + "duration", duration.Round(time.Millisecond), "max_bytes", automaticAttachmentBytes, "packs_sealed", stats.PacksSealed, "blobs_packed", stats.BlobsPacked, @@ -169,13 +200,18 @@ func (m *attachmentMaintenance) runAutomaticRepack(ctx context.Context, emitWarn m.log().Debug("automatic attachment repacking disabled") return nil } + start := time.Now() stats, err := m.repack(ctx, automaticAttachmentBytes) + if stats.PacksRewritten > 0 || stats.PacksRemoved > 0 || stats.MappingsPruned > 0 { + jobctx.RecordProgress(ctx) + } + duration := time.Since(start) if err != nil { - if errors.Is(err, context.Canceled) { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { m.log().Info("automatic attachment repack canceled") return err } - m.logAutomaticRepackSummary("automatic attachment repack progress", stats) + m.logAutomaticRepackSummary("automatic attachment repack progress", stats, duration) const retry = "run `msgvault repack-attachments` to retry" m.log().Warn("automatic attachment repack failed", "error", err, "retry", retry) if emitWarning != nil { @@ -186,12 +222,13 @@ func (m *attachmentMaintenance) runAutomaticRepack(ctx context.Context, emitWarn } return err } - m.logAutomaticRepackSummary("automatic attachment repack complete", stats) + m.logAutomaticRepackSummary("automatic attachment repack complete", stats, duration) return nil } -func (m *attachmentMaintenance) logAutomaticRepackSummary(message string, stats packstore.RepackStats) { +func (m *attachmentMaintenance) logAutomaticRepackSummary(message string, stats packstore.RepackStats, duration time.Duration) { m.log().Info(message, + "duration", duration.Round(time.Millisecond), "max_bytes", automaticAttachmentBytes, "mappings_pruned", stats.MappingsPruned, "packs_selected", stats.PacksSelected, @@ -204,16 +241,79 @@ func (m *attachmentMaintenance) logAutomaticRepackSummary(message string, stats "budget_exhausted", stats.BudgetExhausted) } +// runScheduledPack bounds existing-blob verification as well as new pack bytes. +func (m *attachmentMaintenance) runScheduledPack(ctx context.Context) error { + if !m.packCreationEnabled { + return nil + } + packCtx, pass, err := m.store.BeginPackVerification(ctx, 128, 32<<20) + if err != nil { + return err + } + if err := m.runAutomaticPack(packCtx, nil); err != nil { + return err + } + more, err := m.store.FinishPackVerification(ctx, pass) + if err != nil { + return err + } + if more { + // A completed verification cycle restarts on the next request. Only an + // unfinished window provides a resume point for a budgeted follow-up. + jobctx.RecordProgress(ctx) + m.markPackPending() + } + return nil +} + // daily runs the two bounded phases in order. A failed pack phase stops the // job so the scheduler records the failure instead of obscuring it with a // second maintenance result. func (m *attachmentMaintenance) daily(ctx context.Context) error { - if err := m.runAutomaticPack(ctx, nil); err != nil { + m.packPending.Store(false) + if err := m.runScheduledPack(ctx); err != nil { + m.markPackPending() return err } + if m.packPending.Load() { + return scheduler.ErrReschedule + } return m.runAutomaticRepack(ctx, nil) } +// markPackPending records that loose blobs await the next pack pass. +func (m *attachmentMaintenance) markPackPending() { + if m != nil { + m.packPending.Store(true) + } +} + +// runPendingPack runs one automatic pack pass when a scheduled sync left new +// loose blobs since the last pass. A failed pass leaves the request pending; +// an exhausted byte budget queues another pass behind waiting work. +func (m *attachmentMaintenance) runPendingPack(ctx context.Context) error { + if m == nil || !m.packPending.Swap(false) { + return nil + } + if err := m.runScheduledPack(ctx); err != nil { + m.packPending.Store(true) + return err + } + if m.packPending.Load() { + return scheduler.ErrReschedule + } + return nil +} + +// looseBlobWrites reports the in-process count of loose blobs created under +// this maintenance's attachments directory. +func (m *attachmentMaintenance) looseBlobWrites() int64 { + if m == nil || m.attachmentsDir == "" { + return 0 + } + return export.LooseBlobWrites(m.attachmentsDir) +} + func (m *attachmentMaintenance) log() *slog.Logger { if m != nil && m.logger != nil { return m.logger @@ -274,6 +374,9 @@ func runWithAttachmentMutation( // runScheduledSource distinguishes attachment-producing provider/SyncTech // sources from calendar-only sources while preserving one shared wrapper. +// Packing never runs inline: a pack pass re-reads the whole pack catalog and +// excludes every ingest while it runs, so a scheduled sync only records that +// it wrote new loose blobs and the attachment-pack job packs them later. func runScheduledSource( ctx context.Context, maintenance *attachmentMaintenance, @@ -283,19 +386,37 @@ func runScheduledSource( if !attachmentProducing { return run(ctx) } - return runAfterSuccessfulAttachmentIngest(ctx, maintenance, run, nil) + before := maintenance.looseBlobWrites() + err := runWithAttachmentMutation(ctx, maintenance, run) + if maintenance != nil && maintenance.looseBlobWrites() != before { + maintenance.markPackPending() + } + return err } func registerAttachmentMaintenanceJob(sched *scheduler.Scheduler, maintenance *attachmentMaintenance) error { return sched.AddJob(scheduler.Job{ - Name: attachmentMaintenanceJob, - Schedule: attachmentMaintenanceCron, + Name: attachmentMaintenanceJob, + MaxRuntime: automaticAttachmentMaxRuntime, + Schedule: attachmentMaintenanceCron, Run: func(ctx context.Context) error { return maintenance.daily(ctx) }, }) } +func registerAttachmentPackJob(sched *scheduler.Scheduler, maintenance *attachmentMaintenance) error { + // Recheck the catalog on the first tick after startup. In-memory write + // counts cannot tell us what a previous daemon left loose. + maintenance.markPackPending() + return sched.AddJob(scheduler.Job{ + Name: attachmentPackJob, + MaxRuntime: automaticAttachmentMaxRuntime, + Schedule: attachmentPackCron, + Run: maintenance.runPendingPack, + }) +} + func registerScheduledBeeperJob( sched *scheduler.Scheduler, schedule string, @@ -305,8 +426,9 @@ func registerScheduledBeeperJob( // Every beeper store source (one per beeper AccountID) maps to this // singleton job name via api.SchedulerJobNameForSource. return sched.AddJob(scheduler.Job{ - Name: api.BeeperJobName, - Schedule: schedule, + Name: api.BeeperJobName, + Schedule: schedule, + Preemptible: true, Run: func(ctx context.Context) error { return runScheduledSource(ctx, maintenance, true, run) }, @@ -330,6 +452,7 @@ func attachmentProducingCommand(args []string) bool { "import-eml", "import-emlx", "import-gvoice", + "import-imazing-csv", "import-imessage", importMboxCommand, "import-messenger", diff --git a/cmd/msgvault/cmd/attachment_maintenance_test.go b/cmd/msgvault/cmd/attachment_maintenance_test.go index c774d4eee..712779ab9 100644 --- a/cmd/msgvault/cmd/attachment_maintenance_test.go +++ b/cmd/msgvault/cmd/attachment_maintenance_test.go @@ -11,6 +11,7 @@ import ( "io" "log/slog" "os" + "path" "path/filepath" "strings" "testing" @@ -18,9 +19,13 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.kenn.io/kit/pack" "go.kenn.io/kit/packstore" "go.kenn.io/msgvault/internal/attachmentstore" + "go.kenn.io/msgvault/internal/export" + "go.kenn.io/msgvault/internal/jobctx" + "go.kenn.io/msgvault/internal/mime" "go.kenn.io/msgvault/internal/scheduler" "go.kenn.io/msgvault/internal/store" "go.kenn.io/msgvault/internal/testutil/storetest" @@ -281,94 +286,187 @@ func TestAutomaticAttachmentMaintenanceWarningFailurePreservesIngestSuccess(t *t assert.Contains(logs.String(), warningErr.Error()) } -func TestRunScheduledSourcePacksOnlySuccessfulAttachmentSources(t *testing.T) { - tests := []struct { - name string - attachmentProducing bool - predecessorErr error - wantPacked bool - wantMaintenanceRuns int - }{ - { - name: "successful Gmail IMAP or Teams provider", - attachmentProducing: true, - wantPacked: true, - wantMaintenanceRuns: 1, - }, - { - name: "failed attachment provider", - attachmentProducing: true, - predecessorErr: errors.New("scheduled ingest failed"), - }, - { - name: "successful GCal source", - attachmentProducing: false, - }, - } +func (f *attachmentMaintenanceFixture) ingestLoose(content []byte) string { + f.t.Helper() + f.sequence++ + rel, err := export.StoreAttachmentFile(f.dir, &mime.Attachment{Content: content}) + require.NoError(f.t, err, "store ingested blob") + hash := path.Base(rel) + require.NoError(f.t, f.store.UpsertAttachment( + f.messageID, + fmt.Sprintf("ingested-%d.bin", f.sequence), + "application/octet-stream", + rel, + hash, + len(content), + ), "record ingested attachment") + return hash +} + +func TestRunScheduledSourceSkipsPackWithoutNewBlobs(t *testing.T) { + assert := assert.New(t) + f := newAttachmentMaintenanceFixture(t) + hash := f.addLoose([]byte("already loose before the sync")) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { + require.NoError(t, runScheduledSource(context.Background(), f.maintenance, true, + func(context.Context) error { return nil })) + + assert.Nil(f.packedEntry(hash), "a run that wrote no blobs must not pack") + assert.NotContains(f.logs.String(), "automatic attachment maintenance") + require.NoError(t, f.maintenance.runPendingPack(context.Background())) + assert.Nil(f.packedEntry(hash), "nothing was pending") +} + +func TestRunScheduledSourceDefersPackAfterNewBlobs(t *testing.T) { + assert := assert.New(t) + f := newAttachmentMaintenanceFixture(t) + var hash string + + require.NoError(t, runScheduledSource(context.Background(), f.maintenance, true, + func(context.Context) error { + hash = f.ingestLoose([]byte("new scheduled blob")) + return nil + })) + assert.Nil(f.packedEntry(hash), "scheduled sync must not pack inline") + assert.NotContains(f.logs.String(), "automatic attachment maintenance complete") + + require.NoError(t, f.maintenance.runPendingPack(context.Background())) + assert.NotNil(f.packedEntry(hash), "pending pack packs the new blob") + assert.Equal(1, strings.Count(f.logs.String(), "automatic attachment maintenance complete")) + assert.Contains(f.logs.String(), "duration=") + + require.NoError(t, f.maintenance.runPendingPack(context.Background())) + assert.Equal(1, strings.Count(f.logs.String(), "automatic attachment maintenance complete"), + "a second pending pass is a no-op") +} + +func TestPendingPackContinuesAfterByteBudget(t *testing.T) { + for _, daily := range []bool{false, true} { + t.Run(fmt.Sprintf("daily=%v", daily), func(t *testing.T) { require := require.New(t) assert := assert.New(t) f := newAttachmentMaintenanceFixture(t) - hash := f.addLoose([]byte("scheduled source payload")) - predecessorCalls := 0 - - err := runScheduledSource( - context.Background(), - f.maintenance, - tt.attachmentProducing, - func(context.Context) error { - predecessorCalls++ - assert.Nil(f.packedEntry(hash), "packing must happen only after predecessor success") - return tt.predecessorErr - }, - ) - - if tt.predecessorErr != nil { - require.ErrorIs(err, tt.predecessorErr) + // Exceed the real automatic budget with distinct, compressible blobs. + content := make([]byte, 8<<20) + var hashes []string + for i := range 33 { + content[0] = byte(i) + hashes = append(hashes, f.addLoose(content)) + } + if daily { + f.maintenance.markPackPending() + require.ErrorIs(f.maintenance.daily(context.Background()), scheduler.ErrReschedule, + "a bounded daily pass must request another pass for the remaining backlog") + } + sched := scheduler.New(nil).WithLogger(f.maintenance.logger) + t.Cleanup(func() { <-sched.Stop().Done() }) + jobName := attachmentPackJob + if daily { + jobName = attachmentMaintenanceJob + require.NoError(registerAttachmentMaintenanceJob(sched, f.maintenance)) } else { - require.NoError(err) + require.NoError(registerAttachmentPackJob(sched, f.maintenance)) + } + require.NoError(sched.TriggerJob(jobName)) + require.Eventually(func() bool { + statuses := sched.JobStatus() + return len(statuses) == 1 && !statuses[0].Running && !statuses[0].Queued && !statuses[0].Pending && !statuses[0].LastRun.IsZero() + }, time.Minute, 100*time.Millisecond, "bounded follow-ups drain packing and verification") + assert.Empty(sched.JobStatus()[0].LastError) + if daily { + assert.Contains(f.logs.String(), "automatic attachment repack complete", "the final daily follow-up must repack") + } + + for _, hash := range hashes { + assert.NotNil(f.packedEntry(hash), "the next pass must pack the remaining blobs") } - assert.Equal(1, predecessorCalls) - assert.Equal(tt.wantPacked, f.packedEntry(hash) != nil) - assert.Equal(tt.wantMaintenanceRuns, - strings.Count(f.logs.String(), "automatic attachment maintenance complete"), - "automatic attempts") }) } } -func TestRegisterScheduledBeeperJobPacksAfterSuccessfulSync(t *testing.T) { +func TestRegisterAttachmentPackJobFindsBacklogAfterRestart(t *testing.T) { + f := newAttachmentMaintenanceFixture(t) + hash := f.addLoose([]byte("blob left by the previous daemon")) + sched := scheduler.New(func(context.Context, string) error { return nil }).WithLogger(f.maintenance.logger) + t.Cleanup(func() { <-sched.Stop().Done() }) + require.NoError(t, registerAttachmentPackJob(sched, f.maintenance)) + require.NoError(t, sched.TriggerJob(attachmentPackJob)) + assert.NotNil(t, f.packedEntry(hash), "startup must rediscover loose blobs without a new sync") +} + +func TestRunScheduledSourceMarksPendingOnFailedIngestWithBlobs(t *testing.T) { + assert := assert.New(t) + f := newAttachmentMaintenanceFixture(t) + ingestErr := errors.New("scheduled ingest failed") + var hash string + + err := runScheduledSource(context.Background(), f.maintenance, true, + func(context.Context) error { + hash = f.ingestLoose([]byte("blob written before failure")) + return ingestErr + }) + require.ErrorIs(t, err, ingestErr) + require.NoError(t, f.maintenance.runPendingPack(context.Background())) + assert.NotNil(f.packedEntry(hash), "blobs from a failed run are still packed later") +} + +func TestRunScheduledSourceCalendarNeverPacks(t *testing.T) { + f := newAttachmentMaintenanceFixture(t) + var hash string + require.NoError(t, runScheduledSource(context.Background(), f.maintenance, false, + func(context.Context) error { + hash = f.ingestLoose([]byte("calendar-side blob")) + return nil + })) + require.NoError(t, f.maintenance.runPendingPack(context.Background())) + assert.Nil(t, f.packedEntry(hash), "non-attachment sources do not request packing") +} + +func TestDailyMaintenanceClearsPendingPack(t *testing.T) { + require := require.New(t) + f := newAttachmentMaintenanceFixture(t) + require.NoError(runScheduledSource(context.Background(), f.maintenance, true, + func(context.Context) error { + f.ingestLoose([]byte("blob packed by the daily job")) + return nil + })) + require.NoError(f.maintenance.daily(context.Background())) + before := strings.Count(f.logs.String(), "automatic attachment maintenance complete") + require.NoError(f.maintenance.runPendingPack(context.Background())) + assert.Equal(t, before, strings.Count(f.logs.String(), "automatic attachment maintenance complete"), + "daily pack satisfied the pending request") +} + +func TestRegisterScheduledBeeperJobDefersPackToPackJob(t *testing.T) { require := require.New(t) assert := assert.New(t) f := newAttachmentMaintenanceFixture(t) - hash := f.addLoose([]byte("scheduled beeper payload")) sched := scheduler.New(func(context.Context, string) error { return nil }).WithLogger(f.maintenance.logger) t.Cleanup(func() { ctx := sched.Stop() <-ctx.Done() }) - syncCalls := 0 + var hash string - require.NoError(registerScheduledBeeperJob( - sched, - "*/30 * * * *", - f.maintenance, + require.NoError(registerScheduledBeeperJob(sched, "*/30 * * * *", f.maintenance, func(context.Context) error { - syncCalls++ - assert.Nil(f.packedEntry(hash), "packing must happen after Beeper sync") + hash = f.ingestLoose([]byte("scheduled beeper payload")) return nil - }, - )) + })) + require.NoError(registerAttachmentPackJob(sched, f.maintenance)) require.True(sched.IsJobScheduled("beeper")) + require.True(sched.IsJobScheduled(attachmentPackJob)) require.NoError(sched.TriggerJob("beeper")) - assert.Equal(1, syncCalls) + assert.Nil(f.packedEntry(hash), "Beeper sync leaves packing to the pack job") + require.NoError(sched.TriggerJob(attachmentPackJob)) assert.NotNil(f.packedEntry(hash)) } func TestRegisterAttachmentMaintenanceJobAndTrigger(t *testing.T) { + logger := testLoggerValue() + _ = logger + require := require.New(t) assert := assert.New(t) f := newAttachmentMaintenanceFixture(t) @@ -404,7 +502,7 @@ func TestAttachmentMaintenanceDailyPacksThenRepacks(t *testing.T) { require.NoError(f.maintenance.daily(context.Background())) require.NotNil(f.packedEntry(liveHash)) assert.Equal(live, f.readBlob(liveHash)) - has, err := f.store.HasPackRecord(deadPackID) + has, err := f.store.HasPackRecordContext(t.Context(), deadPackID) require.NoError(err) assert.False(has) logs := f.logs.String() @@ -511,6 +609,7 @@ func TestAttachmentProducingCommandExactAllowlist(t *testing.T) { "import-eml", "import-emlx", "import-gvoice", + "import-imazing-csv", "import-imessage", importMboxCommand, "import-messenger", @@ -575,3 +674,82 @@ func TestAttachmentIngestMutationLeaseWaitsForMaintenance(t *testing.T) { } require.NoError(<-done) } + +func TestPendingPackCheckpointsVerificationAndStillPacksNewBlobs(t *testing.T) { + require := require.New(t) + f := newAttachmentMaintenanceFixture(t) + const existing = 140 + for i := range existing { + f.addLoose([]byte(fmt.Sprintf("existing packed content %d", i))) + } + _, err := f.maintenance.pack(t.Context(), 0) + require.NoError(err) + fresh := f.addLoose([]byte("new attachment while verification is pending")) + f.maintenance.markPackPending() + require.ErrorIs(f.maintenance.runPendingPack(t.Context()), scheduler.ErrReschedule) + require.NotNil(f.packedEntry(fresh), "new blobs must not wait for the whole verification cycle") + // A reconstructed daemon resumes the durable verification cursor. + resumed, err := newAttachmentMaintenance(f.store, f.dir, nil, true) + require.NoError(err) + defer func() { require.NoError(resumed.close()) }() + resumed.markPackPending() + require.NoError(resumed.runPendingPack(t.Context())) + stats, err := resumed.unpack(t.Context()) + require.NoError(err) + assert.Equal(t, existing+1, stats.BlobsRestored, "manual unpack must see the full catalog") +} + +func TestCompletedPackVerificationRecordsNoProgress(t *testing.T) { + f := newAttachmentMaintenanceFixture(t) + ctx := jobctx.WithProgress(t.Context()) + require.NoError(t, f.maintenance.runScheduledPack(ctx)) + assert.False(t, jobctx.HasProgress(ctx), "an empty completed scan provides no resume point for a cancelled repack") +} + +func TestDailyMaintenanceVerificationFollowupsReclaimDeadPacks(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := newAttachmentMaintenanceFixture(t) + deadPackID := f.makeZeroLivePack([]byte("dead pack awaiting daily collection")) + for i := range 140 { + f.addLoose([]byte(fmt.Sprintf("live packed content %d", i))) + } + _, err := f.maintenance.pack(t.Context(), 0) + require.NoError(err) + sched := scheduler.New(nil).WithLogger(f.maintenance.logger) + t.Cleanup(func() { <-sched.Stop().Done() }) + require.NoError(registerAttachmentMaintenanceJob(sched, f.maintenance)) + require.NoError(sched.TriggerJob(attachmentMaintenanceJob)) + require.Eventually(func() bool { + status := sched.JobStatus()[0] + return !status.Running && !status.Queued && !status.Pending && !status.LastRun.IsZero() + }, 10*time.Second, 10*time.Millisecond, "daily follow-ups must finish verification and repacking") + assert.Empty(sched.JobStatus()[0].LastError) + recorded, err := f.store.HasPackRecordContext(t.Context(), deadPackID) + require.NoError(err) + assert.False(recorded, "the final verification window must reach physical GC") + assert.Contains(f.logs.String(), "automatic attachment repack complete") +} + +func TestRemovingRedundantOrphanPackCountsAsProgress(t *testing.T) { + require := require.New(t) + f := newAttachmentMaintenanceFixture(t) + hash := f.addLoose([]byte("blob already served by a recorded pack")) + _, err := f.maintenance.pack(t.Context(), 0) + require.NoError(err) + entry := f.packedEntry(hash) + require.NotNil(entry) + recorded := filepath.Join(f.dir, "packs", entry.PackID[:2], entry.PackID+packstore.PackExt) + data, err := os.ReadFile(recorded) + require.NoError(err) + // An unrecorded copy, as left by a crash after a repack published its + // replacement: every live entry already resolves elsewhere, so Pack deletes it. + orphanID := pack.NewPackID() + orphan := filepath.Join(f.dir, "packs", orphanID[:2], orphanID+packstore.PackExt) + require.NoError(os.MkdirAll(filepath.Dir(orphan), 0o700)) + require.NoError(os.WriteFile(orphan, data, 0o600)) + ctx := jobctx.WithProgress(t.Context()) + require.NoError(f.maintenance.runAutomaticPack(ctx, nil)) + assert.NoFileExists(t, orphan) + assert.True(t, jobctx.HasProgress(ctx), "a removed pack is committed progress a timed-out pass can resume after") +} diff --git a/cmd/msgvault/cmd/attribute_definition_test.go b/cmd/msgvault/cmd/attribute_definition_test.go index 2b6402182..2b152201d 100644 --- a/cmd/msgvault/cmd/attribute_definition_test.go +++ b/cmd/msgvault/cmd/attribute_definition_test.go @@ -2,6 +2,7 @@ package cmd import ( "bytes" + "context" "encoding/json" "net/http" "net/http/httptest" @@ -41,7 +42,7 @@ const testAttributeDefinitionJSON = `{ }` func runAttributeCommand( - t *testing.T, template *cobra.Command, args ...string, + ctx context.Context, t *testing.T, template *cobra.Command, args ...string, ) (string, error) { t.Helper() var output bytes.Buffer @@ -56,7 +57,8 @@ func runAttributeCommand( command.SetOut(&output) command.SetErr(&output) command.SetArgs(args) - err := command.Execute() + command.SetContext(ctx) + err := command.ExecuteContext(ctx) return output.String(), err } @@ -73,11 +75,12 @@ func TestAttributeDefinitionListPrintsRegistryAndForwardsFilter(t *testing.T) { assert.NoError(err) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) + _ = testCtx - output, err := runAttributeCommand(t, attributeDefinitionListCmd, + output, err := runAttributeCommand(testCtx, t, attributeDefinitionListCmd, "--object-type", "person") require.NoError(err) assert.Contains(query, "object_type=person") @@ -95,11 +98,12 @@ func TestAttributeDefinitionCreateDryRunValidatesLocally(t *testing.T) { w.WriteHeader(http.StatusInternalServerError) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) + _ = testCtx - output, err := runAttributeCommand(t, attributeDefinitionCreateCmd, + output, err := runAttributeCommand(testCtx, t, attributeDefinitionCreateCmd, "--definition", `{"object_type":"person","slug":"scratch_note", "label":"Scratch note","value_type":"text","field_type":"text", "is_sensitive":true}`, @@ -112,7 +116,7 @@ func TestAttributeDefinitionCreateDryRunValidatesLocally(t *testing.T) { } func TestAttributeDefinitionCreateDryRunAllowsOmittedSlug(t *testing.T) { - output, err := runAttributeCommand(t, attributeDefinitionCreateCmd, + output, err := runAttributeCommand(t.Context(), t, attributeDefinitionCreateCmd, "--definition", `{"object_type":"person","label":"Favorite color", "value_type":"text","field_type":"text","cardinality":"single"}`, "--dry-run") @@ -149,7 +153,7 @@ func TestAttributeDefinitionCreateDryRunAppliesServerValidationLocally(t *testin } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - _, err := runAttributeCommand(t, attributeDefinitionCreateCmd, + _, err := runAttributeCommand(t.Context(), t, attributeDefinitionCreateCmd, "--definition", test.document, "--dry-run") require.Error(t, err, "dry run must reject what the server would reject") assert.Contains(t, err.Error(), test.wantErr) @@ -158,7 +162,7 @@ func TestAttributeDefinitionCreateDryRunAppliesServerValidationLocally(t *testin } func TestAttributeDefinitionCreateRejectsUnsupportedUniqueness(t *testing.T) { - _, err := runAttributeCommand(t, attributeDefinitionCreateCmd, + _, err := runAttributeCommand(t.Context(), t, attributeDefinitionCreateCmd, "--definition", `{"object_type":"person","slug":"employee_number", "label":"Employee number","value_type":"text","field_type":"text", "is_unique":true}`, @@ -183,11 +187,12 @@ func TestAttributeDefinitionRenameUsesFreshRevisionETag(t *testing.T) { assert.NoError(err) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) + _ = testCtx - _, err := runAttributeCommand(t, attributeDefinitionRenameCmd, + _, err := runAttributeCommand(testCtx, t, attributeDefinitionRenameCmd, "3", "--label", "Conversation starters") require.NoError(err) assert.Equal(`"attribute-definition-3-r7"`, patchIfMatch) @@ -208,11 +213,12 @@ func TestAttributeDefinitionClearDescriptionSendsEmptyString(t *testing.T) { assert.NoError(err) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) + _ = testCtx - _, err := runAttributeCommand(t, attributeDefinitionRenameCmd, + _, err := runAttributeCommand(testCtx, t, attributeDefinitionRenameCmd, "3", "--clear-description") require.NoError(err) assert.Equal(`""`, string(description)) diff --git a/cmd/msgvault/cmd/backfill_beeper_media.go b/cmd/msgvault/cmd/backfill_beeper_media.go index 79f4bdd38..4161c339a 100644 --- a/cmd/msgvault/cmd/backfill_beeper_media.go +++ b/cmd/msgvault/cmd/backfill_beeper_media.go @@ -29,11 +29,15 @@ Examples: msgvault backfill-beeper-media --account signal`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } if !isDaemonCLISubprocess() { return runDaemonCLICommandHTTPFromCobra(cmd, args) } - imp, accountIDs, dbPath, cleanup, err := openBeeperImporter(backfillBeeperMediaAccounts) + imp, accountIDs, dbPath, cleanup, err := openBeeperImporter(backfillBeeperMediaAccounts, state) if err != nil { return err } @@ -42,17 +46,17 @@ Examples: defer stop() for _, accountID := range accountIDs { - opts := beeperImportOptions(accountID) + opts := beeperImportOptions(accountID, state.cfg) opts.Progress = func(s string) { _, _ = fmt.Fprintln(cmd.OutOrStdout(), " "+s) } sum, err := imp.BackfillMedia(ctx, opts) if ctx.Err() != nil { _, _ = fmt.Fprintln(cmd.OutOrStdout(), "\nInterrupted — re-run backfill-beeper-media to resume (idempotent).") - return rebuildCacheAfterWrite(dbPath) + return rebuildCacheAfterWrite(dbPath, state) } if err != nil { return errors.Join( fmt.Errorf("beeper media backfill failed for %s: %w", accountID, err), - rebuildCacheAfterWrite(dbPath), + rebuildCacheAfterWrite(dbPath, state), ) } writeBeeperMediaBackfillSummary(cmd.OutOrStdout(), accountID, sum) @@ -61,7 +65,7 @@ Examples: } } - return rebuildCacheAfterWrite(dbPath) + return rebuildCacheAfterWrite(dbPath, state) }, } cmd.Flags().StringArrayVar(&backfillBeeperMediaAccounts, "account", nil, "Beeper accountID to backfill (repeatable; default: all registered accounts)") diff --git a/cmd/msgvault/cmd/backfill_discord_media.go b/cmd/msgvault/cmd/backfill_discord_media.go index 2c774db34..1962b92fa 100644 --- a/cmd/msgvault/cmd/backfill_discord_media.go +++ b/cmd/msgvault/cmd/backfill_discord_media.go @@ -69,6 +69,9 @@ With no guild argument, every registered guild is processed sequentially. Source URLs are treated as private provenance and are never printed.`, Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + if invocationFromContext(cmd.Context()) != nil && deps.bind != nil { + deps = deps.bind(cmd.Context()) + } selector := "" if len(args) == 1 { selector = args[0] diff --git a/cmd/msgvault/cmd/backfill_slack_media.go b/cmd/msgvault/cmd/backfill_slack_media.go index c593157c9..fd9ab2bee 100644 --- a/cmd/msgvault/cmd/backfill_slack_media.go +++ b/cmd/msgvault/cmd/backfill_slack_media.go @@ -26,6 +26,11 @@ Examples: msgvault backfill-slack-media T0123456789`, Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg if !isDaemonCLISubprocess() { return runDaemonCLICommandHTTPFromCobra(cmd, args) } @@ -34,7 +39,7 @@ Examples: if len(args) > 0 { flagTeam = args[0] } - s, cleanup, err := openWritableStoreAndInitForIngest() + s, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) if err != nil { return err } @@ -62,7 +67,7 @@ Examples: continue } imp := slack.NewImporter(s, slack.NewClient("", token), teamID) - sum, berr := imp.BackfillMedia(ctx, slackImportOptions(teamID, userID)) + sum, berr := imp.BackfillMedia(ctx, slackImportOptions(teamID, userID, state.cfg)) if ctx.Err() != nil { break } @@ -75,7 +80,7 @@ Examples: return slackMediaBackfillExit( ctx.Err(), runErrors, - rebuildCacheAfterWrite(cfg.DatabaseDSN()), + rebuildCacheAfterWrite(cfg.DatabaseDSN(), state), ) }, } diff --git a/cmd/msgvault/cmd/backfill_teams_media.go b/cmd/msgvault/cmd/backfill_teams_media.go index a37615def..672532eb6 100644 --- a/cmd/msgvault/cmd/backfill_teams_media.go +++ b/cmd/msgvault/cmd/backfill_teams_media.go @@ -1,17 +1,12 @@ package cmd import ( - "context" "errors" "fmt" "io" - "os" - "os/signal" - "syscall" "time" "github.com/spf13/cobra" - "go.kenn.io/msgvault/internal/microsoft" "go.kenn.io/msgvault/internal/teams" ) @@ -35,59 +30,36 @@ Examples: msgvault backfill-teams-media user@company.com --only-incomplete`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger if !isDaemonCLISubprocess() { return runDaemonCLICommandHTTPFromCobra(cmd, args) } email := args[0] - s, cleanup, err := openWritableStoreAndInitForIngest() + s, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) if err != nil { return err } defer cleanup() dbPath := cfg.DatabaseDSN() - if cfg.Microsoft.ClientID == "" { - return errors.New("microsoft OAuth not configured\n\n" + - "Add to your config.toml:\n\n" + - " [microsoft]\n" + - " client_id = \"your-azure-app-client-id\"\n\n" + - "See docs for Azure AD app registration setup") + if err := requireMicrosoftOAuthConfig(cfg); err != nil { + return err } - - mgr := microsoft.NewGraphManager( - cfg.Microsoft.ClientID, - cfg.Microsoft.EffectiveTenantID(), - cfg.Microsoft.EffectiveRedirectURI(), - cfg.TokensDir(), - logger, - ) - tokenFn, err := mgr.TokenSource(cmd.Context(), email) + client, err := newTeamsClient(cmd.Context(), cfg, logger, email) if err != nil { return fmt.Errorf("load Teams token: %w (run 'add-teams' first)", err) } - ctx, cancel := context.WithCancel(cmd.Context()) - defer cancel() - - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) - defer signal.Stop(sigChan) - go func() { - select { - case <-sigChan: - _, _ = fmt.Fprintln(cmd.ErrOrStderr(), "\nInterrupted. Stopping...") - cancel() - case <-ctx.Done(): - } - }() - - qps := float64(cfg.Sync.RateLimitQPS) - if qps <= 0 { - qps = 5 - } - client := teams.NewClient("https://graph.microsoft.com/v1.0", teams.TokenFunc(tokenFn), qps) + ctx, stop := withInterruptCancel(cmd, "\nInterrupted. Stopping...") + defer stop() + imp := teams.NewImporter(s, client) _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Backfilling Teams inline media for %s\n\n", email) @@ -101,7 +73,7 @@ Examples: }) if ctx.Err() != nil { _, _ = fmt.Fprintln(cmd.OutOrStdout(), "\nInterrupted — re-run backfill-teams-media to resume (idempotent).") - return rebuildCacheAfterWrite(dbPath) + return rebuildCacheAfterWrite(dbPath, state) } if err != nil { return fmt.Errorf("teams inline-media backfill failed: %w", err) @@ -109,7 +81,7 @@ Examples: writeTeamsMediaBackfillSummary(cmd.OutOrStdout(), sum) - return rebuildCacheAfterWrite(dbPath) + return rebuildCacheAfterWrite(dbPath, state) }, } diff --git a/cmd/msgvault/cmd/backup.go b/cmd/msgvault/cmd/backup.go index 373bda2f7..923067644 100644 --- a/cmd/msgvault/cmd/backup.go +++ b/cmd/msgvault/cmd/backup.go @@ -18,6 +18,7 @@ import ( "go.kenn.io/kit/packstore" "go.kenn.io/msgvault/internal/attachmentstore" "go.kenn.io/msgvault/internal/backupapp" + "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/daemonclient" "go.kenn.io/msgvault/internal/store" ) @@ -103,7 +104,7 @@ var backupRestoreCmd = &cobra.Command{ // resolveBackupRepo applies the standard --repo precedence for every backup // subcommand: an explicit flag wins, else the configured [backup] repo, // else an error naming both ways to set it. -func resolveBackupRepo(flagValue string) (string, error) { +func resolveBackupRepo(flagValue string, cfg *config.Config) (string, error) { if flagValue != "" { return flagValue, nil } @@ -114,7 +115,11 @@ func resolveBackupRepo(flagValue string) (string, error) { } func runBackupInit(cmd *cobra.Command, _ []string) error { - repo, err := resolveBackupRepo(backupInitRepo) + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + repo, err := resolveBackupRepo(backupInitRepo, state.cfg) if err != nil { return err } @@ -130,7 +135,11 @@ func runBackupInit(cmd *cobra.Command, _ []string) error { } func runBackupList(cmd *cobra.Command, _ []string) error { - repo, err := resolveBackupRepo(backupListRepo) + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + repo, err := resolveBackupRepo(backupListRepo, state.cfg) if err != nil { return err } @@ -177,7 +186,11 @@ func printBackupSnapshots(w io.Writer, snapshots []*backup.Manifest) error { } func runBackupVerify(cmd *cobra.Command, args []string) error { - repo, err := resolveBackupRepo(backupVerifyRepo) + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + repo, err := resolveBackupRepo(backupVerifyRepo, state.cfg) if err != nil { return err } @@ -223,7 +236,12 @@ func runBackupVerify(cmd *cobra.Command, args []string) error { // result. Like verify, it never proxies through the daemon: it reads only // the repository and writes only the target, never the live archive. func runBackupRestore(cmd *cobra.Command, args []string) error { - repo, err := resolveBackupRepo(backupRestoreRepo) + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + repo, err := resolveBackupRepo(backupRestoreRepo, cfg) if err != nil { return err } @@ -231,14 +249,14 @@ func runBackupRestore(cmd *cobra.Command, args []string) error { if err != nil { return fmt.Errorf("opening backup repository: %w", err) } - if err := refuseRestoreIntoLiveDaemonHome(backupRestoreTarget); err != nil { + if err := refuseRestoreIntoLiveDaemonHome(backupRestoreTarget, cfg); err != nil { return err } if backupRestoreAfterDaemonPreflight != nil { backupRestoreAfterDaemonPreflight() } targetCoordinator, coordinatedTarget, err := backupRestoreTargetCoordinator( - backupRestoreTarget, backupRestoreOverwrite, + backupRestoreTarget, backupRestoreOverwrite, cfg, ) if err != nil { return err @@ -309,6 +327,7 @@ func backupRestorePackedContentTarget(loose bool) backup.PackedContentTarget { func backupRestoreTargetCoordinator( target string, overwrite bool, + cfg *config.Config, ) (*daemonRestoreTargetCoordinator, bool, error) { if cfg == nil || target == "" || cfg.Data.DataDir == "" { return nil, false, nil @@ -712,8 +731,8 @@ func printBackupRestoreSummary(w io.Writer, target string, res *backup.RestoreRe // like any other directory. Target and home are compared as filesystem // objects, not path strings, so a case-variant or symlinked spelling of the // home is refused too. -func refuseRestoreIntoLiveDaemonHome(target string) error { - configuredHome, err := restoreTargetsConfiguredArchive(target) +func refuseRestoreIntoLiveDaemonHome(target string, cfg *config.Config) error { + configuredHome, err := restoreTargetsConfiguredArchive(target, cfg) if err != nil { return err } @@ -737,7 +756,7 @@ func refuseRestoreIntoLiveDaemonHome(target string) error { return nil } -func restoreTargetsConfiguredArchive(target string) (bool, error) { +func restoreTargetsConfiguredArchive(target string, cfg *config.Config) (bool, error) { if cfg == nil || target == "" || cfg.Data.DataDir == "" { return false, nil } @@ -867,7 +886,7 @@ func runBackupCreate(cmd *cobra.Command, args []string) error { // sensitive. The flag-named plaintext guard lives here so users see their // CLI flags in the error; the engine's own sensitive-source guard is the // backstop. -func backupExtrasSpec() (backup.ExtrasSpec, error) { +func backupExtrasSpec(cfg *config.Config) (backup.ExtrasSpec, error) { if (backupCreateIncludeConfig || backupCreateIncludeTokens) && !backupCreateAllowPlaintextSecrets { var flag string switch { @@ -897,7 +916,12 @@ func backupExtrasSpec() (backup.ExtrasSpec, error) { } func runBackupCreateLocal(cmd *cobra.Command) error { - repo, err := resolveBackupRepo(backupCreateRepo) + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + repo, err := resolveBackupRepo(backupCreateRepo, cfg) if err != nil { return err } @@ -930,7 +954,7 @@ func runBackupCreateLocal(cmd *cobra.Command) error { } defer func() { _ = blobs.Close() }() - freezer, closeFreezer, err := newBackupFreezer(cmd.Context()) + freezer, closeFreezer, err := newBackupFreezer(cmd.Context(), cfg) if err != nil { return err } @@ -948,7 +972,7 @@ func runBackupCreateLocal(cmd *cobra.Command) error { renderer := newBackupProgressRenderer(cmd.OutOrStdout(), mode) defer renderer.finish() - extras, err := backupExtrasSpec() + extras, err := backupExtrasSpec(cfg) if err != nil { return err } @@ -988,7 +1012,13 @@ func runBackupCreateLocal(cmd *cobra.Command) error { // freezeViaDaemon coordinator over it. backup create must never scan a // live-daemon-owned SQLite file unfrozen, so a daemon that cannot be // resolved here is a hard failure rather than a silent unfrozen fallback. -func newBackupFreezer(ctx context.Context) (backup.FreezeCoordinator, func(), error) { +func newBackupFreezer(ctx context.Context, cfg *config.Config) (backup.FreezeCoordinator, func(), error) { + if cfg == nil { + return nil, func() {}, errors.New("configuration is unavailable") + } + if err := cfg.ResolveServerKey(); err != nil { + return nil, func() {}, err + } rt := findDaemonRuntime(cfg.Data.DataDir) if rt == nil { return nil, func() {}, errors.New( @@ -999,7 +1029,7 @@ func newBackupFreezer(ctx context.Context) (backup.FreezeCoordinator, func(), er // cleanup request that releases an already-open freeze window. client, err := newDaemonCLIClient(context.WithoutCancel(ctx), daemonclient.Config{ URL: urlFromDaemonRuntime(rt), - APIKey: cfg.Server.APIKey, + APIKey: cfg.Server.AuthenticationKey(), AllowInsecure: true, }) if err != nil { diff --git a/cmd/msgvault/cmd/backup_daemon_test.go b/cmd/msgvault/cmd/backup_daemon_test.go index 9b27f9863..b921be588 100644 --- a/cmd/msgvault/cmd/backup_daemon_test.go +++ b/cmd/msgvault/cmd/backup_daemon_test.go @@ -38,14 +38,14 @@ func TestNewBackupFreezerUsesCommandContextAndCLIMode(t *testing.T) { t.Cleanup(srv.Close) dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) rt := daemonRuntimeForHTTPServer(t, srv, daemonAPIKeyFingerprint("")) _, err := daemonRuntimeStore(dataDir).Write(rt.Record) require.NoError(err, "write daemon runtime") - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(testCtx) defer cancel() - freezer, closeFreezer, err := newBackupFreezer(ctx) + freezer, closeFreezer, err := newBackupFreezer(ctx, invocationFromContext(testCtx).cfg) require.NoError(err, "newBackupFreezer") t.Cleanup(closeFreezer) @@ -102,20 +102,20 @@ func TestNewBackupFreezerEndsFreezeAfterCommandCancellation(t *testing.T) { t.Cleanup(srv.Close) dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) rt := daemonRuntimeForHTTPServer(t, srv, daemonAPIKeyFingerprint("")) _, err := daemonRuntimeStore(dataDir).Write(rt.Record) require.NoError(err, "write daemon runtime") - commandCtx, cancelCommand := context.WithCancel(context.Background()) - freezer, closeFreezer, err := newBackupFreezer(commandCtx) + commandCtx, cancelCommand := context.WithCancel(testCtx) + freezer, closeFreezer, err := newBackupFreezer(commandCtx, invocationFromContext(testCtx).cfg) require.NoError(err, "newBackupFreezer") t.Cleanup(closeFreezer) require.NoError(freezer.Begin(commandCtx), "begin freeze") cancelCommand() - cleanupCtx, cancelCleanup := context.WithTimeout(context.Background(), 2*time.Second) + cleanupCtx, cancelCleanup := context.WithTimeout(testCtx, 2*time.Second) defer cancelCleanup() require.NoError(freezer.End(cleanupCtx), "end freeze after command cancellation") assert.Equal(apiprotocol.ClientClassCLI, beginMarker.Load()) diff --git a/cmd/msgvault/cmd/backup_test.go b/cmd/msgvault/cmd/backup_test.go index 1dc74f9d3..3a8c16d88 100644 --- a/cmd/msgvault/cmd/backup_test.go +++ b/cmd/msgvault/cmd/backup_test.go @@ -44,6 +44,8 @@ func TestBackupRestorePackedTargetSelection(t *testing.T) { } func TestBackupRestoreTargetCoordinatorMatchesConfiguredDatabasePath(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) lockDir := t.TempDir() @@ -55,7 +57,9 @@ func TestBackupRestoreTargetCoordinatorMatchesConfiguredDatabasePath(t *testing. DataDir: lockDir, DatabaseURL: "file:" + filepath.Join(target, "msgvault.db"), }} - coordinator, coordinated, err := backupRestoreTargetCoordinator(target, true) + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx + coordinator, coordinated, err := backupRestoreTargetCoordinator(target, true, cfg) require.NoError(err, "select restore coordination by configured database path") require.True(coordinated, "restoring the configured database requires ownership coordination") require.NotNil(coordinator, "configured database target receives a coordinator") @@ -63,7 +67,7 @@ func TestBackupRestoreTargetCoordinatorMatchesConfiguredDatabasePath(t *testing. root, err := os.OpenRoot(target) require.NoError(err, "pin restore target") t.Cleanup(func() { require.NoError(root.Close(), "close restore target root") }) - lease, err := coordinator.AcquireRestoreTarget(context.Background(), root) + lease, err := coordinator.AcquireRestoreTarget(testCtx, root) require.NoError(err, "acquire configured database restore coordination") t.Cleanup(func() { require.NoError(lease.Release(), "release restore coordination") }) @@ -81,21 +85,28 @@ func TestBackupRestoreTargetCoordinatorMatchesConfiguredDatabasePath(t *testing. } func TestBackupRestoreTargetCoordinatorRejectsPrimaryDatabaseAsVectorBackend(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) target := t.TempDir() savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = &config.Config{Data: config.DataConfig{DataDir: target}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx + _ = testCtx cfg.Vector.DBPath = filepath.Join(target, "msgvault.db") - coordinator, coordinated, err := backupRestoreTargetCoordinator(target, true) + coordinator, coordinated, err := backupRestoreTargetCoordinator(target, true, cfg) require.ErrorContains(err, "vector database path resolves to the restored archive database") require.Nil(coordinator) require.False(coordinated) } func TestBackupRestoreTargetCoordinatorDefersMissingCaseVariantMatch(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) parent := t.TempDir() probe := filepath.Join(parent, "CaseProbe") @@ -114,8 +125,10 @@ func TestBackupRestoreTargetCoordinatorDefersMissingCaseVariantMatch(t *testing. savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = &config.Config{Data: config.DataConfig{DataDir: configuredDataDir}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx - coordinator, coordinated, err := backupRestoreTargetCoordinator(target, false) + coordinator, coordinated, err := backupRestoreTargetCoordinator(target, false, cfg) require.NoError(err, "select conditional restore coordination") require.True(coordinated, "missing case variants require comparison after Kit pins the target") @@ -126,7 +139,7 @@ func TestBackupRestoreTargetCoordinatorDefersMissingCaseVariantMatch(t *testing. root, err := os.OpenRoot(target) require.NoError(err, "pin folded restore target") t.Cleanup(func() { require.NoError(root.Close(), "close folded restore target root") }) - lease, err := coordinator.AcquireRestoreTarget(context.Background(), root) + lease, err := coordinator.AcquireRestoreTarget(testCtx, root) require.NoError(err, "acquire folded restore target coordination") t.Cleanup(func() { require.NoError(lease.Release(), "release folded restore coordination") }) @@ -140,6 +153,8 @@ func TestBackupRestoreTargetCoordinatorDefersMissingCaseVariantMatch(t *testing. } func TestRunBackupRestorePackedDefaultAndExplicitLooseCleanup(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) ctx := context.Background() @@ -231,6 +246,8 @@ func TestRunBackupRestorePackedDefaultAndExplicitLooseCleanup(t *testing.T) { backupRestoreIntegrityCheck = savedIntegrityCheck }) cfg = &config.Config{Data: config.DataConfig{DataDir: filepath.Join(t.TempDir(), "live")}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx backupRestoreRepo = repoPath backupRestoreOverwrite = false backupRestoreForceUnlock = false @@ -241,7 +258,7 @@ func TestRunBackupRestorePackedDefaultAndExplicitLooseCleanup(t *testing.T) { backupRestoreIntegrityCheck = false var packedOutput bytes.Buffer packedCmd := &cobra.Command{Use: "restore"} - packedCmd.SetContext(ctx) + packedCmd.SetContext(testCtx) packedCmd.SetOut(&packedOutput) require.NoError(runBackupRestore(packedCmd, nil)) assert.Contains(packedOutput.String(), "1 packed in 1 pack(s), 0 loose") @@ -259,7 +276,7 @@ func TestRunBackupRestorePackedDefaultAndExplicitLooseCleanup(t *testing.T) { backupRestoreIntegrityCheck = true var looseOutput bytes.Buffer looseCmd := &cobra.Command{Use: "restore"} - looseCmd.SetContext(ctx) + looseCmd.SetContext(testCtx) looseCmd.SetOut(&looseOutput) require.NoError(runBackupRestore(looseCmd, nil)) assert.Contains(looseOutput.String(), "Pack metadata cleared") @@ -270,6 +287,8 @@ func TestRunBackupRestorePackedDefaultAndExplicitLooseCleanup(t *testing.T) { } func TestRunBackupRestoreIntoNonexistentConfiguredDataDir(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) ctx := context.Background() @@ -310,6 +329,8 @@ func TestRunBackupRestoreIntoNonexistentConfiguredDataDir(t *testing.T) { }) target := filepath.Join(t.TempDir(), "fresh-archive") cfg = &config.Config{Data: config.DataConfig{DataDir: target}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx backupRestoreRepo = repoPath backupRestoreTarget = target backupRestoreOverwrite = false @@ -320,7 +341,7 @@ func TestRunBackupRestoreIntoNonexistentConfiguredDataDir(t *testing.T) { assert.NoDirExists(target, "restore target starts absent") cmd := &cobra.Command{Use: "restore"} - cmd.SetContext(ctx) + cmd.SetContext(testCtx) cmd.SetOut(io.Discard) require.NoError(runBackupRestore(cmd, nil), "restore into fresh configured archive home") assert.FileExists(filepath.Join(target, "msgvault.db"), "restored database") @@ -331,6 +352,8 @@ func TestRunBackupRestoreIntoNonexistentConfiguredDataDir(t *testing.T) { } func TestRunBackupRestoreRejectsDaemonClaimAfterPreflight(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) ctx := context.Background() @@ -376,6 +399,8 @@ func TestRunBackupRestoreRejectsDaemonClaimAfterPreflight(t *testing.T) { require.NoError(os.WriteFile(filepath.Join(target, "msgvault.db"), liveDatabase, 0o600), "seed live target database") cfg = &config.Config{Data: config.DataConfig{DataDir: target}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx backupRestoreRepo = repoPath backupRestoreTarget = target backupRestoreOverwrite = true @@ -395,7 +420,7 @@ func TestRunBackupRestoreRejectsDaemonClaimAfterPreflight(t *testing.T) { }) cmd := &cobra.Command{Use: "restore"} - cmd.SetContext(ctx) + cmd.SetContext(testCtx) cmd.SetOut(io.Discard) err = runBackupRestore(cmd, nil) require.ErrorContains(err, "daemon is already running", @@ -406,6 +431,8 @@ func TestRunBackupRestoreRejectsDaemonClaimAfterPreflight(t *testing.T) { } func TestDaemonRestoreTargetCoordinatorHoldsDaemonAndWriteLeasesThroughoutRestore(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) ctx := context.Background() @@ -439,8 +466,10 @@ func TestDaemonRestoreTargetCoordinatorHoldsDaemonAndWriteLeasesThroughoutRestor savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = &config.Config{Data: config.DataConfig{DataDir: target}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Vector.DBPath = customVectorPath - coordinator, coordinated, err := backupRestoreTargetCoordinator(target, true) + coordinator, coordinated, err := backupRestoreTargetCoordinator(target, true, cfg) require.NoError(err, "select configured restore coordination") require.True(coordinated, "configured archive restore requires ownership coordination") require.NotNil(coordinator, "configured archive restore receives a coordinator") @@ -486,6 +515,8 @@ func TestDaemonRestoreTargetCoordinatorHoldsDaemonAndWriteLeasesThroughoutRestor } func TestDaemonRestoreTargetCoordinatorPreservesVectorsWithoutPublication(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) target := t.TempDir() @@ -499,15 +530,17 @@ func TestDaemonRestoreTargetCoordinatorPreservesVectorsWithoutPublication(t *tes savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = &config.Config{Data: config.DataConfig{DataDir: target}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Vector.DBPath = customVectorPath - coordinator, coordinated, err := backupRestoreTargetCoordinator(target, true) + coordinator, coordinated, err := backupRestoreTargetCoordinator(target, true, cfg) require.NoError(err, "select configured restore coordination") require.True(coordinated, "configured archive restore requires ownership coordination") root, err := os.OpenRoot(target) require.NoError(err, "pin restore target") t.Cleanup(func() { require.NoError(root.Close(), "close restore target root") }) - lease, err := coordinator.AcquireRestoreTarget(t.Context(), root) + lease, err := coordinator.AcquireRestoreTarget(testCtx, root) require.NoError(err, "acquire restore coordination") require.NoError(lease.Release(), "release without publishing a restored database") @@ -543,6 +576,8 @@ func TestDaemonRestoreTargetLeaseRefusesToRemovePublishedDatabase(t *testing.T) } func TestDaemonRestoreTargetCoordinatorCanonicalizesMissingSymlinkedParent(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) ctx := context.Background() sourceDir := t.TempDir() @@ -576,7 +611,9 @@ func TestDaemonRestoreTargetCoordinatorCanonicalizesMissingSymlinkedParent(t *te savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = &config.Config{Data: config.DataConfig{DataDir: configuredDataDir}} - coordinator, coordinated, err := backupRestoreTargetCoordinator(target, false) + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx + coordinator, coordinated, err := backupRestoreTargetCoordinator(target, false, cfg) require.NoError(err, "select restore target coordination") require.True(coordinated, "resolved target must match an absent configured archive beneath a symlinked parent") @@ -616,7 +653,7 @@ func assertRestoredCLIBlob(t *testing.T, target, hash string, want []byte, packe restored, err := store.OpenForTest(filepath.Join(target, "msgvault.db")) require.NoError(err) defer func() { require.NoError(restored.Close()) }() - records, err := restored.ListPackRecords() + records, err := restored.ListPackRecordsContext(t.Context()) require.NoError(err) indexed, err := restored.ListIndexedBlobHashes() require.NoError(err) @@ -720,6 +757,8 @@ func TestPrintBackupRestoreSummaryReportsPackedMixedAndLooseLayouts(t *testing.T const restorePackAForOutput = "01hzy3v7q8r9s0t1a2v3w4x5y6" func TestResolveBackupRepoPrecedence(t *testing.T) { + cfg := testConfigValue() + savedCfg := cfg defer func() { cfg = savedCfg }() @@ -758,7 +797,7 @@ func TestResolveBackupRepoPrecedence(t *testing.T) { assert := assert.New(t) cfg = &config.Config{Backup: config.BackupConfig{Repo: tt.configRepo}} - repo, err := resolveBackupRepo(tt.flagValue) + repo, err := resolveBackupRepo(tt.flagValue, cfg) if tt.wantErr { require.Error(err) @@ -777,6 +816,8 @@ func TestResolveBackupRepoPrecedence(t *testing.T) { // the compatible-runtime lookup, yet it still owns the archive's SQLite // database, so restoring into its home must be refused all the same. func TestRefuseRestoreIntoLiveDaemonHomeBlocksIncompatibleDaemon(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) dataDir := t.TempDir() server := httptest.NewServer(daemon.NewPingHandler(daemon.PingHandlerOptions{ @@ -810,11 +851,13 @@ func TestRefuseRestoreIntoLiveDaemonHomeBlocksIncompatibleDaemon(t *testing.T) { savedCfg := cfg defer func() { cfg = savedCfg }() cfg = &config.Config{Data: config.DataConfig{DataDir: dataDir}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx - err = refuseRestoreIntoLiveDaemonHome(dataDir) + err = refuseRestoreIntoLiveDaemonHome(dataDir, cfg) require.ErrorContains(err, "running daemon", "restore into the live archive home must be refused even when the daemon is incompatible") - require.NoError(refuseRestoreIntoLiveDaemonHome(t.TempDir()), + require.NoError(refuseRestoreIntoLiveDaemonHome(t.TempDir(), cfg), "a target outside the archive home stays allowed") // The guard compares filesystem identity, not path strings, so an @@ -824,11 +867,13 @@ func TestRefuseRestoreIntoLiveDaemonHomeBlocksIncompatibleDaemon(t *testing.T) { if err := os.Symlink(dataDir, alias); err != nil { t.Skip("symlinks not supported on this platform") } - require.ErrorContains(refuseRestoreIntoLiveDaemonHome(alias), "running daemon", + require.ErrorContains(refuseRestoreIntoLiveDaemonHome(alias, cfg), "running daemon", "an aliased path to the archive home must be refused") } func TestRefuseRestoreIntoLiveDaemonHomeBlocksUnverifiableDaemonOwner(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) dataDir := t.TempDir() owner, err := tryAcquireDaemonOwnerLock(dataDir) @@ -850,13 +895,17 @@ func TestRefuseRestoreIntoLiveDaemonHomeBlocksUnverifiableDaemonOwner(t *testing savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = &config.Config{Data: config.DataConfig{DataDir: dataDir}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx - err = refuseRestoreIntoLiveDaemonHome(dataDir) + err = refuseRestoreIntoLiveDaemonHome(dataDir, cfg) require.ErrorContains(err, "running daemon", "held daemon ownership must block restore even when endpoint identity is unverifiable") } func TestRefuseRestoreIntoLiveDaemonHomeFailsClosedWhenLockCannotBeProbed(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) dataDir := t.TempDir() require.NoError(os.Mkdir(daemonOwnerLockPath(dataDir), 0o700), @@ -865,18 +914,22 @@ func TestRefuseRestoreIntoLiveDaemonHomeFailsClosedWhenLockCannotBeProbed(t *tes savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = &config.Config{Data: config.DataConfig{DataDir: dataDir}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx - err := refuseRestoreIntoLiveDaemonHome(dataDir) + err := refuseRestoreIntoLiveDaemonHome(dataDir, cfg) require.ErrorContains(err, "inspect daemon ownership", "restore must fail closed when daemon ownership cannot be determined") } func TestResolveBackupRepoNilConfig(t *testing.T) { + cfg := testConfigValue() + savedCfg := cfg defer func() { cfg = savedCfg }() cfg = nil - repo, err := resolveBackupRepo("/flag/repo") + repo, err := resolveBackupRepo("/flag/repo", cfg) require.NoError(t, err) assert.Equal(t, "/flag/repo", repo) diff --git a/cmd/msgvault/cmd/beeper_media_scheduler.go b/cmd/msgvault/cmd/beeper_media_scheduler.go new file mode 100644 index 000000000..25edb69f4 --- /dev/null +++ b/cmd/msgvault/cmd/beeper_media_scheduler.go @@ -0,0 +1,143 @@ +package cmd + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "log/slog" + "strings" + "time" + + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/attachmentstore" + "go.kenn.io/msgvault/internal/beeper" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/docbankmedia" + "go.kenn.io/msgvault/internal/scheduler" + "go.kenn.io/msgvault/internal/store" +) + +const ( + beeperMediaSubmitJob = "beeper-media-submit" + beeperMediaSubmitCron = "* * * * *" + beeperMediaGateLabel = "Stored media submission" +) + +// beeperMediaGateWait bounds each wait for the operation gate; a busy gate ends +// the pass. Variable only so tests can shorten it. +var beeperMediaGateWait = 30 * time.Second + +// beeperMediaGate takes the operation gate for one media Store step. +func beeperMediaGate(gate api.LabeledOperationGate) func(context.Context) (func(), bool) { + if gate == nil { + return nil + } + return func(ctx context.Context) (func(), bool) { + waitCtx, cancel := context.WithTimeout(ctx, beeperMediaGateWait) + defer cancel() + return gate.BeginLabeledWorkContext(waitCtx, beeperMediaGateLabel) + } +} + +// withBeeperMediaGate runs a setup Store write under the operation gate. +func withBeeperMediaGate(ctx context.Context, gate api.LabeledOperationGate, write func() error) error { + if gate == nil { + return write() + } + release, ok := beeperMediaGate(gate)(ctx) + if !ok { + return errors.New("operation gate unavailable for Beeper media setup") + } + defer release() + return write() +} + +func configureBeeperMediaJob( + ctx context.Context, + sched *scheduler.Scheduler, + gate api.LabeledOperationGate, + st *store.Store, + blobs *attachmentstore.Store, + spoolDir string, + cfg config.DocbankIntegrationConfig, + logger *slog.Logger, +) error { + if !cfg.Enabled { + return removeBeeperMediaRoute(ctx, sched, gate, st) + } + err := addBeeperMediaRoute(ctx, sched, gate, st, blobs, spoolDir, cfg, logger) + if err != nil { + // An idle registered consumer would hold attachment change log cleanup for every provider. + return errors.Join(err, removeBeeperMediaRoute(ctx, sched, gate, st)) + } + return nil +} + +// removeBeeperMediaRoute drops the job and its journal consumer; receipts stay. +func removeBeeperMediaRoute(ctx context.Context, sched *scheduler.Scheduler, gate api.LabeledOperationGate, st *store.Store) error { + sched.RemoveJob(beeperMediaSubmitJob) + err := withBeeperMediaGate(ctx, gate, func() error { + return st.UnregisterAttachmentChangeConsumer(ctx, store.BeeperMediaAttachmentConsumerKey) + }) + if errors.Is(err, store.ErrAttachmentChangeConsumerMissing) { + return nil + } + return err +} + +func addBeeperMediaRoute( + ctx context.Context, + sched *scheduler.Scheduler, + gate api.LabeledOperationGate, + st *store.Store, + blobs *attachmentstore.Store, + spoolDir string, + cfg config.DocbankIntegrationConfig, + logger *slog.Logger, +) error { + endpoint := strings.TrimRight(strings.TrimSpace(cfg.URL), "/") + lookupKey := cfg.ResolveAPIKey + client, err := docbankmedia.NewClient(endpoint, lookupKey) + if err != nil { + return err + } + archiveUID, err := st.ArchiveUIDContext(ctx) + if err != nil { + return err + } + destination := beeperMediaDestinationKey(endpoint, archiveUID) + // Without upload consent the job only records local discovery. + var submitClient *docbankmedia.Client + if cfg.AllSourcesUploadConsent { + submitClient = client + if err := withBeeperMediaGate(ctx, gate, func() error { + return st.ReconsiderBlockedBeeperMediaOperations(ctx, destination) + }); err != nil { + return err + } + } + submitter := beeper.NewMediaSubmitter(st, blobs, submitClient, destination, spoolDir). + WithASRProfile(cfg.ASRProfile).WithOperationGate(beeperMediaGate(gate)) + return sched.AddJob(scheduler.Job{ + Name: beeperMediaSubmitJob, + Schedule: beeperMediaSubmitCron, + Run: func(ctx context.Context) error { + result, err := submitter.RunBatch(ctx) + if err != nil { + return err + } + if logger != nil && (result.Examined > 0 || result.Journaled > 0) { + logger.Debug("Stored media submission pass", "examined", result.Examined, + "journaled", result.Journaled, "pending", result.Pending, + "retained", result.Retained, "blocked", result.Blocked) + } + return nil + }, + }) +} + +func beeperMediaDestinationKey(endpoint, archiveUID string) string { + digest := sha256.Sum256([]byte("beeper-media/v1\x00" + endpoint + "\x00" + archiveUID)) + return "beeper:" + hex.EncodeToString(digest[:]) +} diff --git a/cmd/msgvault/cmd/beeper_media_scheduler_test.go b/cmd/msgvault/cmd/beeper_media_scheduler_test.go new file mode 100644 index 000000000..7494f9a5c --- /dev/null +++ b/cmd/msgvault/cmd/beeper_media_scheduler_test.go @@ -0,0 +1,596 @@ +package cmd + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json/v2" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/attachmentpolicy" + "go.kenn.io/msgvault/internal/attachmentstore" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/scheduler" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil/storetest" +) + +const beeperMediaTestKeyEnv = "MSGVAULT_TEST_BEEPER_MEDIA_KEY" + +// storedBeeperVoiceNote writes the message, raw JSON, attachment row and +// loose CAS blob for one captured WAV voice note. +func storedBeeperVoiceNote(t *testing.T) (*store.Store, *attachmentstore.Store) { + t.Helper() + f := storetest.New(t) + _, err := f.Store.DB().Exec(f.Store.Rebind( + `UPDATE sources SET source_type = 'beeper', identifier = ? WHERE id = ?`), "signal", f.Source.ID) + require.NoError(t, err) + messageID := f.CreateMessage("voice1") + raw, err := json.Marshal(map[string]any{ + "id": "voice1", "timestamp": "2026-09-16T10:11:12.123Z", + "attachments": []map[string]any{{"id": "mxc://beeper.local/voice1", "type": "audio", + "isVoiceNote": true, "mimeType": "audio/wav", "fileName": "voice.wav"}}, + }) + require.NoError(t, err) + require.NoError(t, f.Store.UpsertMessageRawWithFormat(messageID, raw, "beeper_json")) + wav := testWAV() + digest := sha256.Sum256(wav) + hash := hex.EncodeToString(digest[:]) + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, hash[:2]), 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(dir, hash[:2], hash), wav, 0o600)) + require.NoError(t, f.Store.UpsertAttachmentRecord(t.Context(), messageID, store.AttachmentWrite{ + Filename: "voice.wav", MIMEType: "audio/wav", StoragePath: hash[:2] + "/" + hash, ContentHash: hash, + Size: int64(len(wav)), SourceAttachmentID: "beeper:mxc://beeper.local/voice1", + SourcePartKey: "beeper:mxc://beeper.local/voice1", MediaType: "voice_note", + State: attachmentpolicy.StateStored, Role: store.AttachmentRoleStandalone, + RoleSource: store.AttachmentRoleSourceImporterSemantics, + })) + blobs, err := attachmentstore.New(store.NewPackCatalog(f.Store), dir) + require.NoError(t, err) + t.Cleanup(func() { assert.NoError(t, blobs.Close()) }) + return f.Store, blobs +} + +// retentionServer accepts Docbank supplied-media retention requests. +type retentionServer struct { + mu sync.Mutex + requests int + operations []string + hang atomic.Bool + failStatus atomic.Int32 + arrived chan struct{} + release chan struct{} +} + +func newRetentionServer(t *testing.T) (*retentionServer, *httptest.Server) { + t.Helper() + server := &retentionServer{arrived: make(chan struct{}, 10), release: make(chan struct{})} + httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server.mu.Lock() + server.requests++ + server.mu.Unlock() + if status := server.failStatus.Load(); status != 0 { + http.Error(w, "secret response body", int(status)) + server.arrived <- struct{}{} + return + } + if !assert.Equal(t, "synthetic-key", r.Header.Get("X-Api-Key")) { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + reader, err := r.MultipartReader() + if !assert.NoError(t, err) { + return + } + part, err := reader.NextPart() + if !assert.NoError(t, err) { + return + } + var metadata struct { + OperationID string `json:"operation_id"` + } + assert.NoError(t, json.UnmarshalRead(part, &metadata)) + server.arrived <- struct{}{} + _, _ = io.Copy(io.Discard, r.Body) + if server.hang.Load() { + select { + case <-r.Context().Done(): + server.mu.Lock() + server.operations = append(server.operations, metadata.OperationID) + server.mu.Unlock() + return + case <-server.release: + } + } + server.mu.Lock() + server.operations = append(server.operations, metadata.OperationID) + server.mu.Unlock() + data, _ := json.Marshal(map[string]string{"vault_uid": "vault", "source_id": "source", + "source_version_id": "version", "content_version_id": "content", "occurrence_id": "occurrence", + "operation_id": metadata.OperationID, "operation_state": "succeeded", "coverage_state": "unprocessed"}) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(data) + })) + t.Cleanup(httpServer.Close) + return server, httpServer +} + +func (s *retentionServer) requestCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.requests +} + +func retentionRows(t *testing.T, st *store.Store) map[string]string { + t.Helper() + rows, err := st.DB().Query(`SELECT destination_key, retention_state || ':' || error_code || ':' || source_id + FROM beeper_media_occurrences`) + require.NoError(t, err) + defer func() { require.NoError(t, rows.Close()) }() + result := map[string]string{} + for rows.Next() { + var destination, value string + require.NoError(t, rows.Scan(&destination, &value)) + result[destination] = value + } + require.NoError(t, rows.Err()) + return result +} + +func consumerRegistered(t *testing.T, st *store.Store) bool { + t.Helper() + _, err := st.GetAttachmentChangeConsumer(t.Context(), store.BeeperMediaAttachmentConsumerKey) + return err == nil +} + +func TestBeeperMediaConfig(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st, blobs := storedBeeperVoiceNote(t) + server, httpServer := newRetentionServer(t) + archiveUID, err := st.ArchiveUIDContext(t.Context()) + require.NoError(err) + destination := beeperMediaDestinationKey(httpServer.URL, archiveUID) + sched := scheduler.New(nil) + defer func() { <-sched.Stop().Done() }() + + // Absent configuration registers nothing. + require.NoError(configureBeeperMediaJob(t.Context(), sched, nil, st, blobs, t.TempDir(), config.DocbankIntegrationConfig{}, nil)) + assert.False(sched.IsJobScheduled(beeperMediaSubmitJob)) + assert.False(consumerRegistered(t, st)) + + // Remote plaintext is refused before any job exists. + require.Error(configureBeeperMediaJob(t.Context(), sched, nil, st, blobs, t.TempDir(), config.DocbankIntegrationConfig{ + Enabled: true, URL: "http://docbank.example.com", APIKeyEnv: beeperMediaTestKeyEnv, AllSourcesUploadConsent: true}, nil)) + assert.False(sched.IsJobScheduled(beeperMediaSubmitJob)) + + // Without upload consent the job records local discovery only. + t.Setenv(beeperMediaTestKeyEnv, "synthetic-key") + require.NoError(configureBeeperMediaJob(t.Context(), sched, nil, st, blobs, t.TempDir(), config.DocbankIntegrationConfig{ + Enabled: true, URL: httpServer.URL, APIKeyEnv: beeperMediaTestKeyEnv}, nil)) + require.NoError(sched.TriggerJob(beeperMediaSubmitJob)) + assert.Equal(map[string]string{destination: "pending::"}, retentionRows(t, st)) + assert.Zero(server.requestCount()) + assert.True(consumerRegistered(t, st)) + + // A missing credential blocks the operation without scheduling a retry. + t.Setenv(beeperMediaTestKeyEnv, "") + require.NoError(configureBeeperMediaJob(t.Context(), sched, nil, st, blobs, t.TempDir(), config.DocbankIntegrationConfig{ + Enabled: true, URL: httpServer.URL, APIKeyEnv: beeperMediaTestKeyEnv, AllSourcesUploadConsent: true}, nil)) + require.NoError(sched.TriggerJob(beeperMediaSubmitJob)) + var state, code, operationID string + var scheduled bool + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT retention_state, error_code, retention_operation_id, next_action_at IS NOT NULL + FROM beeper_media_occurrences WHERE destination_key = ?`), destination). + Scan(&state, &code, &operationID, &scheduled)) + assert.Equal("blocked", state) + assert.Equal("credential_unavailable", code) + assert.False(scheduled) + assert.Zero(server.requestCount()) + blockedOperationID := operationID + + // Startup reconsideration reopens the same operation. A failing peer still retries. + t.Setenv(beeperMediaTestKeyEnv, "synthetic-key") + require.NoError(configureBeeperMediaJob(t.Context(), sched, nil, st, blobs, t.TempDir(), config.DocbankIntegrationConfig{ + Enabled: true, URL: httpServer.URL, APIKeyEnv: beeperMediaTestKeyEnv, AllSourcesUploadConsent: true}, nil)) + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT retention_state, error_code, retention_operation_id, next_action_at IS NOT NULL + FROM beeper_media_occurrences WHERE destination_key = ?`), destination). + Scan(&state, &code, &operationID, &scheduled)) + assert.Equal("pending", state) + assert.Equal("credential_unavailable", code) + assert.Equal(blockedOperationID, operationID) + assert.True(scheduled) + + server.failStatus.Store(http.StatusServiceUnavailable) + _, err = st.DB().Exec(`UPDATE beeper_media_occurrences SET next_action_at = '2000-01-01 00:00:00.000'`) + require.NoError(err) + require.NoError(sched.TriggerJob(beeperMediaSubmitJob)) + assert.Equal(map[string]string{destination: "pending:server_error:"}, retentionRows(t, st)) + var retryScheduled bool + require.NoError(st.DB().QueryRow(st.Rebind(` + SELECT next_action_at IS NOT NULL FROM beeper_media_occurrences WHERE destination_key = ?`), destination). + Scan(&retryScheduled)) + assert.True(retryScheduled) + assert.Equal(1, server.requestCount()) + + // Disabling removes the job and its journal consumer but keeps receipts. + require.NoError(configureBeeperMediaJob(t.Context(), sched, nil, st, blobs, t.TempDir(), config.DocbankIntegrationConfig{}, nil)) + assert.False(sched.IsJobScheduled(beeperMediaSubmitJob)) + assert.False(consumerRegistered(t, st)) + assert.Len(retentionRows(t, st), 1) +} + +// TestBeeperMediaInvalidConfigUnregisters keeps a failed reconfiguration from +// leaving an idle journal consumer that holds change log cleanup. +func TestBeeperMediaInvalidConfigUnregisters(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st, blobs := storedBeeperVoiceNote(t) + t.Setenv(beeperMediaTestKeyEnv, "synthetic-key") + _, httpServer := newRetentionServer(t) + sched := scheduler.New(nil) + defer func() { <-sched.Stop().Done() }() + cfg := config.DocbankIntegrationConfig{Enabled: true, URL: httpServer.URL, + APIKeyEnv: beeperMediaTestKeyEnv, AllSourcesUploadConsent: true} + require.NoError(configureBeeperMediaJob(t.Context(), sched, nil, st, blobs, t.TempDir(), cfg, nil)) + require.NoError(sched.TriggerJob(beeperMediaSubmitJob)) + require.True(consumerRegistered(t, st)) + require.Len(retentionRows(t, st), 1) + + // An invalid endpoint removes the job and consumer but keeps receipts. + invalid := cfg + invalid.URL = "http://docbank.example.com" + require.Error(configureBeeperMediaJob(t.Context(), sched, nil, st, blobs, t.TempDir(), invalid, nil)) + assert.False(sched.IsJobScheduled(beeperMediaSubmitJob)) + assert.False(consumerRegistered(t, st)) + assert.Len(retentionRows(t, st), 1) + + // A valid restart registers the consumer again. + require.NoError(configureBeeperMediaJob(t.Context(), sched, nil, st, blobs, t.TempDir(), cfg, nil)) + require.NoError(sched.TriggerJob(beeperMediaSubmitJob)) + assert.True(consumerRegistered(t, st)) +} + +func TestBeeperMediaScheduledRoute(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st, blobs := storedBeeperVoiceNote(t) + t.Setenv(beeperMediaTestKeyEnv, "synthetic-key") + server, httpServer := newRetentionServer(t) + gate := api.NewSerialOperationGate() + sched := scheduler.New(nil).WithWorkTracker(labelWorkTracker(gate, "media")) + defer func() { <-sched.Stop().Done() }() + cfg := config.DocbankIntegrationConfig{Enabled: true, URL: httpServer.URL, + APIKeyEnv: beeperMediaTestKeyEnv, AllSourcesUploadConsent: true} + require.NoError(configureBeeperMediaJob(t.Context(), sched, nil, st, blobs, t.TempDir(), cfg, nil)) + archiveUID, err := st.ArchiveUIDContext(t.Context()) + require.NoError(err) + destination := beeperMediaDestinationKey(httpServer.URL, archiveUID) + + // A waiting operation interrupts the upload; the saved request survives. + server.hang.Store(true) + done := make(chan error, 1) + go func() { done <- sched.TriggerJob(beeperMediaSubmitJob) }() + <-server.arrived + requestAcquired := make(chan func(), 1) + go func() { + release, ok := gate.BeginRequestWorkContext(t.Context(), "request") + if !ok { + release = nil + } + requestAcquired <- release + }() + select { + case err := <-done: + require.NoError(err) + case <-time.After(time.Minute): + require.FailNow("scheduled job did not yield") + } + releaseRequest := <-requestAcquired + require.NotNil(releaseRequest) + defer releaseRequest() + assert.Equal(map[string]string{destination: "pending::"}, retentionRows(t, st)) + + // The queued follow-up resumes with the same operation ID after the request. + server.hang.Store(false) + releaseRequest() + require.Eventually(func() bool { + status := sched.JobStatus()[0] + return !status.Running && !status.Queued && !status.Pending && !status.LastRun.IsZero() + }, time.Minute, 10*time.Millisecond) + assert.Equal(map[string]string{destination: "retained::source"}, retentionRows(t, st)) + server.mu.Lock() + require.Len(server.operations, 2) + assert.Equal(server.operations[0], server.operations[1]) + server.mu.Unlock() + + // A new destination starts its own delivery scope. + otherServer, otherHTTP := newRetentionServer(t) + cfg.URL = otherHTTP.URL + require.NoError(configureBeeperMediaJob(t.Context(), sched, nil, st, blobs, t.TempDir(), cfg, nil)) + otherDestination := beeperMediaDestinationKey(otherHTTP.URL, archiveUID) + otherServer.hang.Store(true) + stopped := make(chan error, 1) + go func() { stopped <- sched.TriggerJob(beeperMediaSubmitJob) }() + <-otherServer.arrived + select { + case <-sched.Stop().Done(): + case <-time.After(time.Minute): + require.FailNow("scheduler did not drain the running job") + } + <-stopped + assert.Equal(map[string]string{ + destination: "retained::source", otherDestination: "pending::", + }, retentionRows(t, st)) + require.Error(sched.TriggerJob(beeperMediaSubmitJob)) +} + +func TestStoredMediaSchedulerUsesOtherSourceProfile(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st, blobs := storedBeeperVoiceNote(t) + _, err := st.DB().Exec(`UPDATE attachments SET attachment_role = 'preview' + WHERE message_id = (SELECT id FROM messages WHERE source_message_id = 'voice1')`) + require.NoError(err) + source, err := st.GetOrCreateSource("gmail", "rod@example.com") + require.NoError(err) + conversation, err := st.EnsureConversation(source.ID, "mail-thread-1", "Mail") + require.NoError(err) + messageID, err := st.UpsertMessage(&store.Message{ + ConversationID: conversation, SourceID: source.ID, SourceMessageID: "mail-audio-1", + MessageType: "gmail", SizeEstimate: int64(len(testWAV())), + }) + require.NoError(err) + wav := testWAV() + digest := sha256.Sum256(wav) + hash := hex.EncodeToString(digest[:]) + require.NoError(st.UpsertAttachmentRecord(t.Context(), messageID, store.AttachmentWrite{ + Filename: "meeting.wav", MIMEType: "application/octet-stream", StoragePath: hash[:2] + "/" + hash, + ContentHash: hash, Size: int64(len(wav)), SourceAttachmentID: "mail:attachment:1", + SourcePartKey: "mime:1.2", State: attachmentpolicy.StateStored, + Role: store.AttachmentRoleStandalone, RoleSource: store.AttachmentRoleSourceImporterSemantics, + })) + t.Setenv(beeperMediaTestKeyEnv, "synthetic-key") + _, httpServer := newRetentionServer(t) + sched := scheduler.New(nil) + cfg := config.DocbankIntegrationConfig{Enabled: true, URL: httpServer.URL, + APIKeyEnv: beeperMediaTestKeyEnv, AllSourcesUploadConsent: true, ASRProfile: "asr"} + require.NoError(configureBeeperMediaJob(t.Context(), sched, nil, st, blobs, t.TempDir(), cfg, nil)) + require.NoError(sched.TriggerJob(beeperMediaSubmitJob)) + archiveUID, err := st.ArchiveUIDContext(t.Context()) + require.NoError(err) + destination := beeperMediaDestinationKey(httpServer.URL, archiveUID) + var provider, profile string + require.NoError(st.DB().QueryRow(st.Rebind(`SELECT provider, profile FROM beeper_media_deliveries + WHERE destination_key = ?`), destination).Scan(&provider, &profile)) + assert.Equal("gmail", provider) + assert.Equal("asr", profile) +} + +func TestStoredMediaRequiresAllSourcesConsent(t *testing.T) { + for _, provider := range []string{"beeper", "gmail"} { + t.Run(provider, func(t *testing.T) { + require, assert := require.New(t), assert.New(t) + st, blobs := storedBeeperVoiceNote(t) + _, err := st.DB().Exec(st.Rebind(`UPDATE sources SET source_type = ?`), provider) + require.NoError(err) + t.Setenv("MSGVAULT_HOME", t.TempDir()) + t.Setenv(beeperMediaTestKeyEnv, "synthetic-key") + server, httpServer := newRetentionServer(t) + sched := scheduler.New(nil) + t.Cleanup(func() { <-sched.Stop().Done() }) + path := filepath.Join(t.TempDir(), "config.toml") + content := fmt.Sprintf(`[integrations.docbank] +enabled = true +url = %q +api_key_env = %q +upload_consent = true +`, httpServer.URL, beeperMediaTestKeyEnv) + require.NoError(os.WriteFile(path, []byte(content), 0o600)) + cfg, err := config.Load(path, "") + require.NoError(err) + require.NoError(configureBeeperMediaJob(t.Context(), sched, nil, st, blobs, t.TempDir(), cfg.Integrations.Docbank, nil)) + require.NoError(sched.TriggerJob(beeperMediaSubmitJob)) + assert.Zero(server.requestCount(), "existing consent must keep all audio local") + + content += "all_sources_upload_consent = true\n" + require.NoError(os.WriteFile(path, []byte(content), 0o600)) + cfg, err = config.Load(path, "") + require.NoError(err) + require.NoError(configureBeeperMediaJob(t.Context(), sched, nil, st, blobs, t.TempDir(), cfg.Integrations.Docbank, nil)) + require.NoError(sched.TriggerJob(beeperMediaSubmitJob)) + assert.Equal(1, server.requestCount(), "new consent must allow the stored audio upload") + archiveUID, err := st.ArchiveUIDContext(t.Context()) + require.NoError(err) + assert.Equal(map[string]string{beeperMediaDestinationKey(httpServer.URL, archiveUID): "retained::source"}, retentionRows(t, st)) + }) + } +} + +// TestBeeperMediaGatedStoreWrites composes the daemon schedulers. A long +// upload holds no operation gate, while every media Store write waits for it, +// so a backup freeze sees no writer. +func TestBeeperMediaGatedStoreWrites(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st, blobs := storedBeeperVoiceNote(t) + t.Setenv(beeperMediaTestKeyEnv, "synthetic-key") + wait := beeperMediaGateWait + beeperMediaGateWait = 100 * time.Millisecond + t.Cleanup(func() { beeperMediaGateWait = wait }) + server, httpServer := newRetentionServer(t) + gate := api.NewSerialOperationGate() + logger := slog.New(slog.DiscardHandler) + sched, media := newServeSchedulers(nil, logger, nil, gate) + require.NoError(sched.AddJob(scheduler.Job{Name: "test-gated-job", Schedule: "0 0 1 1 *", + Run: func(context.Context) error { return nil }})) + cfg := config.DocbankIntegrationConfig{Enabled: true, URL: httpServer.URL, + APIKeyEnv: beeperMediaTestKeyEnv, AllSourcesUploadConsent: true} + require.NoError(configureBeeperMediaJob(t.Context(), media, gate, st, blobs, t.TempDir(), cfg, logger)) + archiveUID, err := st.ArchiveUIDContext(t.Context()) + require.NoError(err) + destination := beeperMediaDestinationKey(httpServer.URL, archiveUID) + waitCtx, cancel := context.WithTimeout(t.Context(), time.Minute) + defer cancel() + + // While a backup freeze holds the gate, the pass ends before any write. + freeze, ok := gate.BeginLabeledWorkContext(waitCtx, "backup freeze") + require.True(ok) + require.NoError(media.TriggerJob(beeperMediaSubmitJob)) + assert.Empty(retentionRows(t, st)) + assert.False(consumerRegistered(t, st)) + assert.Zero(server.requestCount()) + freeze() + + // An upload in flight holds no gate, so the freeze starts at once. + server.hang.Store(true) + done := make(chan error, 1) + go func() { done <- media.TriggerJob(beeperMediaSubmitJob) }() + <-server.arrived + _, _, held := gate.Holder() + assert.False(held, "an upload in flight holds no operation gate") + assert.Equal(map[string]string{destination: "pending::"}, retentionRows(t, st)) + freeze, ok = gate.BeginLabeledWorkContext(waitCtx, "backup freeze") + require.True(ok) + + // The finished upload can't record its receipt under the freeze. + close(server.release) + select { + case err := <-done: + require.NoError(err) + case <-waitCtx.Done(): + require.FailNow("upload pass did not end while the gate was held") + } + assert.Equal(map[string]string{destination: "pending::"}, retentionRows(t, st)) + freeze() + + // The next pass replays the saved operation ID and records the receipt. + require.NoError(media.TriggerJob(beeperMediaSubmitJob)) + assert.Equal(map[string]string{destination: "retained::source"}, retentionRows(t, st)) + server.mu.Lock() + require.Len(server.operations, 2) + assert.Equal(server.operations[0], server.operations[1]) + server.mu.Unlock() + + // Daemon shutdown cancels an upload in flight and drains both schedulers. + otherServer, otherHTTP := newRetentionServer(t) + cfg.URL = otherHTTP.URL + require.NoError(configureBeeperMediaJob(t.Context(), media, gate, st, blobs, t.TempDir(), cfg, logger)) + otherServer.hang.Store(true) + stopped := make(chan error, 1) + go func() { stopped <- media.TriggerJob(beeperMediaSubmitJob) }() + <-otherServer.arrived + require.NoError(shutdownServeRuntime(waitCtx, io.Discard, nil, serveSchedulers{sched, media}, gate)) + shutdownWait, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + select { + case err := <-stopped: + require.ErrorIs(err, context.Canceled) + case <-shutdownWait.Done(): + require.FailNow("shutdown returned before the upload stopped") + } + assert.Equal(map[string]string{ + destination: "retained::source", beeperMediaDestinationKey(otherHTTP.URL, archiveUID): "pending::", + }, retentionRows(t, st)) + assert.True(gate.Draining()) + require.Error(media.TriggerJob(beeperMediaSubmitJob)) + require.Error(sched.TriggerJob("test-gated-job")) +} + +// TestBeeperMediaJobStatus shows the API's scheduler adapter lists and runs +// the media job alongside the gated daemon jobs. +func TestBeeperMediaJobStatus(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st, blobs := storedBeeperVoiceNote(t) + t.Setenv(beeperMediaTestKeyEnv, "synthetic-key") + _, httpServer := newRetentionServer(t) + gate := api.NewSerialOperationGate() + logger := slog.New(slog.DiscardHandler) + sched, media := newServeSchedulers(nil, logger, nil, gate) + defer func() { <-serveSchedulers{sched, media}.Stop().Done() }() + require.NoError(sched.AddJob(scheduler.Job{Name: "test-gated-job", Schedule: "0 0 1 1 *", + Run: func(context.Context) error { return nil }})) + require.NoError(configureBeeperMediaJob(t.Context(), media, gate, st, blobs, t.TempDir(), config.DocbankIntegrationConfig{ + Enabled: true, URL: httpServer.URL, APIKeyEnv: beeperMediaTestKeyEnv}, logger)) + var adapter api.SyncScheduler = &schedulerAdapter{scheduler: sched, media: media} + + assert.True(adapter.IsJobScheduled(beeperMediaSubmitJob)) + assert.True(adapter.IsJobScheduled("test-gated-job")) + require.NoError(adapter.TriggerJob(beeperMediaSubmitJob)) + jobs := map[string]api.JobStatus{} + for _, job := range adapter.JobStatus() { + jobs[job.Name] = job + } + require.Contains(jobs, beeperMediaSubmitJob) + require.Contains(jobs, "test-gated-job") + assert.Equal(beeperMediaSubmitCron, jobs[beeperMediaSubmitJob].Schedule) + assert.False(jobs[beeperMediaSubmitJob].LastRun.IsZero()) + assert.Empty(jobs[beeperMediaSubmitJob].LastError) + archiveUID, err := st.ArchiveUIDContext(t.Context()) + require.NoError(err) + assert.Equal(map[string]string{beeperMediaDestinationKey(httpServer.URL, archiveUID): "pending::"}, + retentionRows(t, st)) +} + +func TestBeeperMediaDoesNotPreventIdleShutdown(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + st, blobs := storedBeeperVoiceNote(t) + idleShutdown := make(chan struct{}) + idle := api.NewIdleTracker(3*time.Minute, func() { close(idleShutdown) }) + logger := slog.New(slog.DiscardHandler) + gate := api.NewSerialOperationGate() + sched, media := newServeSchedulers(nil, logger, idle, gate) + defer func() { <-serveSchedulers{sched, media}.Stop().Done() }() + require.NoError(t, configureBeeperMediaJob(t.Context(), media, gate, st, blobs, t.TempDir(), + config.DocbankIntegrationConfig{Enabled: true, URL: "http://127.0.0.1"}, logger)) + go idle.Run(t.Context()) + media.Start() + + // Repeated discovery passes must leave an unused daemon free to stop. + synctest.Sleep(3*time.Minute + time.Second) + require.Len(t, retentionRows(t, st), 1, "scheduled discovery must have run") + select { + case <-idleShutdown: + default: + assert.Fail(t, "background media passes prevented idle shutdown") + } + }) +} + +func testWAV() []byte { + audio := make([]byte, 1600) + data := make([]byte, 44+len(audio)) + copy(data[0:4], "RIFF") + binary.LittleEndian.PutUint32(data[4:8], uint32(len(data)-8)) + copy(data[8:16], "WAVEfmt ") + binary.LittleEndian.PutUint32(data[16:20], 16) + binary.LittleEndian.PutUint16(data[20:22], 1) + binary.LittleEndian.PutUint16(data[22:24], 1) + binary.LittleEndian.PutUint32(data[24:28], 8000) + binary.LittleEndian.PutUint32(data[28:32], 16000) + binary.LittleEndian.PutUint16(data[32:34], 2) + binary.LittleEndian.PutUint16(data[34:36], 16) + copy(data[36:40], "data") + binary.LittleEndian.PutUint32(data[40:44], uint32(len(audio))) + return data +} diff --git a/cmd/msgvault/cmd/build_cache.go b/cmd/msgvault/cmd/build_cache.go index a3eec0974..9f78541ce 100644 --- a/cmd/msgvault/cmd/build_cache.go +++ b/cmd/msgvault/cmd/build_cache.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "io" + "log/slog" "os" "os/exec" "path/filepath" @@ -17,10 +18,12 @@ import ( "strings" "sync" "time" + "unicode/utf8" "github.com/gofrs/flock" _ "github.com/mattn/go-sqlite3" // SQLite driver (database/sql) "github.com/spf13/cobra" + "go.kenn.io/kit/atomicfile" "go.kenn.io/msgvault/internal/api" "go.kenn.io/msgvault/internal/cacheops" "go.kenn.io/msgvault/internal/config" @@ -29,6 +32,7 @@ import ( "go.kenn.io/msgvault/internal/query" "go.kenn.io/msgvault/internal/sqliteutil" "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/textutil" ) var fullRebuild bool @@ -49,11 +53,46 @@ const ( buildCacheModeScheduledAuto ) +// contextMutex serializes cache work while letting callers stop waiting when +// their context is canceled. Lock preserves the context-free behavior for +// synchronous CLI paths; daemon paths use LockContext. +type contextMutex struct { + locked chan struct{} +} + +func newContextMutex() *contextMutex { + return &contextMutex{locked: make(chan struct{}, 1)} +} + +func (m *contextMutex) Lock() { + _ = m.LockContext(context.Background()) +} + +func (m *contextMutex) LockContext(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + select { + case m.locked <- struct{}{}: + if err := ctx.Err(); err != nil { + <-m.locked + return err + } + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (m *contextMutex) Unlock() { + <-m.locked +} + // buildCacheMu serializes concurrent buildCache calls. The scheduler may // trigger syncs for multiple accounts in parallel, each of which calls // buildCache on completion. Without this lock, concurrent writes to shared // files (_last_sync.json, parquet directories) can corrupt the cache. -var buildCacheMu sync.Mutex +var buildCacheMu = newContextMutex() // cacheBuildFileLock returns the reader-coordination lock (see // query.CacheBuildLockPath): DuckDB readers hold it shared per query, and @@ -117,7 +156,7 @@ func invalidateSyncStateFile(stateFile string) error { // self-deadlock on a second file descriptor). A destructive mutation must // not proceed when any protection step fails. func lockCacheAndInvalidateSyncState(analyticsDir string) (func() error, error) { - builderLock, err := acquireCacheBuildLock(analyticsDir) + builderLock, err := acquireCacheBuildLock(context.Background(), analyticsDir) if err != nil { return nil, fmt.Errorf("serialize against cache builders: %w", err) } @@ -173,9 +212,16 @@ var buildCacheBeforeMessagesExportHook func() error // Tests override it to exercise platform-specific snapshot behavior. var cacheSnapshotGOOS = runtime.GOOS -// buildCacheWriteStateFile persists the cache sync state; a test seam for -// simulating state persistence failures. -var buildCacheWriteStateFile = os.WriteFile +// buildCacheWriteStateFile atomically publishes the cache sync state marker; +// a test seam for simulating state persistence failures. +var buildCacheWriteStateFile = writeCacheStateFile + +func writeCacheStateFile(path string, data []byte, perm os.FileMode) error { + if err := atomicfile.WriteFile(path, data, atomicfile.WithPerm(perm)); err != nil { + return fmt.Errorf("write cache state: %w", err) + } + return nil +} // cacheSchemaVersion tracks the Parquet schema layout. Bump this whenever // columns are added/removed/renamed in the COPY queries below so that @@ -234,8 +280,22 @@ type sqlRunner interface { } func readCacheSyncCounters(db sqlRowQuerier) (cacheSyncCounters, error) { + return readCacheSyncCountersWithRow(db.QueryRow) +} + +type contextSQLRowQuerier interface { + QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row +} + +func readCacheSyncCountersContext(ctx context.Context, db contextSQLRowQuerier) (cacheSyncCounters, error) { + return readCacheSyncCountersWithRow(func(query string, args ...any) *sql.Row { + return db.QueryRowContext(ctx, query, args...) + }) +} + +func readCacheSyncCountersWithRow(queryRow func(string, ...any) *sql.Row) (cacheSyncCounters, error) { var counters cacheSyncCounters - err := db.QueryRow(` + err := queryRow(` SELECT COALESCE(SUM(COALESCE(sr.messages_added, 0)), 0), COALESCE(SUM(COALESCE(sr.messages_updated, 0)), 0), @@ -276,6 +336,10 @@ The cache files are stored in ~/.msgvault/analytics/: By default, this performs an incremental update (only adding new messages). Use --full-rebuild to recreate all cache files from scratch.`, RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } mode, err := requestedBuildCacheMode( fullRebuild, buildCacheAutoFlag, @@ -286,7 +350,7 @@ By default, this performs an incremental update (only adding new messages). return err } if isDaemonBuildCacheChild() { - return runBuildCacheLocalMode(mode) + return runBuildCacheLocalMode(mode, state) } if mode == buildCacheModeDerived || mode == buildCacheModeAuto || mode == buildCacheModeScheduledAuto { return errors.New("--auto, --scheduled-auto, and --derived-only are internal daemon-child modes") @@ -371,15 +435,20 @@ func runBuildCacheHTTP(cmd *cobra.Command, fullRebuild bool) error { }) } -func runBuildCacheLocal(fullRebuild, auto bool) error { +func runBuildCacheLocal(fullRebuild, auto bool, state *invocation) error { mode, err := requestedBuildCacheMode(fullRebuild, auto, false, false) if err != nil { return err } - return runBuildCacheLocalMode(mode) + return runBuildCacheLocalMode(mode, state) } -func runBuildCacheLocalMode(mode buildCacheMode) error { +func runBuildCacheLocalMode(mode buildCacheMode, state *invocation) error { + state = invocationState(context.Background(), state) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg dbDSN := cfg.DatabaseDSN() analyticsDir := cfg.AnalyticsDir() builderOverrides := analyticsBuilderOverrides(cfg.Analytics) @@ -399,7 +468,7 @@ func runBuildCacheLocalMode(mode buildCacheMode) error { return fmt.Errorf("database not found: %s\nRun 'msgvault init-db' first", dbPath) } - release, err := acquireBuildCacheWriteLock(cfg) + release, err := acquireBuildCacheWriteLock(cfg, state) if err != nil { return err } @@ -487,7 +556,7 @@ func buildCacheDerivedOnly( buildCacheMu.Lock() defer buildCacheMu.Unlock() - buildLock, err := acquireCacheBuildLock(analyticsDir) + buildLock, err := acquireCacheBuildLock(context.Background(), analyticsDir) if err != nil { return nil, err } @@ -497,15 +566,16 @@ func buildCacheDerivedOnly( dbPath, analyticsDir, acquirePublishLock, + false, builderOverrides..., ) } -func acquireBuildCacheWriteLock(cfg *config.Config) (func(), error) { +func acquireBuildCacheWriteLock(cfg *config.Config, state *invocation) (func(), error) { if isDaemonBuildCacheChild() { return func() {}, nil } - return acquireDirectSQLiteWriteLock(cfg) + return acquireDirectSQLiteWriteLock(cfg, state) } func isDaemonBuildCacheChild() bool { @@ -514,6 +584,7 @@ func isDaemonBuildCacheChild() bool { type buildResult struct { ExportedCount int64 + StagedCount int64 MaxMessageID int64 OutputDir string Skipped bool @@ -529,108 +600,165 @@ type buildResult struct { // Without the primary-email guard a message could be cached as inbound while // its sender is globally marked the owner, and relationship/explore // materialization would drop the actual correspondent. -const ownerParticipantsSelectSQL = ` +func ownerParticipantsSelectSQL(primaryEmailPresence string) string { + // Identity keys must be collision-free: DuckDB's SQLite scanner can pass + // stored invalid bytes through unchecked, so cacheIdentityTextSQL turns + // them into NULL before TRIM, lower, or equality sees the stored bytes. + // NULL never matches, so distinct invalid sequences cannot collapse onto + // one U+FFFD key and mis-attribute messages. + accountIdentities := "(SELECT source_id, " + cacheIdentityTextSQL("address") + " AS address FROM sqlite_db.account_identities) ai" + participants := "(SELECT id, " + cacheIdentityTextSQL("email_address") + " AS email_address, " + + primaryEmailPresence + " AS primary_email_present FROM sqlite_db.participants)" + identifiers := "(SELECT participant_id, " + cacheIdentityTextSQL("identifier_type") + " AS identifier_type, " + + cacheIdentityTextSQL("identifier_value") + " AS identifier_value FROM sqlite_db.participant_identifiers) pi" + return fmt.Sprintf(` SELECT DISTINCT ai.source_id, p.id AS participant_id - FROM sqlite_db.account_identities ai - JOIN sqlite_db.participants p + FROM %s + JOIN %s p ON p.email_address IS NOT NULL AND TRIM(p.email_address) <> '' AND lower(p.email_address) = lower(ai.address) UNION SELECT DISTINCT ai.source_id, pi.participant_id - FROM sqlite_db.account_identities ai - JOIN sqlite_db.participant_identifiers pi + FROM %s + JOIN %s ON (pi.identifier_type = 'email' AND lower(pi.identifier_value) = lower(ai.address) AND NOT EXISTS ( - SELECT 1 FROM sqlite_db.participants guard + SELECT 1 FROM %s guard WHERE guard.id = pi.participant_id - AND guard.email_address IS NOT NULL - AND TRIM(guard.email_address) <> '' + AND guard.primary_email_present )) - OR (pi.identifier_type != 'email' AND pi.identifier_value = ai.address)` + OR (pi.identifier_type != 'email' AND pi.identifier_value = ai.address)`, + accountIdentities, participants, accountIdentities, identifiers, participants) +} + +// messageExportColumns is the messages Parquet column list through is_from_me. +func messageExportColumns(source *cacheSourceSnapshot, ownerParticipant, attribution string) string { + return fmt.Sprintf(` + m.id, + m.source_id, + COALESCE(%s, '') AS source_message_id, + %s AS rfc822_message_id, + m.conversation_id, + CASE WHEN m.subject IS NULL THEN NULL ELSE COALESCE(%s, '') END as subject, + CASE WHEN m.snippet IS NULL THEN NULL ELSE COALESCE(%s, '') END as snippet, + m.sent_at, + m.size_estimate, + m.has_attachments, + COALESCE(TRY_CAST(m.attachment_count AS INTEGER), 0) as attachment_count, + m.deleted_from_source_at, + m.sender_id, + %s AS owner_participant_id, + COALESCE(%s, '') as message_type, + %s AS list_id, + %s AS is_from_me`, source.identityExportSQL("m.source_message_id"), source.identityExportSQL("m.rfc822_message_id"), + source.textSQL("m.subject"), source.textSQL("m.snippet"), ownerParticipant, + source.textSQL("m.message_type"), source.identityExportSQL("m.list_id"), attribution) +} // messageCacheAttributionSQL is the cache-facing form of // store.messageIdentityAttributionMatch. Source-native provenance is always // authoritative. A non-empty From envelope is then the only identity surface -// for that message; falling through to the participant's current aliases -// would reclassify mail after a participant merge. Legacy and non-email rows -// without an envelope use the participant's primary email and identifier rows -// with the same per-type case rules as the store. The companion owner value -// is an attribution candidate: consumers gate it on is_from_me, so it can be +// for that message: presence is byte-level over the raw column, exactly like +// the store's guard that the raw email_address is non-NULL and non-blank +// after TRIM, so a present-but-invalid envelope never matches an identity +// and still suppresses the participant fallback that would reclassify mail +// after a participant merge. Legacy and non-email rows without an envelope +// use the participant's primary email and identifier rows with the same +// per-type case rules as the store. The companion owner value is an +// attribution candidate: consumers gate it on is_from_me, so it can be // resolved independently without duplicating this predicate in the export. func messageCacheAttributionSQL( sourceAttribution string, hasSourceAttribution, hasEnvelope bool, + envelopePresence, primaryEmailPresence string, ) (string, string) { - participantFallback := `( + // Use the same valid-or-NULL keys as owner_participants so both derived + // datasets agree even when the SQLite snapshot contains invalid bytes: + // invalid identity text never matches on either path. + accountIdentities := "(SELECT source_id, " + cacheIdentityTextSQL("address") + " AS address FROM sqlite_db.account_identities) ai" + participants := "(SELECT id, " + cacheIdentityTextSQL("email_address") + " AS email_address, " + + primaryEmailPresence + " AS primary_email_present FROM sqlite_db.participants) sp" + identifiers := "(SELECT participant_id, " + cacheIdentityTextSQL("identifier_type") + " AS identifier_type, " + + cacheIdentityTextSQL("identifier_value") + " AS identifier_value FROM sqlite_db.participant_identifiers) spi" + recipients := "(SELECT message_id, participant_id, " + cacheIdentityTextSQL("recipient_type") + " AS recipient_type" + if hasEnvelope { + // envelopePresence decides byte-level presence (scanner: computed + // over the raw column; CSV: the snapshot's envelope_present view + // column), because the valid-or-NULL identity key cannot distinguish + // a damaged envelope from a missing one. + recipients += ", " + cacheIdentityTextSQL("email_address") + " AS email_address" + recipients += ", " + envelopePresence + " AS envelope_present" + } + recipients += " FROM sqlite_db.message_recipients) smr" + participantFallback := fmt.Sprintf(`( EXISTS ( - SELECT 1 FROM sqlite_db.account_identities ai - JOIN sqlite_db.participants sp ON sp.id = m.sender_id + SELECT 1 FROM %s + JOIN %s ON sp.id = m.sender_id WHERE ai.source_id = m.source_id AND sp.email_address IS NOT NULL AND TRIM(sp.email_address) <> '' AND lower(sp.email_address) = lower(ai.address) ) OR EXISTS ( - SELECT 1 FROM sqlite_db.account_identities ai - JOIN sqlite_db.participant_identifiers spi ON spi.participant_id = m.sender_id + SELECT 1 FROM %s + JOIN %s ON spi.participant_id = m.sender_id WHERE ai.source_id = m.source_id AND spi.identifier_type != 'email' AND spi.identifier_value = ai.address ) OR ( NOT EXISTS ( - SELECT 1 FROM sqlite_db.participants sp + SELECT 1 FROM %s WHERE sp.id = m.sender_id - AND sp.email_address IS NOT NULL - AND TRIM(sp.email_address) <> '' + AND sp.primary_email_present ) AND EXISTS ( - SELECT 1 FROM sqlite_db.account_identities ai - JOIN sqlite_db.participant_identifiers spi ON spi.participant_id = m.sender_id + SELECT 1 FROM %s + JOIN %s ON spi.participant_id = m.sender_id WHERE ai.source_id = m.source_id AND spi.identifier_type = 'email' AND lower(spi.identifier_value) = lower(ai.address) ) ) - )` - singleFromParticipant := `(SELECT CASE + )`, accountIdentities, participants, accountIdentities, identifiers, + participants, accountIdentities, identifiers) + singleFromParticipant := fmt.Sprintf(`(SELECT CASE WHEN COUNT(DISTINCT smr.participant_id) = 1 THEN MIN(smr.participant_id) ELSE NULL END - FROM sqlite_db.message_recipients smr + FROM %s WHERE smr.message_id = m.id - AND smr.recipient_type = 'from')` + AND smr.recipient_type = 'from')`, recipients) if !hasEnvelope { attribution := "(" + sourceAttribution + " OR " + participantFallback + ")" ownerParticipant := "COALESCE(m.sender_id, " + singleFromParticipant + ")" return attribution, ownerParticipant } - envelopePresent := `EXISTS ( - SELECT 1 FROM sqlite_db.message_recipients smr + envelopePresent := fmt.Sprintf(`EXISTS ( + SELECT 1 FROM %s WHERE smr.message_id = m.id AND smr.recipient_type = 'from' - AND smr.email_address IS NOT NULL - AND TRIM(smr.email_address) <> '' - )` - envelopeMatch := `EXISTS ( - SELECT 1 FROM sqlite_db.account_identities ai - JOIN sqlite_db.message_recipients smr ON smr.message_id = m.id + AND smr.envelope_present + )`, recipients) + envelopeMatch := fmt.Sprintf(`EXISTS ( + SELECT 1 FROM %s + JOIN %s ON smr.message_id = m.id WHERE ai.source_id = m.source_id AND smr.recipient_type = 'from' AND smr.email_address IS NOT NULL AND TRIM(smr.email_address) <> '' AND lower(smr.email_address) = lower(ai.address) - )` - envelopeOwnerParticipant := `(SELECT MIN(smr.participant_id) - FROM sqlite_db.account_identities ai - JOIN sqlite_db.message_recipients smr ON smr.message_id = m.id + )`, accountIdentities, recipients) + envelopeOwnerParticipant := fmt.Sprintf(`(SELECT MIN(smr.participant_id) + FROM %s + JOIN %s ON smr.message_id = m.id WHERE ai.source_id = m.source_id AND smr.recipient_type = 'from' AND smr.email_address IS NOT NULL AND TRIM(smr.email_address) <> '' - AND lower(smr.email_address) = lower(ai.address))` + AND lower(smr.email_address) = lower(ai.address))`, accountIdentities, recipients) attribution := "(" + sourceAttribution + " OR (" + envelopeMatch + " OR (NOT " + envelopePresent + " AND " + participantFallback + ")))" sourceNativeAttribution := "FALSE" @@ -685,35 +813,31 @@ func buildCacheScheduled( buildCacheMu.Lock() defer buildCacheMu.Unlock() - buildLock, err := acquireCacheBuildLock(analyticsDir) + buildLock, err := acquireCacheBuildLock(context.Background(), analyticsDir) if err != nil { return nil, err } defer func() { _ = buildLock.Unlock() }() - staleness := cacheNeedsBuildLocked(dbPath, analyticsDir) + staleness := cacheNeedsBuildLocked(context.Background(), dbPath, analyticsDir) if !staleness.NeedsBuild { return &buildResult{Skipped: true, OutputDir: analyticsDir}, nil } if _, throttle := scheduledCacheBuildDelay(staleness, minRebuildInterval, now()); throttle { return &buildResult{Skipped: true, OutputDir: analyticsDir}, nil } - if derivedDriftOnly(staleness) { - return refreshIdentityDatasetsOnly( - dbPath, - analyticsDir, - acquirePublishLock, - builderOverrides..., - ) - } - return buildCacheLocked( - dbPath, - analyticsDir, - staleness.FullRebuild, - false, - acquirePublishLock, - builderOverrides..., - ) + return withFullBuildFallback(func() (*buildResult, error) { + if derivedDriftOnly(staleness) { + return refreshIdentityDatasetsOnly(dbPath, analyticsDir, acquirePublishLock, + builderOverrides...) + } + if relatedDriftOnly(staleness) { + return refreshDerivedDatasetsOnly(context.Background(), dbPath, analyticsDir, + acquirePublishLock, true, builderOverrides...) + } + return buildCacheLockedAttempt(dbPath, analyticsDir, staleness.FullRebuild, false, + acquirePublishLock, builderOverrides...) + }, dbPath, analyticsDir, acquirePublishLock, builderOverrides...) } func buildCacheImpl( @@ -724,7 +848,7 @@ func buildCacheImpl( buildCacheMu.Lock() defer buildCacheMu.Unlock() - buildLock, err := acquireCacheBuildLock(analyticsDir) + buildLock, err := acquireCacheBuildLock(context.Background(), analyticsDir) if err != nil { return nil, err } @@ -743,7 +867,7 @@ func buildCacheImpl( // staging and index construction across processes. Readers are NOT excluded // by this lock — they keep querying the committed generation until the // publication step briefly takes the reader-coordination lock. -func acquireCacheBuildLock(analyticsDir string) (*flock.Flock, error) { +func acquireCacheBuildLock(ctx context.Context, analyticsDir string) (*flock.Flock, error) { buildLock, err := cacheBuilderFileLock(analyticsDir) if err != nil { return nil, err @@ -752,7 +876,7 @@ func acquireCacheBuildLock(analyticsDir string) (*flock.Flock, error) { return nil, fmt.Errorf("acquire cache build lock: %w", err) } else if !locked { fmt.Println("Waiting for another msgvault process to finish a cache build...") - if err := buildLock.Lock(); err != nil { + if _, err := buildLock.TryLockContext(ctx, 50*time.Millisecond); err != nil { return nil, fmt.Errorf("acquire cache build lock: %w", err) } } @@ -762,55 +886,59 @@ func acquireCacheBuildLock(analyticsDir string) (*flock.Flock, error) { // conversationsExportSelectSQL renders the conversations dataset export // query: every conversation with an exportable message inside the watermark, // with NULL-normalized string columns. Shared by the full/incremental export -// and the derived-refresh re-staging (exportDerivedConversations) so the two +// and the derived-refresh re-staging in refreshDerivedDatasetsOnly so the two // can never bake different rows for the same watermark. -func conversationsExportSelectSQL(lastMessageID int64) string { +func (s *cacheSourceSnapshot) conversationsExportSelectSQL(lastMessageID int64) string { return fmt.Sprintf(`SELECT id, - COALESCE(TRY_CAST(source_conversation_id AS VARCHAR), '') as source_conversation_id, - COALESCE(TRY_CAST(title AS VARCHAR), '') as title, - COALESCE(TRY_CAST(conversation_type AS VARCHAR), 'email') as conversation_type + COALESCE(%s, '') as source_conversation_id, + COALESCE(%s, '') as title, + COALESCE(%s, 'email') as conversation_type FROM sqlite_db.conversations c WHERE EXISTS ( SELECT 1 FROM sqlite_db.messages m WHERE m.conversation_id = c.id AND %s AND TRY_CAST(m.id AS BIGINT) <= %d - )`, exportableMessageWhere("m"), lastMessageID) + )`, s.identityExportSQL("source_conversation_id"), s.textSQL("title"), s.textSQL("conversation_type"), exportableMessageWhere("m"), lastMessageID) } // participantIdentifiersExportSelectSQL renders the participant_identifiers // dataset export query. Shared by the full/incremental export and the -// derived-refresh re-staging (exportDerivedParticipantIdentifiers) so the two -// can never bake different rows. -func participantIdentifiersExportSelectSQL() string { - return `SELECT participant_id, - COALESCE(TRY_CAST(identifier_type AS VARCHAR), '') AS identifier_type, - COALESCE(TRY_CAST(identifier_value AS VARCHAR), '') AS identifier_value, - COALESCE(TRY_CAST(display_value AS VARCHAR), '') AS display_value, +// derived-refresh re-staging in refreshDerivedDatasetsOnly so the two +// can never bake different rows. Identifier keys with invalid UTF-8 export as +// the empty string (unknown) because repairing them could collide two +// distinct keys. +func (s *cacheSourceSnapshot) participantIdentifiersExportSelectSQL() string { + return fmt.Sprintf(`SELECT participant_id, + COALESCE(%s, '') AS identifier_type, + COALESCE(%s, '') AS identifier_value, + COALESCE(%s, '') AS display_value, COALESCE(TRY_CAST(is_primary AS BOOLEAN), false) AS is_primary - FROM sqlite_db.participant_identifiers` + FROM sqlite_db.participant_identifiers`, s.identityExportSQL("identifier_type"), s.identityExportSQL("identifier_value"), s.textSQL("display_value")) } // participantsExportSelectSQL renders the participants dataset export. The // full and derived-only builders share this query so participant rows and -// display names cannot drift between cache publication paths. -func participantsExportSelectSQL() string { - return `SELECT +// display names cannot drift between cache publication paths. The email +// address is an identity key: invalid bytes export as the empty string +// (unknown), not as a repaired value that could collide with another address. +func (s *cacheSourceSnapshot) participantsExportSelectSQL() string { + return fmt.Sprintf(`SELECT id, - COALESCE(TRY_CAST(email_address AS VARCHAR), '') AS email_address, - COALESCE(TRY_CAST(domain AS VARCHAR), '') AS domain, - COALESCE(TRY_CAST(display_name AS VARCHAR), '') AS display_name, - COALESCE(TRY_CAST(phone_number AS VARCHAR), '') AS phone_number - FROM sqlite_db.participants` + COALESCE(%s, '') AS email_address, + COALESCE(%s, '') AS domain, + COALESCE(%s, '') AS display_name, + COALESCE(%s, '') AS phone_number + FROM sqlite_db.participants`, s.identityExportSQL("email_address"), s.textSQL("domain"), s.textSQL("display_name"), s.textSQL("phone_number")) } // personDisplayNamesExportSelectSQL keeps full and derived-only exports identical. -func personDisplayNamesExportSelectSQL() string { - return `SELECT pp.participant_id, pp.person_id, - COALESCE(TRY_CAST(p.display_name AS VARCHAR), '') AS display_name +func (s *cacheSourceSnapshot) personDisplayNamesExportSelectSQL() string { + return fmt.Sprintf(`SELECT pp.participant_id, pp.person_id, + COALESCE(%s, '') AS display_name FROM sqlite_db.person_participants pp - JOIN sqlite_db.persons p ON p.id = pp.person_id` + JOIN sqlite_db.persons p ON p.id = pp.person_id`, s.textSQL("p.display_name")) } // derivedDriftOnly reports whether participant-link, conversation-membership, @@ -831,7 +959,17 @@ func derivedDriftOnly(staleness cacheStaleness) bool { staleness.HasParticipantDisplayNameDrift || staleness.HasPersonDisplayNameDrift) && !staleness.HasNew && !staleness.HasDeleted && !staleness.HasUpdated && !staleness.HasAccountIdentityDrift && - !staleness.HasDerivedDataDrift + !staleness.HasDerivedDataDrift && !staleness.HasRelatedRowDrift +} + +func relatedDriftOnly(staleness cacheStaleness) bool { + return staleness.HasUsablePublication && staleness.HasRelatedRowDrift && + !staleness.FullRebuild && !staleness.HasNew && !staleness.HasDeleted && + !staleness.HasUpdated && !staleness.HasDerivedDataDrift && + !staleness.HasIdentityDrift && !staleness.HasAccountIdentityDrift && + !staleness.HasConversationParticipantDrift && !staleness.HasConversationTypeDrift && + !staleness.HasParticipantIdentifierDrift && !staleness.HasParticipantDisplayNameDrift && + !staleness.HasPersonDisplayNameDrift } // refreshIdentityDatasetsOnly rebuilds every identity-derived dataset while @@ -847,6 +985,7 @@ func refreshIdentityDatasetsOnly( dbPath, analyticsDir, locking, + false, builderOverrides..., ) } @@ -859,6 +998,36 @@ func buildCacheLocked( fullRebuild, recheckStaleness bool, locking cachePublishLocking, builderOverrides ...duckdbutil.BuilderOverrides, +) (*buildResult, error) { + return withFullBuildFallback(func() (*buildResult, error) { + return buildCacheLockedAttempt(dbPath, analyticsDir, fullRebuild, recheckStaleness, + locking, builderOverrides...) + }, dbPath, analyticsDir, locking, builderOverrides...) +} + +// withFullBuildFallback runs build and finishes with a full build when a +// derived refresh inside it finds changes it cannot repair. A sync can commit +// after the staleness check selects that refresh. The caller holds the builder +// lock, and build's own resources are released before the full build starts. +func withFullBuildFallback( + build func() (*buildResult, error), + dbPath, analyticsDir string, + locking cachePublishLocking, + builderOverrides ...duckdbutil.BuilderOverrides, +) (*buildResult, error) { + result, err := build() + if !errors.Is(err, ErrDerivedRefreshRequiresFullBuild) { + return result, err + } + fmt.Printf("%v. Forcing full rebuild...\n", err) + return buildCacheLockedAttempt(dbPath, analyticsDir, true, false, locking, builderOverrides...) +} + +func buildCacheLockedAttempt( + dbPath, analyticsDir string, + fullRebuild, recheckStaleness bool, + locking cachePublishLocking, + builderOverrides ...duckdbutil.BuilderOverrides, ) (*buildResult, error) { // Callers pass the configured DSN, which may be a file: URI; everything // below (sqlite ?mode=ro opens, the DuckDB attach, filepath.Dir for the @@ -872,13 +1041,17 @@ func buildCacheLocked( return nil, err } if recheckStaleness { - staleness := cacheNeedsBuildLocked(dbPath, analyticsDir) + staleness := cacheNeedsBuildLocked(context.Background(), dbPath, analyticsDir) if !staleness.NeedsBuild { return &buildResult{Skipped: true, OutputDir: analyticsDir}, nil } if derivedDriftOnly(staleness) { return refreshIdentityDatasetsOnly(dbPath, analyticsDir, locking, builderOverrides...) } + if relatedDriftOnly(staleness) { + return refreshDerivedDatasetsOnly(context.Background(), dbPath, analyticsDir, + locking, true, builderOverrides...) + } fullRebuild = staleness.FullRebuild } @@ -899,6 +1072,9 @@ func buildCacheLocked( fmt.Printf("Cache schema version mismatch (have v%d, need v%d). Forcing full rebuild.\n", state.SchemaVersion, cacheSchemaVersion) fullRebuild = true + } else if state.FullRebuildRequired { + fmt.Println("Previous build published a partial snapshot. Forcing full rebuild...") + fullRebuild = true } else { previousState = state hasPreviousState = true @@ -941,6 +1117,13 @@ func buildCacheLocked( _ = identityStore.Close() return nil, fmt.Errorf("read account identity revision: %w", err) } + if hasPreviousState && !fullRebuild && + accountIdentityRevision != previousState.AccountIdentityRevision { + // Canonical links can reuse message facts; account attribution cannot. + fmt.Println("Account identities changed. Forcing full rebuild...") + fullRebuild = true + lastMessageID = 0 + } participantIdentifierRevision, err := identityStore.ParticipantIdentifierRevision() if err != nil { _ = identityStore.Close() @@ -987,6 +1170,10 @@ func buildCacheLocked( return nil, err } defer func() { _ = db.Close() }() + textRepairs := &cacheTextRepairs{} + if err := registerCacheTextFunctions(context.Background(), db, textRepairs); err != nil { + return nil, err + } sourceSnapshot, err := openCacheSourceSnapshot(db, dbPath) if err != nil { return nil, err @@ -996,10 +1183,12 @@ func buildCacheLocked( // Record the freshness boundary immediately before the first source read. // A sync or deletion that finishes after this instant may not be represented // by the snapshot and must invalidate the cache on the next check. - cacheWatermark := time.Now().UTC().Truncate(time.Second) + exportStarted := time.Now() + cacheWatermark := exportStarted.UTC().Truncate(time.Second) var maxMessageID sql.NullInt64 var lastCompletedSyncRunID int64 + var relatedChangeSeq int64 var syncCounters cacheSyncCounters // Use indexed query: id is PRIMARY KEY, sent_at has an index maxIDQuery := `SELECT MAX(id) FROM messages WHERE sent_at IS NOT NULL` @@ -1028,19 +1217,81 @@ func buildCacheLocked( return nil, fmt.Errorf("get cache sync counters: %w", err) } } + var hasRelatedChangeJournal int + if err := sourceSnapshot.QueryRow(` + SELECT COUNT(*) FROM sqlite_master + WHERE type = 'table' AND name = 'cache_related_change_journal' + `).Scan(&hasRelatedChangeJournal); err != nil { + return nil, fmt.Errorf("check cache related-change journal: %w", err) + } + if hasRelatedChangeJournal > 0 { + if err := sourceSnapshot.QueryRow(` + SELECT COALESCE((SELECT seq FROM sqlite_sequence + WHERE name = 'cache_related_change_journal'), 0) + `).Scan(&relatedChangeSeq); err != nil { + return nil, fmt.Errorf("read cache related-change sequence: %w", err) + } + } + // Messages at or below the committed boundary, counted once and reused + // for the expected total of an incremental build. + var coveredCount int64 + coveredCounted := false if !fullRebuild && hasPreviousState && hasSyncRunsTable > 0 { updatesChanged := syncCounters.updates != previousState.LastCacheUpdateCount - coveredAdditionsChanged := syncCounters.additions != previousState.LastCacheAdditionCount && - maxID <= previousState.LastMessageID + additionsRegressed := syncCounters.additions < previousState.LastCacheAdditionCount + populationChanged := false + if syncCounters.additions != previousState.LastCacheAdditionCount { + if err := sourceSnapshot.QueryRow(coveredCacheMessageCountSQL(), previousState.LastMessageID). + Scan(&coveredCount); err != nil { + return nil, fmt.Errorf("check cached message population: %w", err) + } + coveredCounted = true + populationChanged = coveredCount != previousState.Stats.TotalMessages + } failedSyncChanged := syncCounters.failedRunCount != previousState.LastFailedSyncRunCount || syncCounters.failedRunIDSum != previousState.LastFailedSyncRunIDSum - if updatesChanged || coveredAdditionsChanged || failedSyncChanged { + if updatesChanged || additionsRegressed || failedSyncChanged || populationChanged { fmt.Println("Existing cached messages changed. Forcing full rebuild...") fullRebuild = true lastMessageID = 0 } } + var relatedKinds relatedChangeKinds + if hasPreviousState && !fullRebuild && relatedChangeSeq > previousState.LastRelatedChangeSeq { + relatedKinds, err = inspectRelatedChangeKinds(sourceSnapshot, + previousState.LastRelatedChangeSeq, relatedChangeSeq, previousState.LastMessageID) + if err != nil { + return nil, err + } + if relatedKinds.other { + fullRebuild = true + lastMessageID = 0 + relatedKinds = relatedChangeKinds{} + } + } + if hasPreviousState && !fullRebuild && maxID > lastMessageID && + identityRevision != previousState.IdentityRevision { + // Rebuilding canonical edges reads recipients for old messages too. + // Export their complete population alongside the new message shards. + relatedKinds.recipients = true + } + repairRelated := relatedKinds.recipients || relatedKinds.labels || relatedKinds.attachments + if hasPreviousState && !fullRebuild && hasSyncRunsTable > 0 && + syncCounters.additions != previousState.LastCacheAdditionCount && + maxID <= previousState.LastMessageID && !repairRelated { + // Without journaled child changes there is no bounded repair for an + // addition already inside the committed message boundary. + fullRebuild = true + lastMessageID = 0 + } + if hasPreviousState && maxID <= lastMessageID && !fullRebuild && repairRelated { + if err := sourceSnapshot.Close(); err != nil { + return nil, fmt.Errorf("close SQLite snapshot before related repair: %w", err) + } + return refreshDerivedDatasetsOnly(context.Background(), dbPath, analyticsDir, + locking, true, builderOverrides...) + } if hasPreviousState && maxID <= lastMessageID && !fullRebuild { if err := sourceSnapshot.Close(); err != nil { return nil, fmt.Errorf("close SQLite snapshot after metadata check: %w", err) @@ -1058,10 +1309,14 @@ func buildCacheLocked( if err := sourceSnapshot.QueryRow(expectedCountQuery, maxID, lastMessageID).Scan(&expectedBatchCount); err != nil { return nil, fmt.Errorf("count expected staged messages: %w", err) } - expectedTotalQuery := "SELECT COUNT(*) FROM messages WHERE " + - exportableMessageWhere("") + " AND id <= ?" - if err := sourceSnapshot.QueryRow(expectedTotalQuery, maxID).Scan(&expectedTotalCount); err != nil { - return nil, fmt.Errorf("count expected cached messages: %w", err) + if coveredCounted && !replaceAll { + expectedTotalCount = coveredCount + expectedBatchCount + } else { + expectedTotalQuery := "SELECT COUNT(*) FROM messages WHERE " + + exportableMessageWhere("") + " AND id <= ?" + if err := sourceSnapshot.QueryRow(expectedTotalQuery, maxID).Scan(&expectedTotalCount); err != nil { + return nil, fmt.Errorf("count expected cached messages: %w", err) + } } if buildCacheAfterSnapshotHook != nil { buildCacheAfterSnapshotHook() @@ -1127,33 +1382,11 @@ func buildCacheLocked( idFilter += fmt.Sprintf(" AND TRY_CAST(m.id AS BIGINT) > %d", lastMessageID) } - // Junction rows are searchable exactly when their parent message is - // exportable. This includes calendar invitees and meeting attendees while - // excluding hidden rows and messages without a timestamp. - exportableJunctionWhereFor := func(messageIDColumn string) string { - return fmt.Sprintf( - "TRY_CAST(%s AS BIGINT) IN (SELECT CAST(m.id AS BIGINT) FROM sqlite_db.messages m WHERE %s AND TRY_CAST(m.id AS BIGINT) <= %d)", - messageIDColumn, exportableMessageWhere("m"), maxID, - ) - } - junctionFilterFor := func(messageIDColumn, incremental string) string { - where := exportableJunctionWhereFor(messageIDColumn) - if incremental != "" { - return incremental + " AND " + where - } - return " WHERE " + where - } - junctionFilter := func(incremental string) string { - return junctionFilterFor("message_id", incremental) - } - - junctionFile := "data.parquet" - - // runExport executes a COPY query and prints timing info. - runExport := func(label, copyQuery string) error { + // runExport runs one dataset export and prints timing info. + runExport := func(label string, export func() error) error { start := time.Now() fmt.Printf(" %-25s", label+"...") - if _, err := exportDB.Exec(copyQuery); err != nil { + if err := export(); err != nil { fmt.Println() return query.HintRepairEncoding(err) } @@ -1164,137 +1397,40 @@ func buildCacheLocked( // Export each table separately - this is MUCH faster than joining during export // because DuckDB can use SQLite indexes efficiently for simple queries - // 1. Export message_recipients (large junction table) - recipientsDir := filepath.Join(staging.root, "message_recipients") - escapedRecipientsDir := strings.ReplaceAll(recipientsDir, "'", "''") - // This export joins participants, so every column reference is alias - // qualified and the incremental predicate names mr.message_id rather - // than the bare column the shared junctionFilter helper produces. - recipientsFilter := "" - if !replaceAll && lastMessageID > 0 { - recipientsFilter = fmt.Sprintf(" WHERE mr.message_id > %d", lastMessageID) - } - recipientsFilter = junctionFilterFor("mr.message_id", recipientsFilter) - // Two address columns leave here. envelope_address is the header address - // exactly as the store recorded it (NULL when none was — chat, calendar, - // and mail ingested before the column existed); identity filters key on - // its presence. email_address is the resolved recipient address: the - // envelope when present, otherwise the participant's current address, so - // an address filter over this dataset finds pre-upgrade mail too. Only a - // participant with no email address at all (phone or handle only) leaves - // email_address NULL. Databases from before the envelope column export - // NULL envelopes for every row. - recipientEnvelopeExpression := "NULL::VARCHAR" - if sourceSnapshot.hasRecipientEnvelope { - recipientEnvelopeExpression = "NULLIF(TRY_CAST(mr.email_address AS VARCHAR), '')" - } - if err := runExport("message_recipients", fmt.Sprintf(` - COPY ( - SELECT - mr.message_id, - mr.participant_id, - mr.recipient_type, - COALESCE(TRY_CAST(mr.display_name AS VARCHAR), '') as display_name, - COALESCE(%[1]s, NULLIF(TRY_CAST(p.email_address AS VARCHAR), '')) as email_address, - %[1]s as envelope_address - FROM sqlite_db.message_recipients mr - LEFT JOIN sqlite_db.participants p ON p.id = mr.participant_id%[2]s - ) TO '%[3]s/%[4]s' ( - FORMAT PARQUET, - COMPRESSION 'zstd' - ) - `, recipientEnvelopeExpression, recipientsFilter, escapedRecipientsDir, junctionFile)); err != nil { - return nil, fmt.Errorf("export message_recipients: %w", err) - } - - // 2. Export message_labels (large junction table) - messageLabelsDir := filepath.Join(staging.root, "message_labels") - escapedMessageLabelsDir := strings.ReplaceAll(messageLabelsDir, "'", "''") - messageLabelsFilter := "" - if !replaceAll && lastMessageID > 0 { - messageLabelsFilter = fmt.Sprintf(" WHERE message_id > %d", lastMessageID) - } - messageLabelsFilter = junctionFilter(messageLabelsFilter) - if err := runExport("message_labels", fmt.Sprintf(` - COPY ( - SELECT - message_id, - label_id - FROM sqlite_db.message_labels%s - ) TO '%s/%s' ( - FORMAT PARQUET, - COMPRESSION 'zstd' - ) - `, messageLabelsFilter, escapedMessageLabelsDir, junctionFile)); err != nil { - return nil, fmt.Errorf("export message_labels: %w", err) - } - - // 3. Export attachments - attachmentsDir := filepath.Join(staging.root, tableAttachments) - escapedAttachmentsDir := strings.ReplaceAll(attachmentsDir, "'", "''") - attachmentsFilter := "" - if !replaceAll && lastMessageID > 0 { - attachmentsFilter = fmt.Sprintf(" WHERE message_id > %d", lastMessageID) - } - attachmentsFilter = junctionFilter(attachmentsFilter) - attachmentMIMEExpression := "'' AS mime_type" - if sourceSnapshot.hasAttachmentMIME { - attachmentMIMEExpression = "COALESCE(TRY_CAST(mime_type AS VARCHAR), '') AS mime_type" + publicationPlan := cachePublishPlanForMode(replaceAll) + for dataset, changed := range relatedKinds.datasets() { + if changed { + delete(publicationPlan.Append, dataset) + publicationPlan.Replace[dataset] = true + } } - attachmentMetadataExpression := "NULL::VARCHAR AS attachment_metadata" - if sourceSnapshot.hasAttachmentMetadata { - attachmentMetadataExpression = "TRY_CAST(attachment_metadata AS VARCHAR) AS attachment_metadata" + afterMessageIDs := make(map[string]int64) + for _, dataset := range []string{"message_recipients", "message_labels", tableAttachments, tableLabels} { + if publicationPlan.Append[dataset] { + afterMessageIDs[dataset] = lastMessageID + } else { + afterMessageIDs[dataset] = 0 + } } - if err := runExport(tableAttachments, fmt.Sprintf(` - COPY ( - SELECT - id AS attachment_id, - message_id, - size, - COALESCE(TRY_CAST(filename AS VARCHAR), '') as filename, - %s, - %s - FROM sqlite_db.attachments%s - ) TO '%s/%s' ( - FORMAT PARQUET, - COMPRESSION 'zstd' - ) - `, attachmentMIMEExpression, attachmentMetadataExpression, - attachmentsFilter, escapedAttachmentsDir, junctionFile)); err != nil { - return nil, fmt.Errorf("export attachments: %w", err) + if err := exportRelatedDatasets(context.Background(), exportDB, sourceSnapshot, + maxID, staging.root, afterMessageIDs); err != nil { + return nil, err } // 4. Export participants - participantsDir := filepath.Join(staging.root, tableParticipants) - escapedParticipantsDir := strings.ReplaceAll(participantsDir, "'", "''") - if err := runExport(tableParticipants, fmt.Sprintf(` - COPY ( - %s - ) TO '%s/participants.parquet' ( - FORMAT PARQUET, - COMPRESSION 'zstd' - ) - `, participantsExportSelectSQL(), escapedParticipantsDir)); err != nil { + ctx := context.Background() + copyDataset := func(table, selectSQL string) func() error { + return func() error { + return copyParquet(ctx, exportDB, filepath.Join(staging.root, table), table+".parquet", selectSQL) + } + } + if err := runExport(tableParticipants, copyDataset(tableParticipants, sourceSnapshot.participantsExportSelectSQL())); err != nil { return nil, fmt.Errorf("export participants: %w", err) } - - participantIdentifiersDir := filepath.Join(staging.root, tableParticipantIdentifiers) - escapedParticipantIdentifiersDir := strings.ReplaceAll(participantIdentifiersDir, "'", "''") - if err := runExport(tableParticipantIdentifiers, fmt.Sprintf(` - COPY ( - %s - ) TO '%s/participant_identifiers.parquet' ( - FORMAT PARQUET, - COMPRESSION 'zstd' - ) - `, participantIdentifiersExportSelectSQL(), escapedParticipantIdentifiersDir)); err != nil { + if err := runExport(tableParticipantIdentifiers, copyDataset(tableParticipantIdentifiers, sourceSnapshot.participantIdentifiersExportSelectSQL())); err != nil { return nil, fmt.Errorf("export participant identifiers: %w", err) } - - personDisplayNamesDir := filepath.Join(staging.root, tablePersonDisplayNames) - if err := runExport(tablePersonDisplayNames, fmt.Sprintf( - `COPY (%s) TO '%s/person_display_names.parquet' (FORMAT PARQUET, COMPRESSION 'zstd')`, - personDisplayNamesExportSelectSQL(), quoteCacheSQL(personDisplayNamesDir))); err != nil { + if err := runExport(tablePersonDisplayNames, copyDataset(tablePersonDisplayNames, sourceSnapshot.personDisplayNamesExportSelectSQL())); err != nil { return nil, fmt.Errorf("export person names: %w", err) } @@ -1303,15 +1439,8 @@ func buildCacheLocked( // ownerParticipantsSelectSQL for the resolution rules). Always fully // replaced (not filtered by lastMessageID) since identities are cheap to // recompute and independent of the message ID watermark. - ownerParticipantsDir := filepath.Join(staging.root, tableOwnerParticipants) - escapedOwnerParticipantsDir := strings.ReplaceAll(ownerParticipantsDir, "'", "''") - if err := runExport(tableOwnerParticipants, fmt.Sprintf(` - COPY (%s - ) TO '%s/owner_participants.parquet' ( - FORMAT PARQUET, - COMPRESSION 'zstd' - ) - `, ownerParticipantsSelectSQL, escapedOwnerParticipantsDir)); err != nil { + primaryEmailPresence := sourceSnapshot.identityPresenceSQL("email_address", "primary_email_present") + if err := runExport(tableOwnerParticipants, copyDataset(tableOwnerParticipants, ownerParticipantsSelectSQL(primaryEmailPresence))); err != nil { return nil, fmt.Errorf("export owner participants: %w", err) } @@ -1321,106 +1450,31 @@ func buildCacheLocked( // table so it can be COPYed to Parquet like every other dataset. Always // written, even when there are no linked participants, so the dataset // directory required by RequiredParquetDirs always exists. - participantClustersDir := filepath.Join(staging.root, tableParticipantClusters) - escapedParticipantClustersDir := strings.ReplaceAll(participantClustersDir, "'", "''") - if _, err := exportDB.Exec( - `CREATE TEMP TABLE tmp_participant_clusters (participant_id BIGINT, canonical_id BIGINT)`, - ); err != nil { - return nil, fmt.Errorf("create participant clusters temp table: %w", err) - } - if len(participantClusters) > 0 { - values := make([]string, 0, len(participantClusters)) - for participantID, canonicalID := range participantClusters { - values = append(values, fmt.Sprintf("(%d, %d)", participantID, canonicalID)) - } - insertSQL := "INSERT INTO tmp_participant_clusters (participant_id, canonical_id) VALUES " + - strings.Join(values, ", ") - if _, err := exportDB.Exec(insertSQL); err != nil { - return nil, fmt.Errorf("populate participant clusters temp table: %w", err) - } + if err := stageParticipantClusters(ctx, exportDB, participantClusters); err != nil { + return nil, err } - if err := runExport(tableParticipantClusters, fmt.Sprintf(` - COPY ( - SELECT participant_id, canonical_id FROM tmp_participant_clusters - ) TO '%s/participant_clusters.parquet' ( - FORMAT PARQUET, - COMPRESSION 'zstd' - ) - `, escapedParticipantClustersDir)); err != nil { + if err := runExport(tableParticipantClusters, copyDataset(tableParticipantClusters, participantClustersSelectSQL)); err != nil { return nil, fmt.Errorf("export participant clusters: %w", err) } if _, err := exportDB.Exec(`DROP TABLE tmp_participant_clusters`); err != nil { return nil, fmt.Errorf("drop participant clusters temp table: %w", err) } - conversationParticipantsDir := filepath.Join(staging.root, tableConversationParticipants) - escapedConversationParticipantsDir := strings.ReplaceAll(conversationParticipantsDir, "'", "''") - if err := runExport(tableConversationParticipants, fmt.Sprintf(` - COPY ( - SELECT - cp.conversation_id, - cp.participant_id - FROM sqlite_db.conversation_participants cp - WHERE EXISTS ( - SELECT 1 FROM sqlite_db.messages m - WHERE m.conversation_id = cp.conversation_id - AND `+exportableMessageWhere("m")+` - AND TRY_CAST(m.id AS BIGINT) <= %d - ) - ) TO '%s/conversation_participants.parquet' ( - FORMAT PARQUET, - COMPRESSION 'zstd' - ) - `, maxID, escapedConversationParticipantsDir)); err != nil { + if err := runExport(tableConversationParticipants, copyDataset(tableConversationParticipants, conversationParticipantsSelectSQL(maxID))); err != nil { return nil, fmt.Errorf("export conversation participants: %w", err) } - // 5. Export labels - labelsDir := filepath.Join(staging.root, tableLabels) - escapedLabelsDir := strings.ReplaceAll(labelsDir, "'", "''") - if err := runExport(tableLabels, fmt.Sprintf(` - COPY ( - SELECT - id, - COALESCE(TRY_CAST(name AS VARCHAR), '') as name - FROM sqlite_db.labels - ) TO '%s/labels.parquet' ( - FORMAT PARQUET, - COMPRESSION 'zstd' - ) - `, escapedLabelsDir)); err != nil { - return nil, fmt.Errorf("export labels: %w", err) - } - // 6. Export sources - sourcesDir := filepath.Join(staging.root, "sources") - escapedSourcesDir := strings.ReplaceAll(sourcesDir, "'", "''") - if err := runExport("sources", fmt.Sprintf(` - COPY ( - SELECT + if err := runExport("sources", copyDataset("sources", fmt.Sprintf(`SELECT id, - identifier as account_email, - COALESCE(TRY_CAST(source_type AS VARCHAR), 'gmail') as source_type - FROM sqlite_db.sources - ) TO '%s/sources.parquet' ( - FORMAT PARQUET, - COMPRESSION 'zstd' - ) - `, escapedSourcesDir)); err != nil { + COALESCE(%s, '') as account_email, + COALESCE(%s, 'gmail') as source_type + FROM sqlite_db.sources`, sourceSnapshot.identityExportSQL("identifier"), sourceSnapshot.textSQL("source_type")))); err != nil { return nil, fmt.Errorf("export sources: %w", err) } // 7. Export conversations (for Gmail thread IDs) - conversationsDir := filepath.Join(staging.root, tableConversations) - escapedConversationsDir := strings.ReplaceAll(conversationsDir, "'", "''") - if err := runExport(tableConversations, fmt.Sprintf(` - COPY ( - %s - ) TO '%s/conversations.parquet' ( - FORMAT PARQUET, - COMPRESSION 'zstd' - ) - `, conversationsExportSelectSQL(maxID), escapedConversationsDir)); err != nil { + if err := runExport(tableConversations, copyDataset(tableConversations, sourceSnapshot.conversationsExportSelectSQL(maxID))); err != nil { return nil, fmt.Errorf("export conversations: %w", err) } @@ -1438,28 +1492,15 @@ func buildCacheLocked( messageSourceAttribution, sourceSnapshot.hasMessageSourceAttribution, sourceSnapshot.hasRecipientEnvelope, + sourceSnapshot.identityPresenceSQL("email_address", "envelope_present"), + primaryEmailPresence, ) - if err := runExport(tableMessages, fmt.Sprintf(` + columns := messageExportColumns(sourceSnapshot, messageOwnerParticipant, messageAttribution) + if err := runExport(tableMessages, func() error { + _, err := exportDB.Exec(fmt.Sprintf(` COPY ( - SELECT - m.id, - m.source_id, - m.source_message_id, - TRY_CAST(m.rfc822_message_id AS VARCHAR) AS rfc822_message_id, - m.conversation_id, - CASE WHEN m.subject IS NULL THEN NULL ELSE COALESCE(TRY_CAST(m.subject AS VARCHAR), '') END as subject, - CASE WHEN m.snippet IS NULL THEN NULL ELSE COALESCE(TRY_CAST(m.snippet AS VARCHAR), '') END as snippet, - m.sent_at, - m.size_estimate, - m.has_attachments, - COALESCE(TRY_CAST(m.attachment_count AS INTEGER), 0) as attachment_count, - m.deleted_from_source_at, - m.sender_id, - %s AS owner_participant_id, - COALESCE(TRY_CAST(m.message_type AS VARCHAR), '') as message_type, - TRY_CAST(m.list_id AS VARCHAR) AS list_id, - %s AS is_from_me, + SELECT %s, CAST(EXTRACT(YEAR FROM m.sent_at) AS INTEGER) as year, CAST(EXTRACT(MONTH FROM m.sent_at) AS INTEGER) as month FROM sqlite_db.messages m @@ -1470,7 +1511,9 @@ func buildCacheLocked( OVERWRITE_OR_IGNORE, COMPRESSION 'zstd' ) - `, messageOwnerParticipant, messageAttribution, idFilter, escapedMessagesDir)); err != nil { + `, columns, idFilter, escapedMessagesDir)) + return err + }); err != nil { return nil, fmt.Errorf("export messages: %w", err) } @@ -1482,39 +1525,12 @@ func buildCacheLocked( // Only the cleared-directory builds need this; an incremental no-op // leaves the previous shards in place. if expectedTotalCount == 0 && replaceAll { - emptyShardDir := filepath.Join(messagesDir, "year=0") - if err := os.MkdirAll(emptyShardDir, 0755); err != nil { - return nil, fmt.Errorf("create empty messages shard dir: %w", err) - } - escapedEmptyShard := strings.ReplaceAll( - filepath.Join(emptyShardDir, "empty.parquet"), "'", "''") // Same column list as the partitioned export minus the year // partition column, which hive_partitioning derives from the path. - if _, err := exportDB.Exec(fmt.Sprintf(` - COPY ( - SELECT - m.id, - m.source_id, - m.source_message_id, - TRY_CAST(m.rfc822_message_id AS VARCHAR) AS rfc822_message_id, - m.conversation_id, - CASE WHEN m.subject IS NULL THEN NULL ELSE COALESCE(TRY_CAST(m.subject AS VARCHAR), '') END as subject, - CASE WHEN m.snippet IS NULL THEN NULL ELSE COALESCE(TRY_CAST(m.snippet AS VARCHAR), '') END as snippet, - m.sent_at, - m.size_estimate, - m.has_attachments, - COALESCE(TRY_CAST(m.attachment_count AS INTEGER), 0) as attachment_count, - m.deleted_from_source_at, - m.sender_id, - %s AS owner_participant_id, - COALESCE(TRY_CAST(m.message_type AS VARCHAR), '') as message_type, - TRY_CAST(m.list_id AS VARCHAR) AS list_id, - %s AS is_from_me, + if err := copyParquet(ctx, exportDB, filepath.Join(messagesDir, "year=0"), "empty.parquet", `SELECT `+columns+`, CAST(EXTRACT(MONTH FROM m.sent_at) AS INTEGER) as month FROM sqlite_db.messages m - WHERE 1 = 0 - ) TO '%s' (FORMAT PARQUET, COMPRESSION 'zstd') - `, messageOwnerParticipant, messageAttribution, escapedEmptyShard)); err != nil { + WHERE 1 = 0`); err != nil { return nil, fmt.Errorf("export empty messages shard: %w", err) } } @@ -1523,19 +1539,24 @@ func buildCacheLocked( if replaceAll { buildMode = identityindex.ModeFull } + if relatedKinds.recipients && !replaceAll { + buildMode = identityindex.ModeAppendRepair + delete(publicationPlan.Append, identityindex.DatasetActivity) + publicationPlan.Replace[identityindex.DatasetActivity] = true + } derived, err := identityindex.Build(context.Background(), exportDB, identityindex.BuildOptions{ - Mode: buildMode, - CommittedRoot: analyticsDir, - StagedBaseRoot: staging.root, - OutputRoot: staging.root, - EffectiveAt: cacheWatermark, - Progress: reportIdentityBuildProgress, + Mode: buildMode, + CommittedRoot: analyticsDir, + StagedBaseRoot: staging.root, + ReplacedBaseDatasets: publicationPlan.Replace, + OutputRoot: staging.root, + EffectiveAt: cacheWatermark, + Progress: reportIdentityBuildProgress, }) if err != nil { return nil, fmt.Errorf("build identity index: %w", err) } reportRelationshipActivityStats(derived.Activity) - publicationPlan := cachePublishPlanForMode(replaceAll) fmt.Printf(" %-25s %s\n", "Total:", time.Since(buildStart).Round(time.Millisecond)) @@ -1567,6 +1588,7 @@ func buildCacheLocked( if buildCacheBeforeStateWriteHook != nil { buildCacheBeforeStateWriteHook() } + partialSnapshot := false if hasSyncRunsTable > 0 { checkDB, openErr := sql.Open("sqlite3", dbPath+"?mode=ro") if openErr != nil { @@ -1581,13 +1603,23 @@ func buildCacheLocked( return nil, fmt.Errorf("close sqlite after cache consistency check: %w", closeErr) } if currentCounters != syncCounters { - return nil, fmt.Errorf( - "sync counters changed during cache export (additions %d→%d, updates %d→%d, failed runs count %d→%d, id sum %d→%d); retry", - syncCounters.additions, currentCounters.additions, - syncCounters.updates, currentCounters.updates, - syncCounters.failedRunCount, currentCounters.failedRunCount, - syncCounters.failedRunIDSum, currentCounters.failedRunIDSum, - ) + // The journal tracks children committed after their parent was + // exported. Keep the snapshot watermarks so the next build can + // append new messages or repair those children. Legacy archives + // without a journal still need a conservative full repair. + partialSnapshot = hasRelatedChangeJournal == 0 + level := slog.LevelInfo + if partialSnapshot { + level = slog.LevelWarn + } + slog.Log(context.Background(), level, + "sync counters changed during cache export; published read snapshot", + "full_rebuild_required", partialSnapshot, + "additions", fmt.Sprintf("%d→%d", syncCounters.additions, currentCounters.additions), + "updates", fmt.Sprintf("%d→%d", syncCounters.updates, currentCounters.updates), + "failed_runs", fmt.Sprintf("%d→%d", syncCounters.failedRunCount, currentCounters.failedRunCount), + "failed_run_id_sum", fmt.Sprintf("%d→%d", syncCounters.failedRunIDSum, currentCounters.failedRunIDSum), + "elapsed", time.Since(exportStarted).Round(time.Second)) } } @@ -1600,6 +1632,7 @@ func buildCacheLocked( LastCompletedSyncRunID: lastCompletedSyncRunID, LastCacheAdditionCount: syncCounters.additions, LastCacheUpdateCount: syncCounters.updates, + LastRelatedChangeSeq: relatedChangeSeq, LastFailedSyncRunCount: syncCounters.failedRunCount, LastFailedSyncRunIDSum: syncCounters.failedRunIDSum, IdentityRevision: identityRevision, @@ -1611,6 +1644,7 @@ func buildCacheLocked( ConversationParticipantsFingerprint: derived.ConversationParticipantsFingerprint, ConversationTypesFingerprint: typesFingerprint, Stats: derived.Stats, + FullRebuildRequired: partialSnapshot, } stateData, err := json.Marshal(state, json.Deterministic(true)) if err != nil { @@ -1619,9 +1653,14 @@ func buildCacheLocked( if err := publishCache(staging, analyticsDir, publicationPlan, stateData, locking); err != nil { return nil, err } + reportCacheTextRepairs(os.Stderr, textRepairs) + if hasRelatedChangeJournal > 0 { + warnRelatedChangePrune(dbPath, relatedChangeSeq, derivedDataRevision) + } return &buildResult{ ExportedCount: expectedTotalCount, + StagedCount: stagedCount, MaxMessageID: maxID, OutputDir: analyticsDir, }, nil @@ -1641,13 +1680,6 @@ func reportRelationshipActivityStats(stats identityindex.ActivityStats) { stats.FinalRows, stats.ExpansionRatio, ) - if stats.ExpansionRatio > 4 { - fmt.Printf( - " Warning: relationship membership fan-out is %.2fx; "+ - "consider a normalized conversation-member index if this archive keeps growing\n", - stats.ExpansionRatio, - ) - } } func countStagedMessages(db sqlRowQuerier, messagesDir string, requireShard bool) (int64, error) { @@ -1832,12 +1864,34 @@ type cacheSourceSnapshot struct { hasAttachmentMetadata bool hasMessageSourceAttribution bool hasRecipientEnvelope bool + // csvSnapshot records that the sqlite_db tables are CSV views exported + // from SQLite, not the attached database itself. It is set once at + // construction: prepareTables closes the SQLite transaction before the + // Parquet exports run, so sqliteTx cannot identify the path at that + // point. Address presence uses the snapshot's boolean columns instead + // of the raw email_address bytes. + csvSnapshot bool } type cacheSnapshotTable struct { name string query string typeOverrides string + // identityCols lists query columns that attribution comparisons key on. + // The CSV fallback writes invalid UTF-8 in these columns as NULL instead + // of repairing it, matching cacheIdentityTextSQL on the sqlite_scanner + // path: two distinct invalid byte sequences must never collapse onto one + // repaired key and mis-attribute messages. + identityCols map[string]bool +} + +// identityColumns builds the identityCols set for a cacheSnapshotTable. +func identityColumns(columns ...string) map[string]bool { + set := make(map[string]bool, len(columns)) + for _, column := range columns { + set[column] = true + } + return set } func openCacheSourceSnapshot(duckDB *sql.DB, dbPath string) (*cacheSourceSnapshot, error) { @@ -1888,6 +1942,7 @@ func openCacheSourceSnapshot(duckDB *sql.DB, dbPath string) (*cacheSourceSnapsho } return &cacheSourceSnapshot{ duckDB: duckDB, sqliteDB: sqliteDB, sqliteTx: sqliteTx, tmpDir: tmpDir, + csvSnapshot: true, }, nil } @@ -1915,6 +1970,16 @@ func (s *cacheSourceSnapshot) DuckDB() sqlRunner { return s.duckDB } +// identityPresenceSQL preserves raw address presence independently of UTF-8 +// validity. CSV snapshots carry a boolean computed by SQLite before invalid +// identity bytes become NULL; scanner snapshots still have the raw bytes. +func (s *cacheSourceSnapshot) identityPresenceSQL(emailRef, presenceRef string) string { + if s.csvSnapshot { + return "COALESCE(" + presenceRef + ", FALSE)" + } + return cacheIdentityPresenceSQL(emailRef) +} + // Prepare materializes the CSV fallback after metadata has pinned the SQLite // read transaction. sqlite_scanner needs no preparation because DuckDB's // transaction reads the attached database directly. @@ -1963,9 +2028,16 @@ func (s *cacheSourceSnapshot) tables() []cacheSnapshotTable { // mr.email_address to derive both cache columns. NULL travels through // the CSV fallback as the \N sentinel, so a row with no recorded header // address stays distinguishable from one carrying an empty value. + // envelope_present records byte-level presence computed by SQLite over + // the raw bytes: invalid UTF-8 is already NULL in this CSV, so the + // identity key alone cannot distinguish a damaged envelope from a + // missing one. recipientEnvelopeColumn := "NULL AS email_address" + recipientEnvelopePresence := "FALSE AS envelope_present" + const emailPresence = "CASE WHEN email_address IS NOT NULL AND TRIM(email_address) <> '' THEN TRUE ELSE FALSE END" if s.hasRecipientEnvelope { recipientEnvelopeColumn = "email_address" + recipientEnvelopePresence = emailPresence + " AS envelope_present" } messageColumns := "id, source_id, source_message_id, rfc822_message_id, conversation_id, subject, snippet, sent_at, size_estimate, has_attachments, attachment_count, deleted_from_source_at, deleted_at, sender_id, message_type, list_id, is_from_me" messageTypes := "types={'id': 'BIGINT', 'source_id': 'BIGINT', 'source_message_id': 'VARCHAR', 'rfc822_message_id': 'VARCHAR', 'conversation_id': 'BIGINT', 'subject': 'VARCHAR', 'snippet': 'VARCHAR', 'sent_at': 'TIMESTAMP', 'size_estimate': 'BIGINT', 'has_attachments': 'BOOLEAN', 'attachment_count': 'INTEGER', 'deleted_from_source_at': 'TIMESTAMP', 'deleted_at': 'TIMESTAMP', 'sender_id': 'BIGINT', 'message_type': 'VARCHAR', 'list_id': 'VARCHAR', 'is_from_me': 'BOOLEAN'" @@ -1984,29 +2056,33 @@ func (s *cacheSourceSnapshot) tables() []cacheSnapshotTable { // `deleted_at IS NULL` filter on this path the same way it does // on the sqlite_scanner path; otherwise DuckDB binds against a // CSV view that lacks the column and the export fails on Windows. - {tableMessages, "SELECT " + messageColumns + " FROM messages WHERE sent_at IS NOT NULL", messageTypes}, - {"message_recipients", "SELECT message_id, participant_id, recipient_type, display_name, " + recipientEnvelopeColumn + " FROM message_recipients", - "types={'message_id': 'BIGINT', 'participant_id': 'BIGINT', 'recipient_type': 'VARCHAR', 'display_name': 'VARCHAR', 'email_address': 'VARCHAR'}"}, + {tableMessages, "SELECT " + messageColumns + " FROM messages WHERE sent_at IS NOT NULL", messageTypes, identityColumns("source_message_id", "rfc822_message_id", "list_id")}, + {"message_recipients", "SELECT message_id, participant_id, recipient_type, display_name, " + recipientEnvelopeColumn + ", " + recipientEnvelopePresence + " FROM message_recipients", + "types={'message_id': 'BIGINT', 'participant_id': 'BIGINT', 'recipient_type': 'VARCHAR', 'display_name': 'VARCHAR', 'email_address': 'VARCHAR', 'envelope_present': 'BOOLEAN'}", + identityColumns("recipient_type", "email_address")}, {"message_labels", "SELECT message_id, label_id FROM message_labels", - "types={'message_id': 'BIGINT', 'label_id': 'BIGINT'}"}, + "types={'message_id': 'BIGINT', 'label_id': 'BIGINT'}", nil}, {tableAttachments, attachmentQuery, - "types={'id': 'BIGINT', 'message_id': 'BIGINT', 'size': 'BIGINT', 'filename': 'VARCHAR', 'mime_type': 'VARCHAR', 'attachment_metadata': 'VARCHAR'}"}, - {"persons", "SELECT id, display_name FROM persons", "types={'id': 'BIGINT', 'display_name': 'VARCHAR'}"}, - {"person_participants", "SELECT person_id, participant_id FROM person_participants", "types={'person_id': 'BIGINT', 'participant_id': 'BIGINT'}"}, - {tableParticipants, "SELECT id, email_address, domain, display_name, phone_number FROM participants", - "types={'id': 'BIGINT', 'email_address': 'VARCHAR', 'domain': 'VARCHAR', 'display_name': 'VARCHAR', 'phone_number': 'VARCHAR'}"}, + "types={'id': 'BIGINT', 'message_id': 'BIGINT', 'size': 'BIGINT', 'filename': 'VARCHAR', 'mime_type': 'VARCHAR', 'attachment_metadata': 'VARCHAR'}", nil}, + {"persons", "SELECT id, display_name FROM persons", "types={'id': 'BIGINT', 'display_name': 'VARCHAR'}", nil}, + {"person_participants", "SELECT person_id, participant_id FROM person_participants", "types={'person_id': 'BIGINT', 'participant_id': 'BIGINT'}", nil}, + {tableParticipants, "SELECT id, email_address, domain, display_name, phone_number, " + emailPresence + " AS primary_email_present FROM participants", + "types={'id': 'BIGINT', 'email_address': 'VARCHAR', 'domain': 'VARCHAR', 'display_name': 'VARCHAR', 'phone_number': 'VARCHAR', 'primary_email_present': 'BOOLEAN'}", + identityColumns("email_address")}, {"account_identities", "SELECT source_id, address FROM account_identities", - "types={'source_id': 'BIGINT', 'address': 'VARCHAR'}"}, + "types={'source_id': 'BIGINT', 'address': 'VARCHAR'}", + identityColumns("address")}, {tableParticipantIdentifiers, "SELECT participant_id, identifier_type, identifier_value, display_value, is_primary FROM participant_identifiers", - "types={'participant_id': 'BIGINT', 'identifier_type': 'VARCHAR', 'identifier_value': 'VARCHAR', 'display_value': 'VARCHAR', 'is_primary': 'BOOLEAN'}"}, + "types={'participant_id': 'BIGINT', 'identifier_type': 'VARCHAR', 'identifier_value': 'VARCHAR', 'display_value': 'VARCHAR', 'is_primary': 'BOOLEAN'}", + identityColumns("identifier_type", "identifier_value")}, {tableLabels, "SELECT id, name FROM labels", - "types={'id': 'BIGINT', 'name': 'VARCHAR'}"}, + "types={'id': 'BIGINT', 'name': 'VARCHAR'}", nil}, {"sources", "SELECT id, identifier, source_type FROM sources", - "types={'id': 'BIGINT', 'identifier': 'VARCHAR', 'source_type': 'VARCHAR'}"}, + "types={'id': 'BIGINT', 'identifier': 'VARCHAR', 'source_type': 'VARCHAR'}", identityColumns("identifier")}, {tableConversations, "SELECT id, source_conversation_id, title, COALESCE(conversation_type, 'email_thread') AS conversation_type FROM conversations", - "types={'id': 'BIGINT', 'source_conversation_id': 'VARCHAR', 'title': 'VARCHAR', 'conversation_type': 'VARCHAR'}"}, + "types={'id': 'BIGINT', 'source_conversation_id': 'VARCHAR', 'title': 'VARCHAR', 'conversation_type': 'VARCHAR'}", identityColumns("source_conversation_id")}, {tableConversationParticipants, "SELECT conversation_id, participant_id FROM conversation_participants", - "types={'conversation_id': 'BIGINT', 'participant_id': 'BIGINT'}"}, + "types={'conversation_id': 'BIGINT', 'participant_id': 'BIGINT'}", nil}, } } @@ -2016,7 +2092,7 @@ func (s *cacheSourceSnapshot) prepareTables(tables []cacheSnapshotTable) error { } for _, t := range tables { csvPath := filepath.Join(s.tmpDir, t.name+".csv") - if err := exportToCSV(s.sqliteTx, t.query, csvPath); err != nil { + if err := exportToCSV(s.sqliteTx, t.query, csvPath, t.identityCols); err != nil { return fmt.Errorf("export %s to CSV: %w", t.name, err) } } @@ -2034,9 +2110,12 @@ func (s *cacheSourceSnapshot) prepareTables(tables []cacheSnapshotTable) error { // DuckDB handles both forward and backslash paths, but normalize to forward. escaped := strings.ReplaceAll(csvPath, "\\", "/") escaped = strings.ReplaceAll(escaped, "'", "''") - csvOpts := "header=true, nullstr='\\N'" + // Pin quoting to what encoding/csv writes. It only quotes fields that + // need it, so the sniffer's 20,480-row sample may see none and settle + // on quote=(empty), splitting a later quoted field into extra columns. + csvOpts := "header=true, quote='\"', escape='\"', nullstr='\\N'" if t.typeOverrides != "" { - csvOpts += ", " + t.typeOverrides + csvOpts += ", " + strings.TrimSuffix(t.typeOverrides, "}") + ", '__invalid_utf8': 'VARCHAR'}" } viewSQL := fmt.Sprintf( `CREATE VIEW sqlite_db."%s" AS SELECT * FROM read_csv_auto('%s', %s)`, @@ -2090,8 +2169,11 @@ func (s *cacheSourceSnapshot) Close() error { const csvNullStr = `\N` // exportToCSV exports the results of a SQL query to a CSV file. -// NULL values are written as \N (PostgreSQL convention). -func exportToCSV(db sqlRunner, query string, dest string) error { +// NULL values are written as \N (PostgreSQL convention). Invalid UTF-8 in +// identity columns is also written as NULL so attribution keys cannot +// collide after repair; every other invalid value is repaired to U+FFFD and +// marked for counting only when its output column is exported. +func exportToCSV(db sqlRunner, query string, dest string, identityCols map[string]bool) error { rows, err := db.Query(query) if err != nil { return err @@ -2110,7 +2192,7 @@ func exportToCSV(db sqlRunner, query string, dest string) error { if err != nil { return err } - if err := w.Write(cols); err != nil { + if err := w.Write(append(cols, "__invalid_utf8")); err != nil { return err } @@ -2124,12 +2206,20 @@ func exportToCSV(db sqlRunner, query string, dest string) error { if err := rows.Scan(ptrs...); err != nil { return err } - record := make([]string, len(cols)) + record := make([]string, len(cols)+1) for i, v := range values { - if v.Valid { - record[i] = v.String - } else { + if v.Valid && !utf8.ValidString(v.String) { + // Avoid CSV quotes even when the first damage is beyond the + // reader's sample window. Column names are fixed by tables(). + record[len(cols)] += "|" + cols[i] + "|" + } + switch { + case !v.Valid: record[i] = csvNullStr + case identityCols[cols[i]] && !utf8.ValidString(v.String): + record[i] = csvNullStr + default: + record[i] = textutil.SanitizeUTF8(v.String) } } if err := w.Write(record); err != nil { @@ -2147,7 +2237,12 @@ func exportToCSV(db sqlRunner, query string, dest string) error { // rebuildCacheAfterWrite refreshes the SQLite-backed analytics cache after a // write operation. Cache maintenance is part of the operation result: SQLite // remains authoritative, but callers must surface any refresh failure. -func rebuildCacheAfterWrite(dbPath string) error { +func rebuildCacheAfterWrite(dbPath string, state *invocation) error { + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := repairLogger(state.logger) if store.IsPostgresURL(dbPath) { return nil } @@ -2170,6 +2265,43 @@ func rebuildCacheAfterWrite(dbPath string) error { return nil } +// rebuildCacheAfterManualSync applies the same minimum interval as scheduled +// syncs. A usable committed cache stays available after a small import; an +// absent or incompatible cache still recovers immediately. +func rebuildCacheAfterManualSync(dbPath string, state *invocation) error { + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := repairLogger(state.logger) + if isDaemonCLISubprocess() { + // The parent daemon inspects the durable result and owns any detached + // refresh. A CLI child must not keep its HTTP caller waiting on a build. + return nil + } + if store.IsPostgresURL(dbPath) { + return nil + } + if !cfg.Analytics.AutoBuildCache { + return nil + } + staleness, err := cacheNeedsBuildForQuery(context.Background(), dbPath, cfg.AnalyticsDir()) + if err != nil { + return fmt.Errorf("inspect analytics cache after manual sync: %w", err) + } + if !staleness.NeedsBuild { + return nil + } + if remaining, deferBuild := scheduledCacheBuildDelay( + staleness, cfg.Analytics.MinRebuildInterval, time.Now(), + ); deferBuild { + logger.Info("skipping cache rebuild after manual sync: minimum interval not elapsed", + "remaining", remaining.String(), "published_at", staleness.PublishedAt) + return nil + } + return rebuildCacheAfterWrite(dbPath, state) +} + // buildCacheSubprocess runs `msgvault build-cache` as a child process // instead of calling buildCache in-process. // @@ -2207,7 +2339,9 @@ func buildCacheSubprocess(ctx context.Context, fullRebuild, auto bool) error { func buildCacheSubprocessMode(ctx context.Context, mode buildCacheMode) error { // Serialize with each other so parallel per-account syncs in the // daemon don't spawn concurrent cache builds racing on shared files. - buildCacheMu.Lock() + if err := buildCacheMu.LockContext(ctx); err != nil { + return err + } defer buildCacheMu.Unlock() cmd, err := newBuildCacheSubprocessCommand(ctx, mode) @@ -2244,7 +2378,9 @@ func buildCacheSubprocessStream( fullRebuild, auto bool, emit func(api.CLICacheBuildEvent) error, ) error { - buildCacheMu.Lock() + if err := buildCacheMu.LockContext(ctx); err != nil { + return err + } defer buildCacheMu.Unlock() mode := buildCacheModeDefault @@ -2309,7 +2445,7 @@ func newBuildCacheSubprocessCommand(ctx context.Context, mode buildCacheMode) (* return nil, fmt.Errorf("locate msgvault executable: %w", err) } - args := globalConfigFlagArgs() + args := globalConfigFlagArgs(optionsFromContext(ctx)) args = append(args, "--no-log-file", "build-cache") switch mode { case buildCacheModeDefault: @@ -2328,7 +2464,7 @@ func newBuildCacheSubprocessCommand(ctx context.Context, mode buildCacheMode) (* // exe is this binary (os.Executable) and args are our own fixed subcommand // plus operator-controlled config flags, not untrusted input. cmd := exec.CommandContext(ctx, exe, args...) //nolint:gosec // exe is os.Executable; args are internally constructed - cmd.Env = buildCacheDaemonChildEnv(os.Environ(), os.Getpid()) + cmd.Env = daemonRuntimeChildEnv(ctx, buildCacheDaemonChildEnv(os.Environ(), os.Getpid())) return cmd, nil } @@ -2385,31 +2521,35 @@ func buildCacheDaemonChildEnv(base []string, parentPID int) []string { // globalConfigFlagArgs reconstructs the persistent flags that affect // configuration resolution so a child process loads the same config as // the running one. -func globalConfigFlagArgs() []string { +func globalConfigFlagArgs(options ...invocationOptions) []string { + if len(options) == 0 { + return nil + } + o := options[0] var args []string - if cfgFile != "" { - args = append(args, "--config", cfgFile) + if o.cfgFile != "" { + args = append(args, "--config", o.cfgFile) } - if homeDir != "" { - args = append(args, "--home", homeDir) + if o.homeDir != "" { + args = append(args, "--home", o.homeDir) } - if useLocal { + if o.useLocal { args = append(args, "--local") } // Forward the logging flags so an explicit level survives into subprocesses. // The daemon CLI subprocess otherwise quiets to WARN, defeating a user's // explicit --log-level/--verbose/--log-sql request. - if logLevel != "" { - args = append(args, "--log-level", logLevel) + if o.logLevel != "" { + args = append(args, "--log-level", o.logLevel) } - if verbose { + if o.verbose { args = append(args, "--verbose") } - if logSQL { + if o.logSQL { args = append(args, "--log-sql") } - if logSQLSlow != 0 { - args = append(args, "--log-sql-slow-ms", strconv.FormatInt(logSQLSlow, 10)) + if o.logSQLSlow != 0 { + args = append(args, "--log-sql-slow-ms", strconv.FormatInt(o.logSQLSlow, 10)) } return args } @@ -2419,7 +2559,17 @@ func globalConfigFlagArgs() []string { // PostgreSQL DSNs. The build runs in a subprocess (see buildCacheSubprocess) // to keep DuckDB's bundled SQLite library out of a long-lived daemon's // address space (issue #379). +// +// Readiness and throttle checks run in the background refresher, away from +// the scheduler's operation gate. The caller returns as soon as the request +// is queued; the refresher validates the committed shard fingerprint before +// deciding whether a recent publication can be throttled. func rebuildCacheAfterScheduledSync(ctx context.Context, identifier string) error { + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg if !cfg.Analytics.AutoBuildCache { // AutoBuildCache opts out of automatic daemon rebuilds, even when startup // selected a usable DuckDB cache; engine = "sql" is the live-data choice. @@ -2429,8 +2579,38 @@ func rebuildCacheAfterScheduledSync(ctx context.Context, identifier string) erro if store.IsPostgresURL(dbPath) { return nil } + if refresher := daemonCacheRefresher; refresher != nil { + refresher.Request(identifier) + return nil + } + return rebuildCacheNow(ctx, identifier, nil) +} + +// rebuildCacheNow runs the locked staleness check and, when a build is due +// and not throttled, the build subprocess. +func rebuildCacheNow( + ctx context.Context, + identifier string, + scheduleRetry func(time.Duration, string), +) error { + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := repairLogger(state.logger) + if !cfg.Analytics.AutoBuildCache { + return nil + } + dbPath := cfg.DatabaseDSN() + if store.IsPostgresURL(dbPath) { + return nil + } analyticsDir := cfg.AnalyticsDir() - staleness := cacheNeedsBuild(dbPath, analyticsDir) + staleness := cacheNeedsBuildContext(ctx, dbPath, analyticsDir) + if err := ctx.Err(); err != nil { + return err + } if !staleness.NeedsBuild { return nil } @@ -2444,6 +2624,11 @@ func rebuildCacheAfterScheduledSync(ctx context.Context, identifier string) erro "min_rebuild_interval", cfg.Analytics.MinRebuildInterval.String(), "published_at", staleness.PublishedAt, "remaining", remaining.String()) + if scheduleRetry != nil { + // Readiness was checked authoritatively off the operation gate. Keep + // one retry for the first point at which the publication can rebuild. + scheduleRetry(remaining, identifier) + } return nil } logger.Info("rebuilding cache after sync", @@ -2453,6 +2638,33 @@ func rebuildCacheAfterScheduledSync(ctx context.Context, identifier string) erro logger.Error("cache build failed", "error", err) return fmt.Errorf("refresh analytics cache: %w", err) } + staleness = cacheNeedsBuildContext(ctx, dbPath, analyticsDir) + if err := ctx.Err(); err != nil { + return err + } + if staleness.NeedsBuild { + if remaining, throttle := scheduledCacheBuildDelay( + staleness, + cfg.Analytics.MinRebuildInterval, + scheduledCacheBuildNow(), + ); throttle { + if scheduleRetry != nil { + logger.Info("cache rebuild was superseded by a newer publication; retrying after minimum interval", + "identifier", identifier, + "published_at", staleness.PublishedAt, + "remaining", remaining.String()) + scheduleRetry(remaining, identifier) + } else { + logger.Warn("cache remains stale under minimum rebuild interval; no retry scheduler is available", + "identifier", identifier, + "published_at", staleness.PublishedAt, + "remaining", remaining.String()) + } + return nil + } + logger.Warn("cache remains stale after scheduled rebuild", "reason", staleness.Reason) + return nil + } logger.Info("cache build completed") return nil } diff --git a/cmd/msgvault/cmd/build_cache_http_test.go b/cmd/msgvault/cmd/build_cache_http_test.go index 6bbe0482b..086e620f8 100644 --- a/cmd/msgvault/cmd/build_cache_http_test.go +++ b/cmd/msgvault/cmd/build_cache_http_test.go @@ -27,9 +27,9 @@ func TestBuildCacheAutostartFulfilledSkipsRedundantHTTPRequest(t *testing.T) { w.WriteHeader(http.StatusInternalServerError) })) t.Cleanup(server.Close) - stubBuildCacheDaemonAutostart(t, server, startupCacheBuildOutcomeFulfilled, nil) + testCtx := stubBuildCacheDaemonAutostart(t, server, startupCacheBuildOutcomeFulfilled, nil) - cmd, stdout := buildCacheHTTPTestCommand() + cmd, stdout := buildCacheHTTPTestCommand(testCtx) var err error captureStderrDuring(t, func() { err = runBuildCacheHTTP(cmd, false) @@ -53,9 +53,9 @@ func TestBuildCacheAutostartFailedReturnsErrorWithoutRetry(t *testing.T) { })) t.Cleanup(server.Close) logPath := filepath.Join(t.TempDir(), "serve.log") - stubBuildCacheDaemonAutostart(t, server, startupCacheBuildOutcomeFailed, &logPath) + testCtx := stubBuildCacheDaemonAutostart(t, server, startupCacheBuildOutcomeFailed, &logPath) - cmd, _ := buildCacheHTTPTestCommand() + cmd, _ := buildCacheHTTPTestCommand(testCtx) var err error captureStderrDuring(t, func() { err = runBuildCacheHTTP(cmd, false) @@ -78,9 +78,9 @@ func TestBuildCacheAutostartFatalDuckDBFailureDoesNotReportSQLFallback(t *testin })) t.Cleanup(server.Close) logPath := filepath.Join(t.TempDir(), "serve.log") - stubBuildCacheDaemonAutostart(t, server, startupCacheBuildOutcomeFatal, &logPath) + testCtx := stubBuildCacheDaemonAutostart(t, server, startupCacheBuildOutcomeFatal, &logPath) - cmd, _ := buildCacheHTTPTestCommand() + cmd, _ := buildCacheHTTPTestCommand(testCtx) var err error captureStderrDuring(t, func() { err = runBuildCacheHTTP(cmd, false) @@ -104,9 +104,9 @@ func TestBuildCacheAutostartUnconsumedUsesHTTPRequest(t *testing.T) { _, _ = w.Write([]byte(`{"type":"complete"}` + "\n")) })) t.Cleanup(server.Close) - stubBuildCacheDaemonAutostart(t, server, startupCacheBuildOutcomeUnconsumed, nil) + testCtx := stubBuildCacheDaemonAutostart(t, server, startupCacheBuildOutcomeUnconsumed, nil) - cmd, stdout := buildCacheHTTPTestCommand() + cmd, stdout := buildCacheHTTPTestCommand(testCtx) var err error captureStderrDuring(t, func() { err = runBuildCacheHTTP(cmd, false) @@ -123,10 +123,10 @@ func TestBuildCacheFullRebuildPassesFullStartupIntent(t *testing.T) { })) t.Cleanup(server.Close) var gotIntent startupCacheBuildIntent - stubBuildCacheDaemonAutostart(t, server, startupCacheBuildOutcomeFulfilled, nil, + testCtx := stubBuildCacheDaemonAutostart(t, server, startupCacheBuildOutcomeFulfilled, nil, func(intent startupCacheBuildIntent) { gotIntent = intent }) - cmd, _ := buildCacheHTTPTestCommand() + cmd, _ := buildCacheHTTPTestCommand(testCtx) var err error captureStderrDuring(t, func() { err = runBuildCacheHTTP(cmd, true) @@ -137,6 +137,8 @@ func TestBuildCacheFullRebuildPassesFullStartupIntent(t *testing.T) { } func TestBuildCacheUsesConfiguredRemoteHTTPAndPreservesOutput(t *testing.T) { + logger := testLoggerValue() + assert := assert.New(t) var requests atomic.Int32 @@ -155,7 +157,7 @@ func TestBuildCacheUsesConfiguredRemoteHTTPAndPreservesOutput(t *testing.T) { t.Cleanup(server.Close) dataDir := t.TempDir() - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ HomeDir: dataDir, Data: config.DataConfig{DataDir: dataDir}, Remote: config.RemoteConfig{ @@ -174,6 +176,7 @@ func TestBuildCacheUsesConfiguredRemoteHTTPAndPreservesOutput(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer cmd := &cobra.Command{Use: buildCacheCmd.Use, RunE: buildCacheCmd.RunE} + cmd.SetContext(testCtx) cmd.Flags().BoolVar(&fullRebuild, "full-rebuild", false, "Rebuild all cache files from scratch") cmd.SetArgs([]string{"--full-rebuild"}) cmd.SetOut(&stdout) @@ -210,7 +213,8 @@ func TestBuildCacheRunningLocalDaemonUsesSingleHTTPRequest(t *testing.T) { t.Cleanup(server.Close) dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + _ = testCtx rt := daemonRuntimeForHTTPServer(t, server, daemonAPIKeyFingerprint("")) _, err := daemonRuntimeStore(dataDir).Write(rt.Record) require.NoError(err, "write running daemon record") @@ -222,7 +226,7 @@ func TestBuildCacheRunningLocalDaemonUsesSingleHTTPRequest(t *testing.T) { return nil, errors.New("unreachable daemon restart") }) - cmd, stdout := buildCacheHTTPTestCommand() + cmd, stdout := buildCacheHTTPTestCommand(testCtx) err = runBuildCacheHTTP(cmd, false) require.NoError(err) @@ -230,10 +234,10 @@ func TestBuildCacheRunningLocalDaemonUsesSingleHTTPRequest(t *testing.T) { assert.Equal("Built through running daemon.\n", stdout.String()) } -func buildCacheHTTPTestCommand() (*cobra.Command, *bytes.Buffer) { +func buildCacheHTTPTestCommand(ctx context.Context) (*cobra.Command, *bytes.Buffer) { stdout := &bytes.Buffer{} cmd := &cobra.Command{Use: "build-cache"} - cmd.SetContext(context.Background()) + cmd.SetContext(ctx) cmd.SetOut(stdout) cmd.SetErr(&bytes.Buffer{}) return cmd, stdout @@ -245,10 +249,11 @@ func stubBuildCacheDaemonAutostart( outcome startupCacheBuildOutcome, logPathOverride *string, observeIntent ...func(startupCacheBuildIntent), -) { +) context.Context { t.Helper() dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + _ = testCtx waitCh := make(chan error) logPath := filepath.Join(dataDir, "serve.log") if logPathOverride != nil { @@ -274,4 +279,5 @@ func stubBuildCacheDaemonAutostart( rt.Record.Metadata[runtimeStartupCacheBuildOutcome] = string(outcome) return rt, true, nil }) + return testCtx } diff --git a/cmd/msgvault/cmd/build_cache_identity_test.go b/cmd/msgvault/cmd/build_cache_identity_test.go index 34bf23d86..52e40c5bc 100644 --- a/cmd/msgvault/cmd/build_cache_identity_test.go +++ b/cmd/msgvault/cmd/build_cache_identity_test.go @@ -28,7 +28,7 @@ func TestBuildCache_SlackDefaultIdentityResolvesOwnerParticipant(t *testing.T) { src, err := st.GetOrCreateSource("slack", "T01:UME") require.NoError(err) - confirmDefaultSlackIdentity(io.Discard, st, src.ID, "T01", "UME") + confirmDefaultSlackIdentity(io.Discard, st, src.ID, "T01", "UME", testDiscardLogger()) ownerID, err := st.EnsureParticipantByIdentifier("slack", "T01:UME", "Me") require.NoError(err) convID, err := st.EnsureConversationWithType(src.ID, "D01", "direct_chat", "Alice") @@ -71,6 +71,53 @@ func TestBuildCache_SlackDefaultIdentityResolvesOwnerParticipant(t *testing.T) { assert.True(t, isFromMe, "owner resolution must feed Slack relationship analytics") } +func TestBuildCache_FromRecipientChangeRefreshesOwner(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + dbPath := filepath.Join(t.TempDir(), "msgvault.db") + analyticsDir := filepath.Join(t.TempDir(), "analytics") + st, err := store.Open(dbPath) + require.NoError(err) + t.Cleanup(func() { _ = st.Close() }) + require.NoError(st.InitSchema()) + src, err := st.GetOrCreateSource("slack", "synthetic-owner") + require.NoError(err) + conv, err := st.EnsureConversationWithType(src.ID, "thread", "direct_chat", "Synthetic") + require.NoError(err) + first, err := st.EnsureParticipant("first@example.test", "First", "example.test") + require.NoError(err) + second, err := st.EnsureParticipant("second@example.test", "Second", "example.test") + require.NoError(err) + id, err := st.UpsertMessage(&store.Message{ + ConversationID: conv, SourceID: src.ID, SourceMessageID: "message", MessageType: "slack", + IsFromMe: true, SentAt: sql.NullTime{Time: time.Date(2025, 1, 1, 12, 0, 0, 0, time.UTC), Valid: true}, + }) + require.NoError(err) + require.NoError(st.ReplaceMessageRecipients(id, "from", []int64{first}, []string{"First"})) + _, err = buildCacheAuto(dbPath, analyticsDir) + require.NoError(err) + require.NoError(st.ReplaceMessageRecipients(id, "from", []int64{second}, []string{"Second"})) + var isFromMe bool + require.NoError(st.DB().QueryRow(`SELECT is_from_me FROM messages WHERE id = ?`, id).Scan(&isFromMe)) + require.True(isFromMe) + assert.True(cacheNeedsBuild(dbPath, analyticsDir).FullRebuild) + _, err = buildCacheAuto(dbPath, analyticsDir) + require.NoError(err) + db, err := sql.Open("duckdb", "") + require.NoError(err) + t.Cleanup(func() { _ = db.Close() }) + var owner int64 + require.NoError(db.QueryRow(`SELECT owner_participant_id FROM read_parquet(?, hive_partitioning=true) WHERE id = ?`, + filepath.Join(analyticsDir, "messages", "**", "*.parquet"), id).Scan(&owner)) + assert.Equal(second, owner) + var isSender, isOwner bool + require.NoError(db.QueryRow(`SELECT is_sender, is_owner FROM read_parquet(?, hive_partitioning=true) + WHERE message_id = ? AND canonical_id = ?`, + filepath.Join(analyticsDir, "relationship_activity", "**", "*.parquet"), id, second).Scan(&isSender, &isOwner)) + assert.True(isSender) + assert.True(isOwner) +} + // TestBuildCache_DerivesIsFromMeAndIdentityDatasets verifies that: // - messages Parquet gains a derived is_from_me column: true when the // sender's participant email case-insensitively matches a confirmed @@ -783,3 +830,95 @@ func TestBuildCache_OwnerParticipantsHonorPrimaryEmailGuard(t *testing.T) { assert.Equal(contactID, *counterpartsByTitle["Source-native sender before owner envelope"], "the matching envelope participant remains a counterpart for source-native attribution") } + +func TestSourceAttributionChangeRebuildsCachedOwner(t *testing.T) { + for _, upgrade := range []bool{false, true} { + name := "fresh" + if upgrade { + name = "upgrade" + } + t.Run(name, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + dbPath := filepath.Join(t.TempDir(), "msgvault.db") + analyticsDir := filepath.Join(t.TempDir(), "analytics") + st, err := store.OpenForTest(dbPath) + require.NoError(err) + t.Cleanup(func() { _ = st.Close() }) + require.NoError(st.InitSchema()) + src, err := st.GetOrCreateSource("test", "synthetic@example.test") + require.NoError(err) + conv, err := st.EnsureConversationWithType(src.ID, "thread", "email_thread", "Synthetic") + require.NoError(err) + sender, err := st.EnsureParticipant("sender@example.test", "Sender", "example.test") + require.NoError(err) + envelope, err := st.EnsureParticipant("envelope@example.test", "Envelope", "example.test") + require.NoError(err) + require.NoError(st.AddAccountIdentity(src.ID, "sender@example.test", "manual")) + require.NoError(st.AddAccountIdentity(src.ID, "envelope@example.test", "manual")) + msg := &store.Message{ConversationID: conv, SourceID: src.ID, + SourceMessageID: "message", MessageType: "email", + SenderID: sql.NullInt64{Int64: sender, Valid: true}, + SentAt: sql.NullTime{Time: time.Date(2025, 1, 1, 12, 0, 0, 0, time.UTC), Valid: true}} + id, err := st.PersistMessage(&store.MessagePersistData{Message: msg, + Recipients: []store.RecipientSet{{Type: "from", ParticipantIDs: []int64{envelope}, + DisplayNames: []string{"Envelope"}, EmailAddresses: []string{"envelope@example.test"}}}}) + require.NoError(err) + if upgrade { + _, err = st.DB().Exec(` + DROP TRIGGER trg_cache_message_facts_update; + CREATE TRIGGER trg_cache_message_facts_update + AFTER UPDATE OF sender_id, is_from_me, has_attachments, attachment_count + ON messages FOR EACH ROW + WHEN OLD.sender_id IS NOT NEW.sender_id OR OLD.is_from_me IS NOT NEW.is_from_me + OR OLD.has_attachments IS NOT NEW.has_attachments + OR OLD.attachment_count IS NOT NEW.attachment_count BEGIN + INSERT INTO cache_related_change_journal (dataset, message_id) + VALUES ('message_facts', NEW.id); + END; + DELETE FROM applied_migrations WHERE name = 'cache_message_source_attribution'; + `) + require.NoError(err) + } + _, err = buildCache(dbPath, analyticsDir, true) + require.NoError(err) + db, err := sql.Open("duckdb", "") + require.NoError(err) + t.Cleanup(func() { _ = db.Close() }) + var cachedOwner int64 + require.NoError(db.QueryRow(`SELECT owner_participant_id FROM read_parquet(?, hive_partitioning=true) WHERE id = ?`, + filepath.Join(analyticsDir, "messages", "**", "*.parquet"), id).Scan(&cachedOwner)) + require.Equal(envelope, cachedOwner) + msg.IsFromMe = true + _, err = st.UpsertMessage(msg) + require.NoError(err) + var sourceNative, effective bool + require.NoError(st.DB().QueryRow(`SELECT source_is_from_me, is_from_me FROM messages WHERE id = ?`, id).Scan(&sourceNative, &effective)) + require.True(sourceNative) + require.True(effective) + if upgrade { + require.False(cacheNeedsBuild(dbPath, analyticsDir).NeedsBuild, + "legacy trigger misses source-only attribution changes") + require.NoError(st.InitSchema()) + } + stale := cacheNeedsBuild(dbPath, analyticsDir) + assert.True(stale.NeedsBuild) + assert.True(stale.FullRebuild) + automatic, err := buildCacheAuto(dbPath, analyticsDir) + require.NoError(err) + assert.False(automatic.Skipped) + var refreshedOwner int64 + require.NoError(db.QueryRow(`SELECT owner_participant_id FROM read_parquet(?, hive_partitioning=true) WHERE id = ?`, + filepath.Join(analyticsDir, "messages", "**", "*.parquet"), id).Scan(&refreshedOwner)) + assert.Equal(sender, refreshedOwner) + require.NoError(st.InitSchema()) + assert.False(cacheNeedsBuild(dbPath, analyticsDir).NeedsBuild, + "reopening an upgraded archive must not invalidate its cache again") + msg.IsFromMe = false + _, err = st.UpsertMessage(msg) + require.NoError(err) + assert.True(cacheNeedsBuild(dbPath, analyticsDir).FullRebuild, + "the installed trigger must also capture later source attribution edits") + }) + } +} diff --git a/cmd/msgvault/cmd/build_cache_test.go b/cmd/msgvault/cmd/build_cache_test.go index 4e087497d..1c9ea6866 100644 --- a/cmd/msgvault/cmd/build_cache_test.go +++ b/cmd/msgvault/cmd/build_cache_test.go @@ -8,6 +8,7 @@ import ( "encoding/json" "errors" "fmt" + "io/fs" "net/url" "os" "os/exec" @@ -24,6 +25,7 @@ import ( "github.com/stretchr/testify/require" "go.kenn.io/msgvault/internal/identityindex" "go.kenn.io/msgvault/internal/query" + "go.kenn.io/msgvault/internal/search" ) func TestDaemonBuildCacheChildUsesQuietConsolePolicy(t *testing.T) { @@ -320,10 +322,12 @@ func enableSQLiteWAL(t *testing.T, dbPath string) { // account_identities before that ingest's confirmDefaultIdentity would // suppress the source's own address. func TestRunBuildCacheLocalSkipsDeferredIdentityMigration(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) c, s := openTestDaemonAnalyticsStore(t) - withTUIConfig(t, c) + testCtx := withTUIConfig(t, c) cfg.Identity.Addresses = []string{"legacy@example.com"} _, err := s.DB().Exec(` INSERT INTO sources (id, source_type, identifier) VALUES (1, 'gmail', 'user@example.com'); @@ -334,7 +338,10 @@ func TestRunBuildCacheLocalSkipsDeferredIdentityMigration(t *testing.T) { `) require.NoError(err, "insert test data") - require.NoError(runBuildCacheLocal(false, false), "runBuildCacheLocal") + require.NoError( + runBuildCacheLocal(false, false, invocationFromContext(testCtx)), + "runBuildCacheLocal", + ) var identities int require.NoError(s.DB().QueryRow("SELECT COUNT(*) FROM account_identities").Scan(&identities)) @@ -650,7 +657,7 @@ func TestBuildCacheFailedStateWriteLeavesIncrementalDrift(t *testing.T) { return errors.New("simulated state write failure") } _, err = buildCache(dbPath, analyticsDir, false) - buildCacheWriteStateFile = os.WriteFile + buildCacheWriteStateFile = writeCacheStateFile require.ErrorContains(err, "simulated state write failure", "incremental build must fail when the sync state cannot be persisted") @@ -686,7 +693,7 @@ func TestBuildCacheFailedStateWriteLeavesFullRebuildDrift(t *testing.T) { return errors.New("simulated state write failure") } _, err = buildCache(dbPath, analyticsDir, true) - buildCacheWriteStateFile = os.WriteFile + buildCacheWriteStateFile = writeCacheStateFile require.ErrorContains(err, "simulated state write failure", "full rebuild must fail when the sync state cannot be persisted") @@ -1482,7 +1489,7 @@ func runBuildCacheSQLiteMutation(t *testing.T, dbPath, operation string) { require.NoErrorf(t, err, "run concurrent SQLite mutation\noutput:\n%s", output) } -func TestBuildCache_RejectsTerminalAdditionDuringExport(t *testing.T) { +func TestBuildCache_PublishesSnapshotWhenTerminalAdditionLandsDuringExport(t *testing.T) { for _, terminalStatus := range []string{"completed", "failed"} { t.Run(terminalStatus, func(t *testing.T) { require := require.New(t) @@ -1547,14 +1554,44 @@ func TestBuildCache_RejectsTerminalAdditionDuringExport(t *testing.T) { t.Cleanup(func() { buildCacheBeforeStateWriteHook = nil }) _, err = buildCache(dbPath, analyticsDir, false) - require.Error(err) - assert.Contains(err.Error(), "sync counters changed during cache export") + require.NoError(err, "a counter change during export publishes the snapshot") buildCacheBeforeStateWriteHook = nil stateAfter, readErr := os.ReadFile(query.CacheStatePath(analyticsDir)) require.NoError(readErr) - assert.Equal(stateBefore, stateAfter, "counter mismatch preserves committed state") - assert.Equal(filesBefore, snapshotCacheParquet(t, analyticsDir), - "counter mismatch preserves committed Parquet") + assert.NotEqual(stateBefore, stateAfter, "snapshot publication replaces committed state") + assert.NotEqual(filesBefore, snapshotCacheParquet(t, analyticsDir), + "snapshot publication includes the parent message") + published, readErr := query.ReadCacheSyncState(analyticsDir) + require.NoError(readErr) + assert.True(published.FullRebuildRequired, "partial snapshot forces the next build to be full") + stale := cacheNeedsBuild(dbPath, analyticsDir) + require.True(stale.NeedsBuild, "partial snapshot is stale: %+v", stale) + require.True(stale.FullRebuild, "partial snapshot needs a full rebuild: %+v", stale) + + // A sync that overlaps this build also queues another automatic + // refresh. It must wait even though the next build must be full. + deferred, err := buildCacheScheduled(dbPath, analyticsDir, 6*time.Hour, func() time.Time { + return published.PublishedAt.Add(time.Hour) + }) + require.NoError(err) + assert.True(deferred.Skipped, "partial publication must honor the rebuild interval") + + // A newer message arriving first must not turn the repair into an + // incremental build above the parent, which would skip its children. + db, err = sql.Open("sqlite3", dbPath) + require.NoError(err) + _, err = db.Exec(` + INSERT INTO messages ( + id, source_id, source_message_id, conversation_id, + subject, snippet, sent_at, size_estimate, message_type + ) VALUES ( + 7, 1, 'meeting-later', 105, + 'Later Meeting', 'Arrived after the partial snapshot', + '2026-07-13 10:00:00', 500, 'meeting_transcript' + ); + `) + require.NoError(err) + require.NoError(db.Close()) _, err = buildCache(dbPath, analyticsDir, false) require.NoError(err) @@ -1579,7 +1616,7 @@ func TestBuildCache_RejectsTerminalAdditionDuringExport(t *testing.T) { } } -func TestBuildCache_RejectsZeroCounterFailedRunDuringExport(t *testing.T) { +func TestBuildCache_PublishesSnapshotWhenZeroCounterRunFailsDuringExport(t *testing.T) { require := require.New(t) assert := assert.New(t) tmpDir := setupTestSQLite(t) @@ -1612,7 +1649,6 @@ func TestBuildCache_RejectsZeroCounterFailedRunDuringExport(t *testing.T) { require.NoError(err) stateBefore, err := os.ReadFile(query.CacheStatePath(analyticsDir)) require.NoError(err) - filesBefore := snapshotCacheParquet(t, analyticsDir) buildCacheBeforeStateWriteHook = func() { hookDB, hookErr := sql.Open("sqlite3", dbPath) @@ -1628,14 +1664,14 @@ func TestBuildCache_RejectsZeroCounterFailedRunDuringExport(t *testing.T) { t.Cleanup(func() { buildCacheBeforeStateWriteHook = nil }) _, err = buildCache(dbPath, analyticsDir, true) - require.Error(err) - assert.Contains(err.Error(), "sync counters changed during cache export") + require.NoError(err, "a counter change during export publishes the snapshot") buildCacheBeforeStateWriteHook = nil stateAfter, readErr := os.ReadFile(query.CacheStatePath(analyticsDir)) require.NoError(readErr) - assert.Equal(stateBefore, stateAfter, "counter mismatch preserves committed state") - assert.Equal(filesBefore, snapshotCacheParquet(t, analyticsDir), - "counter mismatch preserves committed Parquet") + assert.NotEqual(stateBefore, stateAfter, "snapshot publication replaces committed state") + stale := cacheNeedsBuild(dbPath, analyticsDir) + assert.True(stale.NeedsBuild, "run failing during export leaves the cache stale: %+v", stale) + assert.True(stale.FullRebuild, "partial snapshot needs a full rebuild: %+v", stale) } func TestCacheNeedsBuild_DetectsOlderRunFailingAfterNewerFailure(t *testing.T) { @@ -1688,7 +1724,7 @@ func TestCacheNeedsBuild_DetectsOlderRunFailingAfterNewerFailure(t *testing.T) { assert.Contains(staleness.Reason, "failed sync") } -func TestBuildCache_RejectsOlderRunFailingDuringExport(t *testing.T) { +func TestBuildCache_PublishesSnapshotWhenOlderRunFailsDuringExport(t *testing.T) { require := require.New(t) assert := assert.New(t) tmpDir := setupTestSQLite(t) @@ -1736,8 +1772,16 @@ func TestBuildCache_RejectsOlderRunFailingDuringExport(t *testing.T) { t.Cleanup(func() { buildCacheBeforeStateWriteHook = nil }) _, err = buildCache(dbPath, analyticsDir, true) - require.Error(err) - assert.Contains(err.Error(), "sync counters changed during cache export") + require.NoError(err, "a counter change during export publishes the snapshot") + buildCacheBeforeStateWriteHook = nil + stale := cacheNeedsBuild(dbPath, analyticsDir) + assert.True(stale.NeedsBuild, "older run failing during export leaves the cache stale: %+v", stale) + assert.True(stale.FullRebuild, "partial snapshot needs a full rebuild: %+v", stale) + + _, err = buildCache(dbPath, analyticsDir, false) + require.NoError(err) + fresh := cacheNeedsBuild(dbPath, analyticsDir) + assert.False(fresh.NeedsBuild, "full follow-up build converges: %+v", fresh) } func TestCacheNeedsBuild_AddOnlySyncUsesIncrementalBuild(t *testing.T) { @@ -2283,73 +2327,593 @@ func TestBuildCache_UTF8Handling(t *testing.T) { assert.Equal("Test émoji 🎉 and unicode", subject, "unicode should be preserved") } -func TestBuildCacheCSVInvalidUTF8ExplainsRepairPath(t *testing.T) { +func TestBuildCacheCSVInvalidUTF8IsRepairedWithWarning(t *testing.T) { + for _, tc := range []struct{ name, setup, check string }{ + {"attachment filename", `UPDATE attachments SET filename = CAST(X'80' AS TEXT) WHERE id = 1`, "SELECT filename FROM read_parquet('%s/attachments/*.parquet') WHERE attachment_id = 1"}, + {"row past CSV sample", `INSERT INTO messages (source_id, source_message_id, sent_at) + WITH RECURSIVE seq(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM seq WHERE i < 30000) + SELECT 1, 'bulk-' || i, datetime('2024-04-01', '+' || i || ' minutes') FROM seq; + UPDATE messages SET subject = CAST(X'80' AS TEXT) WHERE id = (SELECT MAX(id) FROM messages)`, + "SELECT subject FROM read_parquet('%s/messages/**/*.parquet', hive_partitioning=true) ORDER BY id DESC LIMIT 1"}, + } { + t.Run(tc.name, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + t.Setenv("MSGVAULT_FORCE_CSV_SNAPSHOT", "1") + tmpDir := setupTestSQLite(t) + dbPath, analyticsDir := filepath.Join(tmpDir, "test.db"), filepath.Join(tmpDir, "analytics") + db, err := sql.Open("sqlite3", dbPath) + require.NoError(err) + _, err = db.Exec(tc.setup) + require.NoError(err) + require.NoError(db.Close()) + var buildErr error + stderr := captureStderrDuring(t, func() { _, buildErr = buildCache(dbPath, analyticsDir, true) }) + require.NoError(buildErr) + assert.Contains(stderr, "msgvault repair-encoding") + duckDB, err := sql.Open("duckdb", "") + require.NoError(err) + defer func() { _ = duckDB.Close() }() + var got string + require.NoError(duckDB.QueryRow(fmt.Sprintf(tc.check, filepath.ToSlash(analyticsDir))).Scan(&got)) + assert.Equal("�", got) + }) + } +} + +func TestBuildCacheSQLiteScannerInvalidUTF8KeepsFastSearchWorking(t *testing.T) { require := require.New(t) assert := assert.New(t) - t.Setenv("MSGVAULT_FORCE_CSV_SNAPSHOT", "1") + oldGOOS := cacheSnapshotGOOS + cacheSnapshotGOOS = "linux" + t.Cleanup(func() { cacheSnapshotGOOS = oldGOOS }) + t.Setenv("MSGVAULT_FORCE_CSV_SNAPSHOT", "") + probe, err := sql.Open("duckdb", "") + require.NoError(err) + requireSQLiteScanner(t, probe) + require.NoError(probe.Close()) tmpDir := setupTestSQLite(t) - dbPath := filepath.Join(tmpDir, "test.db") - + dbPath, analyticsDir := filepath.Join(tmpDir, "test.db"), filepath.Join(tmpDir, "analytics") db, err := sql.Open("sqlite3", dbPath) require.NoError(err) - _, err = db.Exec(`UPDATE attachments SET filename = CAST(X'80' AS TEXT) WHERE id = 1`) + _, err = db.Exec(`UPDATE messages SET snippet = CAST(X'43616c656e6461723a206c756e636820f09f' AS TEXT) WHERE id = 1`) require.NoError(err) require.NoError(db.Close()) + var buildErr error + stderr := captureStderrDuring(t, func() { _, buildErr = buildCache(dbPath, analyticsDir, true) }) + require.NoError(buildErr) + assert.NotContains(stderr, "using CSV fallback") + assert.Contains(stderr, "msgvault repair-encoding") + engine, err := query.NewDuckDBEngine(analyticsDir, "", nil) + require.NoError(err) + defer func() { _ = engine.Close() }() + q := search.Parse("from:alice@example.com") + result, err := engine.SearchFastWithStats(context.Background(), q, "from:alice@example.com", query.MessageFilter{}, query.ViewSenders, 50, 0) + require.NoError(err) + var snippet string + for _, m := range result.Messages { + if m.ID == 1 { + snippet = m.Snippet + } + } + assert.Equal("Calendar: lunch ��", snippet) +} + +func TestBuildCacheInvalidUTF8IdentityKeysNeverMatch(t *testing.T) { + for _, tc := range []struct { + name string + forceCSV bool + }{ + {name: "sqlite scanner"}, + {name: "CSV fallback", forceCSV: true}, + } { + t.Run(tc.name, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + oldGOOS := cacheSnapshotGOOS + cacheSnapshotGOOS = "linux" + t.Cleanup(func() { cacheSnapshotGOOS = oldGOOS }) + if tc.forceCSV { + t.Setenv("MSGVAULT_FORCE_CSV_SNAPSHOT", "1") + } else { + t.Setenv("MSGVAULT_FORCE_CSV_SNAPSHOT", "") + probe, err := sql.Open("duckdb", "") + require.NoError(err) + requireSQLiteScanner(t, probe) + require.NoError(probe.Close()) + } + + tmpDir := setupTestSQLite(t) + dbPath, analyticsDir := filepath.Join(tmpDir, "test.db"), filepath.Join(tmpDir, "analytics") + db, err := sql.Open("sqlite3", dbPath) + require.NoError(err) + + // Byte-identical invalid keys plus distinct invalid bytes that a + // U+FFFD repair would collapse onto one key: neither may attribute. + for _, address := range []string{ + "envelope\x80@example.com", "primary\x80@example.com", + "identifier\x80@example.com", "handle\x80", + "a\x80@example.com", + } { + _, err = db.Exec(`INSERT INTO account_identities (source_id, address) VALUES (1, CAST(? AS TEXT))`, []byte(address)) + require.NoError(err) + } + _, err = db.Exec(`UPDATE message_recipients SET email_address = CAST(? AS TEXT) + WHERE message_id = 1 AND recipient_type = 'from'`, []byte("envelope\x80@example.com")) + require.NoError(err) + // a\x81 and a\x80 sanitize to the same U+FFFD string; the + // envelope-equality path must not match them. + _, err = db.Exec(`UPDATE message_recipients SET email_address = CAST(? AS TEXT) + WHERE message_id = 2 AND recipient_type = 'from'`, []byte("a\x81@example.com")) + require.NoError(err) + _, err = db.Exec(`UPDATE participants SET email_address = CAST(? AS TEXT) WHERE id = 4`, []byte("primary\x80@example.com")) + require.NoError(err) + _, err = db.Exec(`UPDATE participants SET email_address = '' WHERE id = 2`) + require.NoError(err) + _, err = db.Exec(`UPDATE participant_identifiers SET identifier_value = CAST(? AS TEXT) + WHERE participant_id = 2`, []byte("identifier\x80@example.com")) + require.NoError(err) + _, err = db.Exec(`INSERT INTO participant_identifiers (participant_id, identifier_type, identifier_value) + VALUES (3, 'handle', CAST(? AS TEXT))`, []byte("handle\x80")) + require.NoError(err) + // Distinct invalid bytes against the a\x80 identity exercise the + // primary-email and non-email identifier paths. + _, err = db.Exec(`INSERT INTO participants (id, email_address, domain) + VALUES (5, CAST(? AS TEXT), 'example.com')`, []byte("a\x81@example.com")) + require.NoError(err) + _, err = db.Exec(`INSERT INTO participant_identifiers (participant_id, identifier_type, identifier_value) + VALUES (5, 'handle', CAST(? AS TEXT))`, []byte("a\x81@example.com")) + require.NoError(err) + _, err = db.Exec(`UPDATE messages SET sender_id = CASE id WHEN 3 THEN 3 WHEN 4 THEN 4 WHEN 5 THEN 2 END + WHERE id IN (3, 4, 5)`) + require.NoError(err) + // Reviewer scenario: the From envelope exists but its bytes are + // invalid, while the sender participant carries a VALID primary + // address that byte-equals a valid account identity. The recorded + // envelope is authoritative, so the participant fallback must not + // reclassify the message as sent by the owner. + _, err = db.Exec(`INSERT INTO participants (id, email_address, domain) + VALUES (6, 'owner@example.com', 'example.com')`) + require.NoError(err) + _, err = db.Exec(`INSERT INTO account_identities (source_id, address) VALUES (1, 'owner@example.com')`) + require.NoError(err) + // A nonblank primary address must suppress email-alias fallback, + // even when its invalid bytes cannot be used as a matching key. + _, err = db.Exec(`INSERT INTO participant_identifiers (participant_id, identifier_type, identifier_value) + VALUES (4, 'email', 'owner@example.com')`) + require.NoError(err) + _, err = db.Exec(`INSERT INTO messages (id, source_id, source_message_id, conversation_id, subject, snippet, sent_at, size_estimate, sender_id) + VALUES (6, 1, 'msg6', 101, 'Damaged envelope', 'Preview 6', '2024-03-05 09:00:00', 400, 6)`) + require.NoError(err) + _, err = db.Exec(`INSERT INTO message_recipients (message_id, participant_id, recipient_type, display_name, email_address) VALUES + (6, 6, 'from', 'Owner Account', CAST(? AS TEXT))`, []byte("owner-damaged\x80@example.com")) + require.NoError(err) + // Control: the same sender without any recorded envelope still + // attributes through the participant fallback with valid keys. + _, err = db.Exec(`INSERT INTO messages (id, source_id, source_message_id, conversation_id, subject, snippet, sent_at, size_estimate, sender_id) + VALUES (7, 1, 'msg7', 101, 'No envelope', 'Preview 7', '2024-03-06 09:00:00', 400, 6)`) + require.NoError(err) + _, err = db.Exec(`INSERT INTO message_recipients (message_id, participant_id, recipient_type, display_name) VALUES + (7, 6, 'from', 'Owner Account')`) + require.NoError(err) + messageIDs := []struct { + raw string + want sql.NullString + }{ + {"broken-\xff@example.test", sql.NullString{}}, + {"broken-\xfe@example.test", sql.NullString{}}, + {"broken-\uFFFD@example.test", sql.NullString{String: "broken-\uFFFD@example.test", Valid: true}}, + } + for i, messageID := range messageIDs { + _, err = db.Exec(`UPDATE messages SET rfc822_message_id = ? WHERE id = ?`, messageID.raw, i+1) + require.NoError(err) + } + require.NoError(db.Close()) - _, err = buildCache(dbPath, filepath.Join(tmpDir, "analytics"), true) - require.Error(err) - assert.Contains(err.Error(), "msgvault repair-encoding") - assert.Contains(err.Error(), "not a msgvault option") + _, err = buildCache(dbPath, analyticsDir, true) + require.NoError(err) + + duckDB, err := sql.Open("duckdb", "") + require.NoError(err) + defer func() { _ = duckDB.Close() }() + rows, err := duckDB.Query(`SELECT id, is_from_me, owner_participant_id + FROM read_parquet(?, hive_partitioning=true) ORDER BY id`, filepath.Join(analyticsDir, "messages", "**", "*.parquet")) + require.NoError(err) + defer func() { _ = rows.Close() }() + type attribution struct { + fromMe bool + owner sql.NullInt64 + } + got := make(map[int]attribution) + for rows.Next() { + var id int + var a attribution + require.NoError(rows.Scan(&id, &a.fromMe, &a.owner)) + got[id] = a + } + require.NoError(rows.Err()) + // The fixture messages carry no source_is_from_me and default + // is_from_me to false, so attribution can only come from the + // invalid keys above. owner_participant_id still follows the + // export's COALESCE fallback: sender_id first, else the single + // from-recipient. + assert.Equal(attribution{false, sql.NullInt64{Int64: 1, Valid: true}}, got[1], "invalid envelope key never matches") + assert.Equal(attribution{false, sql.NullInt64{Int64: 1, Valid: true}}, got[2], "distinct invalid envelope bytes never match") + assert.Equal(attribution{false, sql.NullInt64{Int64: 3, Valid: true}}, got[3], "invalid non-email identifier key never matches") + assert.Equal(attribution{false, sql.NullInt64{Int64: 4, Valid: true}}, got[4], + "an invalid nonblank primary email still blocks a valid matching email alias") + assert.Equal(attribution{false, sql.NullInt64{Int64: 2, Valid: true}}, got[5], "invalid email identifier key never matches") + assert.Equal(attribution{false, sql.NullInt64{Int64: 6, Valid: true}}, got[6], + "a present-but-invalid envelope is authoritative: the participant fallback must not attribute the owner's address") + assert.Equal(attribution{true, sql.NullInt64{Int64: 6, Valid: true}}, got[7], + "an absent envelope with valid keys still attributes via the participant fallback") + for i, messageID := range messageIDs { + var gotID sql.NullString + err = duckDB.QueryRow(`SELECT rfc822_message_id FROM read_parquet(?, hive_partitioning=true) WHERE id = ?`, + filepath.Join(analyticsDir, "messages", "**", "*.parquet"), i+1).Scan(&gotID) + require.NoError(err) + assert.Equal(messageID.want, gotID, "invalid IDs must not collapse onto a valid replacement-character ID") + } + + // The damaged from-row exports no address at all: substituting the + // participant's current address would mis-attribute the message. + var resolvedAddr, envelopeAddr sql.NullString + err = duckDB.QueryRow(`SELECT email_address, envelope_address FROM read_parquet(?) + WHERE message_id = 6 AND recipient_type = 'from'`, + filepath.Join(analyticsDir, "message_recipients", "*.parquet")).Scan(&resolvedAddr, &envelopeAddr) + require.NoError(err) + assert.False(resolvedAddr.Valid, "a present-but-invalid envelope must not resolve to the participant address") + assert.False(envelopeAddr.Valid, "a present-but-invalid envelope exports NULL") + + ownerRows, err := duckDB.Query(`SELECT participant_id FROM read_parquet(?) WHERE source_id = 1 ORDER BY participant_id`, + filepath.Join(analyticsDir, "owner_participants", "*.parquet")) + require.NoError(err) + defer func() { _ = ownerRows.Close() }() + var owners []int64 + for ownerRows.Next() { + var id int64 + require.NoError(ownerRows.Scan(&id)) + owners = append(owners, id) + } + require.NoError(ownerRows.Err()) + assert.Equal([]int64{6}, owners, + "only the valid identity's participant is an owner candidate; invalid identity keys never populate owner_participants") + }) + } } -func TestBuildCacheCSVInvalidUTF8InUnrepairedFieldScopesGuidance(t *testing.T) { - require := require.New(t) - assert := assert.New(t) - t.Setenv("MSGVAULT_FORCE_CSV_SNAPSHOT", "1") - tmpDir := setupTestSQLite(t) - dbPath := filepath.Join(tmpDir, "test.db") +func TestBuildCacheInvalidIdentifiersDoNotMatchValidKeys(t *testing.T) { + for _, tc := range []struct{ name, forceCSV string }{{"sqlite scanner", ""}, {"CSV fallback", "1"}} { + t.Run(tc.name, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + oldGOOS := cacheSnapshotGOOS + cacheSnapshotGOOS = "linux" + t.Cleanup(func() { cacheSnapshotGOOS = oldGOOS }) + t.Setenv("MSGVAULT_FORCE_CSV_SNAPSHOT", tc.forceCSV) + if tc.forceCSV == "" { + probe, err := sql.Open("duckdb", "") + require.NoError(err) + requireSQLiteScanner(t, probe) + require.NoError(probe.Close()) + } + tmpDir := setupTestSQLite(t) + dbPath, analyticsDir := filepath.Join(tmpDir, "test.db"), filepath.Join(tmpDir, "analytics") + db, err := sql.Open("sqlite3", dbPath) + require.NoError(err) + _, err = db.Exec(` + UPDATE messages SET source_message_id = CAST(X'80' AS TEXT) WHERE id = 1; + UPDATE messages SET source_message_id = CAST(X'81' AS TEXT) WHERE id = 2; + UPDATE messages SET source_message_id = '�' WHERE id = 3; + UPDATE messages SET list_id = CAST(X'80' AS TEXT) WHERE id = 1; + UPDATE messages SET list_id = CAST(X'81' AS TEXT) WHERE id = 2; + UPDATE messages SET list_id = '�' WHERE id = 3; + UPDATE conversations SET source_conversation_id = CAST(X'80' AS TEXT) WHERE id = 101; + UPDATE conversations SET source_conversation_id = CAST(X'81' AS TEXT) WHERE id = 102; + UPDATE conversations SET source_conversation_id = '�' WHERE id = 103; + UPDATE sources SET identifier = CAST(X'80' AS TEXT) WHERE id = 1; + INSERT INTO sources (id, identifier) VALUES (2, CAST(X'81' AS TEXT)), (3, '�'); + `) + require.NoError(err) + require.NoError(db.Close()) + var buildErr error + stderr := captureStderrDuring(t, func() { _, buildErr = buildCache(dbPath, analyticsDir, true) }) + require.NoError(buildErr) + assert.Contains(stderr, "8 identity value(s) exported as unknown") + assert.NotContains(stderr, "repair(s) applied") + + engine, err := query.NewDuckDBEngine(analyticsDir, "", nil) + require.NoError(err) + defer func() { _ = engine.Close() }() + for _, key := range []struct { + table, column, ids string + validID int64 + want [][]any + }{ + {"messages", "source_message_id", "1,2,3", 3, [][]any{{int64(1), ""}, {int64(2), ""}, {int64(3), "�"}}}, + {"messages", "list_id", "1,2,3", 3, [][]any{{int64(1), nil}, {int64(2), nil}, {int64(3), "�"}}}, + {"conversations", "source_conversation_id", "101,102,103", 103, [][]any{{int64(101), ""}, {int64(102), ""}, {int64(103), "�"}}}, + {"sources", "account_email", "1,2,3", 3, [][]any{{int64(1), ""}, {int64(2), ""}, {int64(3), "�"}}}, + } { + got, err := engine.QuerySQL(t.Context(), fmt.Sprintf( + "SELECT id, %s FROM %s WHERE id IN (%s) ORDER BY id", key.column, key.table, key.ids)) + require.NoError(err) + assert.Equal(key.want, got.Rows, key.table) + matches, err := engine.QuerySQL(t.Context(), fmt.Sprintf( + "SELECT id FROM %s WHERE %s = '�' ORDER BY id", key.table, key.column)) + require.NoError(err) + assert.Equal([][]any{{key.validID}}, matches.Rows, key.table+" must not add false identifier matches") + } + messages, err := engine.ListMessages(t.Context(), query.MessageFilter{ListID: "�"}) + require.NoError(err) + require.Len(messages, 1, "damaged List-IDs must not match a valid mailing list") + assert.Equal(int64(3), messages[0].ID) + }) + } +} + +func TestBuildCacheCountsOnlyRepairedExportValues(t *testing.T) { + for _, tc := range []struct { + name string + forceCSV bool + }{ + {name: "sqlite scanner"}, + {name: "CSV fallback", forceCSV: true}, + } { + t.Run(tc.name, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + oldGOOS := cacheSnapshotGOOS + cacheSnapshotGOOS = "linux" + t.Cleanup(func() { cacheSnapshotGOOS = oldGOOS }) + if tc.forceCSV { + t.Setenv("MSGVAULT_FORCE_CSV_SNAPSHOT", "1") + } else { + t.Setenv("MSGVAULT_FORCE_CSV_SNAPSHOT", "") + probe, err := sql.Open("duckdb", "") + require.NoError(err) + requireSQLiteScanner(t, probe) + require.NoError(probe.Close()) + } + tmpDir := setupTestSQLite(t) + dbPath, analyticsDir := filepath.Join(tmpDir, "test.db"), filepath.Join(tmpDir, "analytics") + db, err := sql.Open("sqlite3", dbPath) + require.NoError(err) + _, err = db.Exec(`INSERT INTO participants (id, email_address, domain, display_name) + VALUES (6, CAST(? AS TEXT), 'example.com', CAST(? AS TEXT))`, + []byte("primary\x80@example.com"), []byte("Display\x80Name")) + require.NoError(err) + _, err = db.Exec(`UPDATE messages SET sender_id = 6 WHERE id = 4`) + require.NoError(err) + _, err = db.Exec(`INSERT INTO account_identities (source_id, address) VALUES (1, ?), (1, CAST(? AS TEXT))`, + "primary�@example.com", []byte("unmatched\x80@example.com")) + require.NoError(err) + // One envelope contributes two unknown output columns. One person + // name is exported once for each of its two linked participants. + _, err = db.Exec(`UPDATE message_recipients SET email_address = CAST(X'80' AS TEXT) + WHERE message_id = 1 AND recipient_type = 'from'`) + require.NoError(err) + _, err = db.Exec(`INSERT INTO persons (id, vcard_uid, display_name) + VALUES (1, '00000000-0000-0000-0000-000000000001', CAST(X'80' AS TEXT)); + INSERT INTO person_participants (person_id, participant_id) VALUES (1, 1), (1, 2)`) + require.NoError(err) + require.NoError(db.Close()) + + var buildErr error + stderr := captureStderrDuring(t, func() { _, buildErr = buildCache(dbPath, analyticsDir, true) }) + require.NoError(buildErr) + assert.Contains(stderr, "Warning: 3 invalid UTF-8 repair(s) applied while building the analytics cache", + "count each exported display value, including the two joined person-name rows") + assert.Contains(stderr, "3 identity value(s) exported as unknown") + + duckDB, err := sql.Open("duckdb", "") + require.NoError(err) + defer func() { _ = duckDB.Close() }() + var fromMe bool + err = duckDB.QueryRow(`SELECT is_from_me FROM read_parquet(?, hive_partitioning=true) WHERE id = 4`, + filepath.Join(analyticsDir, "messages", "**", "*.parquet")).Scan(&fromMe) + require.NoError(err) + assert.False(fromMe, "a sanitized-literal identity must not match an invalid participant email") + var emailAddress, displayName string + err = duckDB.QueryRow(`SELECT email_address, display_name FROM read_parquet(?) WHERE id = 6`, + filepath.Join(analyticsDir, "participants", "*.parquet")).Scan(&emailAddress, &displayName) + require.NoError(err) + assert.Empty(emailAddress, "invalid identity text exports as unknown, not as a repair") + assert.Equal("Display\uFFFDName", displayName, "invalid display text is repaired") + }) + } +} + +func corruptEveryExportedTextColumn(t *testing.T, dbPath string) { + t.Helper() db, err := sql.Open("sqlite3", dbPath) - require.NoError(err) - _, err = db.Exec(`UPDATE messages SET source_message_id = CAST(X'80' AS TEXT) WHERE id = 1`) - require.NoError(err) - require.NoError(db.Close()) + require.NoError(t, err) + defer func() { _ = db.Close() }() + bad := func(col string) string { + return col + " = CAST(CAST(COALESCE(" + col + ", '') AS BLOB) || X'80' AS TEXT)" + } + // Identity keys are corrupted too: on both snapshot paths they export as + // NULL (or '' where the schema forbids NULL) instead of being repaired, + // so no cached string may be invalid UTF-8 regardless. + stmts := []string{ + `UPDATE messages SET ` + bad("source_message_id") + `, ` + bad("rfc822_message_id") + `, ` + bad("subject") + `, ` + bad("snippet") + `, ` + bad("list_id") + ` WHERE id = 5`, + `UPDATE message_recipients SET ` + bad("display_name") + `, ` + bad("recipient_type") + `, ` + bad("email_address") + ` WHERE message_id = 1 AND recipient_type = 'from'`, + `UPDATE attachments SET ` + bad("filename") + `, ` + bad("mime_type") + ` WHERE id = 1`, + `UPDATE participants SET ` + bad("email_address") + `, ` + bad("domain") + `, ` + bad("display_name") + `, ` + bad("phone_number") + ` WHERE id = 4`, + `UPDATE participant_identifiers SET ` + bad("identifier_type") + `, ` + bad("identifier_value") + `, ` + bad("display_value") + ` WHERE participant_id = 2`, + `UPDATE labels SET ` + bad("name") + ` WHERE id = 2`, + `UPDATE sources SET ` + bad("identifier") + ` WHERE id = 1`, + `UPDATE conversations SET ` + bad("source_conversation_id") + `, ` + bad("title") + ` WHERE id = 101`, + `INSERT INTO persons (id, vcard_uid, display_name) VALUES (1, 'uid-utf8', CAST(X'426f6280' AS TEXT))`, + `INSERT INTO person_participants (person_id, participant_id) VALUES (1, 2)`, + } + for _, stmt := range stmts { + _, err := db.Exec(stmt) + require.NoError(t, err, stmt) + } +} - _, err = buildCache(dbPath, filepath.Join(tmpDir, "analytics"), true) - require.Error(err) - assert.Contains(err.Error(), "msgvault repair-encoding") - assert.Contains(err.Error(), "common archived text fields") - assert.Contains(err.Error(), "if the cache rebuild still fails") - assert.Contains(err.Error(), "messages") +func requireEveryCachedStringDecodes(t *testing.T, analyticsDir string) { + t.Helper() + duckDB, err := sql.Open("duckdb", "") + require.NoError(t, err) + defer func() { _ = duckDB.Close() }() + dirs, err := os.ReadDir(analyticsDir) + require.NoError(t, err) + checked := 0 + for _, dir := range dirs { + if !dir.IsDir() { + continue + } + root := filepath.Join(analyticsDir, dir.Name()) + hasParquet := false + _ = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err == nil && !d.IsDir() && strings.HasSuffix(path, ".parquet") { + hasParquet = true + return fs.SkipAll + } + return nil + }) + if !hasParquet { + continue + } + glob := filepath.ToSlash(filepath.Join(root, "**", "*.parquet")) + source := fmt.Sprintf("read_parquet('%s', hive_partitioning=true, union_by_name=true)", glob) + cols := func() []string { + rows, err := duckDB.Query("SELECT column_name FROM (DESCRIBE SELECT * FROM " + source + ") WHERE column_type = 'VARCHAR'") + require.NoError(t, err, dir.Name()) + defer func() { require.NoError(t, rows.Close()) }() + var cols []string + for rows.Next() { + var c string + require.NoError(t, rows.Scan(&c)) + cols = append(cols, c) + } + require.NoError(t, rows.Err()) + return cols + }() + for _, c := range cols { + var n sql.NullInt64 + require.NoError(t, duckDB.QueryRow(fmt.Sprintf(`SELECT max(length("%s")) FROM %s`, c, source)).Scan(&n), "%s.%s must decode", dir.Name(), c) + checked++ + } + } + require.Positive(t, checked) +} + +func TestBuildCacheExportsValidUTF8InEveryTextColumn(t *testing.T) { + for _, tc := range []struct{ name, forceCSV string }{{"sqlite scanner", ""}, {"CSV fallback", "1"}} { + t.Run(tc.name, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + oldGOOS := cacheSnapshotGOOS + cacheSnapshotGOOS = "linux" + t.Cleanup(func() { cacheSnapshotGOOS = oldGOOS }) + t.Setenv("MSGVAULT_FORCE_CSV_SNAPSHOT", tc.forceCSV) + if tc.forceCSV == "" { + probe, err := sql.Open("duckdb", "") + require.NoError(err) + requireSQLiteScanner(t, probe) + require.NoError(probe.Close()) + } + tmpDir := setupTestSQLite(t) + dbPath, analyticsDir := filepath.Join(tmpDir, "test.db"), filepath.Join(tmpDir, "analytics") + corruptEveryExportedTextColumn(t, dbPath) + var buildErr error + stderr := captureStderrDuring(t, func() { _, buildErr = buildCache(dbPath, analyticsDir, true) }) + require.NoError(buildErr) + assert.Contains(stderr, "invalid UTF-8") + assert.Contains(stderr, "msgvault repair-encoding") + requireEveryCachedStringDecodes(t, analyticsDir) + if tc.forceCSV != "" { + duckDB, err := sql.Open("duckdb", "") + require.NoError(err) + defer func() { _ = duckDB.Close() }() + // Identity columns export unknown rather than repaired: NULL + // where the column allows it, '' where the schema keeps its + // no-NULL contract. + for _, check := range []struct { + dataset, column, where string + want sql.NullString + }{ + {"participants", "email_address", "id = 4", sql.NullString{String: "", Valid: true}}, + {"participant_identifiers", "identifier_type", "participant_id = 2", sql.NullString{String: "", Valid: true}}, + {"participant_identifiers", "identifier_value", "participant_id = 2", sql.NullString{String: "", Valid: true}}, + {"message_recipients", "envelope_address", "message_id = 1 AND participant_id = 1", sql.NullString{}}, + } { + var got sql.NullString + path := filepath.ToSlash(filepath.Join(analyticsDir, check.dataset, "*.parquet")) + err := duckDB.QueryRow(fmt.Sprintf("SELECT %s FROM read_parquet('%s') WHERE %s", check.column, path, check.where)).Scan(&got) + require.NoError(err) + assert.Equal(check.want, got) + } + } + }) + } } -// TestBuildCacheCSVInvalidUTF8PastSampleExplainsRepairPath covers the case -// where DuckDB's CSV sniffer does not reach the invalid row, so the error -// surfaces during the Parquet export instead of view creation. -func TestBuildCacheCSVInvalidUTF8PastSampleExplainsRepairPath(t *testing.T) { +func TestBuildCacheCSVQuotedFieldPastSniffSampleRoundTrips(t *testing.T) { require := require.New(t) assert := assert.New(t) t.Setenv("MSGVAULT_FORCE_CSV_SNAPSHOT", "1") tmpDir := setupTestSQLite(t) dbPath := filepath.Join(tmpDir, "test.db") + analyticsDir := filepath.Join(tmpDir, "analytics") + + // encoding/csv only quotes a field when it must, and DuckDB's sniffer + // only looks at the first 20,480 rows to guess the quote character. Put + // the first field that needs quoting well past that window so a rebuild + // that trusts the sniffer sees quote=(empty) and splits the row. + const bulkRows = 30000 + quotedName := "Example \"Co\", LLC\nAttn: Sales" db, err := sql.Open("sqlite3", dbPath) require.NoError(err) _, err = db.Exec(` - INSERT INTO messages (source_id, source_message_id, sent_at) - WITH RECURSIVE seq(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM seq WHERE i < 30000) - SELECT 1, 'bulk-' || i, datetime('2024-04-01', '+' || i || ' minutes') FROM seq; - `) + INSERT INTO messages (id, source_id, source_message_id, sent_at) + WITH RECURSIVE seq(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM seq WHERE i < ?) + SELECT 1000 + i, 1, 'bulk-' || i, datetime('2024-04-01', '+' || i || ' minutes') FROM seq; + `, bulkRows) + require.NoError(err) + _, err = db.Exec(` + INSERT INTO message_recipients (message_id, participant_id, recipient_type, display_name) + WITH RECURSIVE seq(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM seq WHERE i < ?) + SELECT 1000 + i, 1, 'to', 'Recipient ' || i FROM seq; + `, bulkRows) require.NoError(err) - _, err = db.Exec(`UPDATE messages SET subject = CAST(X'80' AS TEXT) WHERE id = (SELECT MAX(id) FROM messages)`) + _, err = db.Exec(` + INSERT INTO messages (id, source_id, source_message_id, sent_at) + VALUES (?, 1, 'quoted', '2024-05-01 12:00:00') + `, 1000+bulkRows+1) + require.NoError(err) + _, err = db.Exec(` + INSERT INTO message_recipients (message_id, participant_id, recipient_type, display_name) + VALUES (?, 2, 'from', ?) + `, 1000+bulkRows+1, quotedName) require.NoError(err) require.NoError(db.Close()) - _, err = buildCache(dbPath, filepath.Join(tmpDir, "analytics"), true) - require.Error(err) - assert.Contains(err.Error(), "export messages") - assert.Contains(err.Error(), "msgvault repair-encoding") - assert.Contains(err.Error(), "not a msgvault option") + _, err = buildCache(dbPath, analyticsDir, true) + require.NoError(err, "a quoted CSV field past the sniffer sample must not break the rebuild") + + duckdb, err := sql.Open("duckdb", "") + require.NoError(err) + defer func() { _ = duckdb.Close() }() + glob := filepath.Join(analyticsDir, "message_recipients", "*.parquet") + + var got string + err = duckdb.QueryRow( + `SELECT display_name FROM read_parquet(?) WHERE message_id = ? AND recipient_type = 'from'`, + glob, 1000+bulkRows+1, + ).Scan(&got) + require.NoError(err) + assert.Equal(quotedName, got, "commas, embedded quotes, and newlines must round-trip through the CSV snapshot") + + var total int64 + err = duckdb.QueryRow(`SELECT COUNT(*) FROM read_parquet(?)`, glob).Scan(&total) + require.NoError(err) + assert.Equal(int64(12+bulkRows+1), total, "every fixture, bulk, and quoted recipient row is exported") } func TestBuildCacheExportsAttachmentMetadataForRawQuery(t *testing.T) { @@ -2506,7 +3070,7 @@ func TestCSVFallbackPath(t *testing.T) { for _, tbl := range tables { csvPath := filepath.Join(csvDir, tbl.name+".csv") - if err := exportToCSV(sqliteDB, tbl.query, csvPath); err != nil { + if err := exportToCSV(sqliteDB, tbl.query, csvPath, nil); err != nil { _ = sqliteDB.Close() require.NoError(err, "exportToCSV %s", tbl.name) } @@ -4065,14 +4629,6 @@ func BenchmarkBuildCacheIncremental(b *testing.B) { // config resolution are forwarded to the build-cache subprocess so it // loads identical configuration to the daemon that spawned it. func TestGlobalConfigFlagArgs(t *testing.T) { - // Save and restore the package globals these flags bind to. - origCfg, origHome, origLocal := cfgFile, homeDir, useLocal - origLevel, origVerbose, origSQL, origSlow := logLevel, verbose, logSQL, logSQLSlow - t.Cleanup(func() { - cfgFile, homeDir, useLocal = origCfg, origHome, origLocal - logLevel, verbose, logSQL, logSQLSlow = origLevel, origVerbose, origSQL, origSlow - }) - tests := []struct { name string cfgFile string @@ -4110,9 +4666,12 @@ func TestGlobalConfigFlagArgs(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - cfgFile, homeDir, useLocal = tt.cfgFile, tt.homeDir, tt.local - logLevel, verbose, logSQL, logSQLSlow = tt.logLevel, tt.verbose, tt.logSQL, tt.logSQLSlow - assert.Equal(t, tt.want, globalConfigFlagArgs()) + options := invocationOptions{ + cfgFile: tt.cfgFile, homeDir: tt.homeDir, useLocal: tt.local, + logLevel: tt.logLevel, verbose: tt.verbose, logSQL: tt.logSQL, + logSQLSlow: tt.logSQLSlow, + } + assert.Equal(t, tt.want, globalConfigFlagArgs(options)) }) } } @@ -4127,3 +4686,96 @@ func TestBuildCacheDaemonChildEnvMarksSubprocess(t *testing.T) { assert.Contains(t, got, buildCacheDaemonSubprocessEnv+"=4242", "marks daemon-owned subprocess") assert.NotContains(t, got, buildCacheDaemonSubprocessEnv+"=0", "replaces stale marker") } + +func TestBuildCacheWarningExportScope(t *testing.T) { + for _, csv := range []bool{false, true} { + name := "scanner" + if csv { + name = "csv" + } + for _, mode := range []string{"identity", "incremental", "derived", "derived damaged"} { + t.Run(name+"/"+mode, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + old := cacheSnapshotGOOS + cacheSnapshotGOOS = "linux" + t.Cleanup(func() { cacheSnapshotGOOS = old }) + value := "" + if csv { + value = "1" + } + t.Setenv("MSGVAULT_FORCE_CSV_SNAPSHOT", value) + if !csv { + probe, err := sql.Open("duckdb", "") + require.NoError(err) + requireSQLiteScanner(t, probe) + require.NoError(probe.Close()) + } + dir := setupTestSQLite(t) + dbPath := filepath.Join(dir, "test.db") + cache := filepath.Join(dir, "analytics") + db, err := sql.Open("sqlite3", dbPath) + require.NoError(err) + defer func() { _ = db.Close() }() + statement := `UPDATE messages SET subject=CAST(X'80' AS TEXT), rfc822_message_id=CAST(X'80' AS TEXT) WHERE id=1` + if mode == "identity" { + statement = `UPDATE messages SET rfc822_message_id=CAST(X'80' AS TEXT) WHERE id=1` + } + _, err = db.Exec(statement) + require.NoError(err) + var result *buildResult + stderr := captureStderrDuring(t, func() { result, err = buildCache(dbPath, cache, true) }) + require.NoError(err) + if mode == "identity" { + duck, err := sql.Open("duckdb", "") + require.NoError(err) + defer func() { _ = duck.Close() }() + var got sql.NullString + require.NoError(duck.QueryRow(`SELECT rfc822_message_id FROM read_parquet(?, hive_partitioning=true) WHERE id=1`, filepath.Join(cache, "messages", "**", "*.parquet")).Scan(&got)) + assert.False(got.Valid) + assert.Contains(stderr, "1 identity value(s) exported as unknown") + t.Logf("invalid identity exported NULL; warning=%q", stderr) + return + } + shards, err := filepath.Glob(filepath.Join(cache, "messages", "year=*", "*.parquet")) + require.NoError(err) + require.NotEmpty(shards) + original, err := os.ReadFile(shards[0]) + require.NoError(err) + if mode == "incremental" { + _, err = db.Exec(`INSERT INTO messages(id,source_id,source_message_id,conversation_id,subject,sent_at) VALUES(6,1,'probe-clean',101,'clean','2025-01-01')`) + require.NoError(err) + stderr = captureStderrDuring(t, func() { result, err = buildCache(dbPath, cache, false) }) + require.NoError(err) + assert.Equal(int64(6), result.ExportedCount) + after, err := os.ReadFile(shards[0]) + require.NoError(err) + assert.Equal(original, after) + } else { + title := "changed clean title" + if mode == "derived damaged" { + title = "changed\x80title" + } + _, err = db.Exec(`UPDATE conversations SET title=CAST(? AS TEXT) WHERE id=101`, []byte(title)) + require.NoError(err) + stderr = captureStderrDuring(t, func() { + result, err = refreshDerivedDatasetsOnly(context.Background(), dbPath, cache, acquirePublishLock, false) + }) + require.NoError(err) + assert.True(result.IdentityOnly) + assert.False(result.Skipped) + after, err := os.ReadFile(shards[0]) + require.NoError(err) + assert.Equal(original, after) + } + if mode == "derived damaged" { + assert.Contains(stderr, "Warning: 1 invalid UTF-8 repair(s)") + assert.NotContains(stderr, "identity value(s)") + } else { + assert.NotContains(stderr, "Warning:") + } + t.Logf("mode=%s exported=%d identityOnly=%v warning=%q", mode, result.ExportedCount, result.IdentityOnly, strings.TrimSpace(stderr)) + }) + } + } +} diff --git a/cmd/msgvault/cmd/cache_build_resilience_test.go b/cmd/msgvault/cmd/cache_build_resilience_test.go new file mode 100644 index 000000000..7d813b9b6 --- /dev/null +++ b/cmd/msgvault/cmd/cache_build_resilience_test.go @@ -0,0 +1,179 @@ +package cmd + +import ( + "crypto/sha256" + "database/sql" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/query" +) + +func TestCacheSnapshotSyncCatchUp(t *testing.T) { + tests := []struct { + name string + csv string + appendMessage bool + wantMessages int + }{ + {name: "scanner/children_only", wantMessages: 1}, + {name: "scanner/with_append", appendMessage: true, wantMessages: 2}, + {name: "csv/children_only", csv: "1", wantMessages: 1}, + {name: "csv/with_append", csv: "1", appendMessage: true, wantMessages: 2}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert, require := assert.New(t), require.New(t) + t.Setenv("MSGVAULT_FORCE_CSV_SNAPSHOT", tt.csv) + c, st := openTestDaemonAnalyticsStore(t) + _, err := st.DB().Exec(` + INSERT INTO sources (id, source_type, identifier) VALUES (1, 'gmail', 'owner@example.com'); + INSERT INTO conversations (id, source_id, source_conversation_id, conversation_type) + VALUES (1, 1, 'thread', 'email_thread'); + INSERT INTO messages (id, source_id, source_message_id, conversation_id, message_type, sent_at) + VALUES (1, 1, 'message-1', 1, 'email', '2024-01-01 00:00:00'); + INSERT INTO labels (id, name) VALUES (1, 'synthetic'); + `) + require.NoError(err) + // A separate connection writes after the export's read transaction + // is pinned, before any source tables are copied. + buildCacheAfterSnapshotHook = func() { + writer, err := sql.Open("sqlite3", c.DatabaseDSN()) + require.NoError(err) + defer func() { require.NoError(writer.Close()) }() + if tt.appendMessage { + _, err = writer.Exec(`INSERT INTO messages + (id, source_id, source_message_id, conversation_id, message_type, sent_at) + VALUES (2, 1, 'message-2', 1, 'email', '2024-01-02 00:00:00')`) + require.NoError(err) + } + _, err = writer.Exec(` + INSERT INTO message_labels (message_id, label_id) VALUES (1, 1); + INSERT INTO sync_runs (source_id, started_at, completed_at, status, messages_added) + VALUES (1, datetime('now'), datetime('now'), 'completed', 1); + `) + require.NoError(err) + } + t.Cleanup(func() { buildCacheAfterSnapshotHook = nil }) + _, err = buildCache(c.DatabaseDSN(), c.AnalyticsDir(), true) + require.NoError(err) + buildCacheAfterSnapshotHook = nil + state, err := query.ReadCacheSyncState(c.AnalyticsDir()) + require.NoError(err) + assert.Equal(int64(1), state.LastMessageID) + assert.Zero(state.LastCacheAdditionCount, "stamp the snapshot's counters") + assert.False(state.FullRebuildRequired, "journal tracks late child rows") + before := snapshotMessagesDatasetBytes(t, c.AnalyticsDir()) + stale := cacheNeedsBuild(c.DatabaseDSN(), c.AnalyticsDir()) + assert.True(stale.NeedsBuild) + assert.False(stale.FullRebuild, "journaled catch-up: %s", stale.Reason) + _, err = buildCacheAuto(c.DatabaseDSN(), c.AnalyticsDir()) + require.NoError(err) + after := snapshotMessagesDatasetBytes(t, c.AnalyticsDir()) + for path, contents := range before { + assert.Equal(sha256.Sum256([]byte(contents)), sha256.Sum256([]byte(after[path])), + "retain message shard %s", path) + } + assert.False(cacheNeedsBuild(c.DatabaseDSN(), c.AnalyticsDir()).NeedsBuild) + engine, err := openDaemonDuckDBEngine(c, st) + require.NoError(err) + t.Cleanup(func() { require.NoError(engine.Close()) }) + rows, err := engine.QuerySQL(t.Context(), "SELECT message_id, label_id FROM message_labels") + require.NoError(err) + require.Len(rows.Rows, 1) + assert.EqualValues(1, rows.Rows[0][0]) + rows, err = engine.QuerySQL(t.Context(), "SELECT COUNT(*), COUNT(DISTINCT id) FROM messages") + require.NoError(err) + assert.EqualValues(tt.wantMessages, rows.Rows[0][0]) + assert.EqualValues(tt.wantMessages, rows.Rows[0][1]) + }) + } +} + +// A sync can commit between the staleness check that selects a child-row +// refresh and the refresh's own snapshot. The refresh must reject changes it +// cannot repair, and every build entry point must finish them with a full +// rebuild. +func TestCacheRelatedRefreshRaceFallsBackToFullBuild(t *testing.T) { + races := []struct { + name string + write string + wantMessages int64 + }{ + { + name: "old message becomes exportable", + write: `UPDATE messages SET sent_at = '2024-01-01 00:00:00' WHERE id = 1`, + wantMessages: 2, + }, + { + name: "sync updates cached messages", + write: `INSERT INTO sync_runs (source_id, started_at, completed_at, status, messages_updated) + VALUES (1, datetime('now'), datetime('now'), 'completed', 1)`, + wantMessages: 1, + }, + } + builds := []struct { + name string + build func(dbPath, analyticsDir string) (*buildResult, error) + }{ + {name: "auto", build: func(dbPath, analyticsDir string) (*buildResult, error) { + return buildCacheAuto(dbPath, analyticsDir) + }}, + {name: "scheduled", build: func(dbPath, analyticsDir string) (*buildResult, error) { + return buildCacheScheduled(dbPath, analyticsDir, 0, time.Now) + }}, + } + for _, entry := range builds { + for _, race := range races { + t.Run(entry.name+"/"+race.name, func(t *testing.T) { + assert, require := assert.New(t), require.New(t) + c, st := openTestDaemonAnalyticsStore(t) + _, err := st.DB().Exec(` + INSERT INTO sources (id, source_type, identifier) VALUES (1, 'gmail', 'owner@example.com'); + INSERT INTO conversations (id, source_id, source_conversation_id, conversation_type) + VALUES (1, 1, 'thread', 'email_thread'); + INSERT INTO messages (id, source_id, source_message_id, conversation_id, message_type, sent_at) + VALUES (1, 1, 'pending', 1, 'email', NULL), + (2, 1, 'message-2', 1, 'email', '2024-01-02 00:00:00'); + INSERT INTO labels (id, name) VALUES (1, 'synthetic'); + `) + require.NoError(err) + _, err = buildCache(c.DatabaseDSN(), c.AnalyticsDir(), true) + require.NoError(err) + _, err = st.DB().Exec(` + INSERT INTO message_labels (message_id, label_id) VALUES (2, 1); + INSERT INTO sync_runs (source_id, started_at, completed_at, status, messages_added) + VALUES (1, datetime('now'), datetime('now'), 'completed', 1); + `) + require.NoError(err) + stale := cacheNeedsBuild(c.DatabaseDSN(), c.AnalyticsDir()) + require.True(stale.HasRelatedRowDrift) + require.False(stale.FullRebuild, "the label alone is a child-row repair: %s", stale.Reason) + + derivedRefreshBeforeSnapshotHook = func() { + _, err := st.DB().Exec(race.write) + require.NoError(err) + } + t.Cleanup(func() { derivedRefreshBeforeSnapshotHook = nil }) + built, err := entry.build(c.DatabaseDSN(), c.AnalyticsDir()) + require.NoError(err) + derivedRefreshBeforeSnapshotHook = nil + assert.Equal(race.wantMessages, built.StagedCount, "full rebuild stages every message") + assert.False(cacheNeedsBuild(c.DatabaseDSN(), c.AnalyticsDir()).NeedsBuild) + + engine, err := openDaemonDuckDBEngine(c, st) + require.NoError(err) + t.Cleanup(func() { require.NoError(engine.Close()) }) + rows, err := engine.QuerySQL(t.Context(), "SELECT COUNT(*) FROM messages") + require.NoError(err) + assert.EqualValues(race.wantMessages, rows.Rows[0][0]) + rows, err = engine.QuerySQL(t.Context(), "SELECT message_id FROM message_labels") + require.NoError(err) + require.Len(rows.Rows, 1) + assert.EqualValues(2, rows.Rows[0][0]) + }) + } + } +} diff --git a/cmd/msgvault/cmd/cache_derived.go b/cmd/msgvault/cmd/cache_derived.go index 65d0eb360..dd2786e2b 100644 --- a/cmd/msgvault/cmd/cache_derived.go +++ b/cmd/msgvault/cmd/cache_derived.go @@ -23,10 +23,16 @@ var ErrDerivedRefreshRequiresFullBuild = errors.New( var derivedPublishBeforeMarkerHook func() error +// derivedRefreshBeforeSnapshotHook is a deterministic test seam for writes +// that commit after a staleness check selects a refresh but before the refresh +// pins its source snapshot. +var derivedRefreshBeforeSnapshotHook func() + func refreshDerivedDatasetsOnly( ctx context.Context, dbPath, analyticsDir string, locking cachePublishLocking, + repairRelated bool, builderOverrides ...duckdbutil.BuilderOverrides, ) (*buildResult, error) { readiness, err := query.InspectCacheReadiness(analyticsDir) @@ -75,9 +81,17 @@ func refreshDerivedDatasetsOnly( return nil, fmt.Errorf("read derived-data revision: %w", err) } if derivedDataRevision != state.DerivedDataRevision { - _ = st.Close() - return nil, fmt.Errorf("%w: derived-data revision changed", - ErrDerivedRefreshRequiresFullBuild) + relatedOnly, relatedErr := st.RelatedDerivedRevisionsOnly(ctx, + state.DerivedDataRevision, derivedDataRevision) + if relatedErr != nil { + _ = st.Close() + return nil, fmt.Errorf("classify derived-data revision: %w", relatedErr) + } + if !repairRelated || !relatedOnly { + _ = st.Close() + return nil, fmt.Errorf("%w: derived-data revision changed", + ErrDerivedRefreshRequiresFullBuild) + } } accountIdentityRevision, err := st.AccountIdentityRevision() if err != nil { @@ -122,13 +136,20 @@ func refreshDerivedDatasetsOnly( return nil, fmt.Errorf("open bounded DuckDB for derived refresh: %w", err) } defer func() { _ = duckDB.Close() }() + textRepairs := &cacheTextRepairs{} + if err := registerCacheTextFunctions(ctx, duckDB, textRepairs); err != nil { + return nil, err + } + if derivedRefreshBeforeSnapshotHook != nil { + derivedRefreshBeforeSnapshotHook() + } sourceSnapshot, err := openCacheSourceSnapshot(duckDB, dbPath) if err != nil { return nil, err } defer func() { _ = sourceSnapshot.Close() }() - if err := sourceSnapshot.PrepareDatasets( + datasets := []string{ tableMessages, tableConversations, tableConversationParticipants, @@ -136,7 +157,59 @@ func refreshDerivedDatasetsOnly( tableParticipants, tableParticipantIdentifiers, "persons", "person_participants", - ); err != nil { + } + var relatedChangeSeq int64 + var relatedKinds relatedChangeKinds + if repairRelated { + // A late terminal sync can add children to an already-exported parent. + // Acknowledge its addition counter only from the same snapshot used to + // repair those children. Changed facts or failed runs remain full repairs. + counters, err := readCacheSyncCounters(sourceSnapshot) + if err != nil { + return nil, fmt.Errorf("read related-refresh sync counters: %w", err) + } + if counters.updates != state.LastCacheUpdateCount || + counters.failedRunCount != state.LastFailedSyncRunCount || + counters.failedRunIDSum != state.LastFailedSyncRunIDSum || + counters.additions < state.LastCacheAdditionCount { + return nil, fmt.Errorf("%w: sync counters require a full repair", + ErrDerivedRefreshRequiresFullBuild) + } + if counters.additions != state.LastCacheAdditionCount { + if err := verifyRelatedOnlyAdditions(sourceSnapshot, state); err != nil { + return nil, err + } + } + state.LastCacheAdditionCount = counters.additions + if err := sourceSnapshot.QueryRow(`SELECT COALESCE((SELECT seq FROM sqlite_sequence + WHERE name = 'cache_related_change_journal'), 0)`).Scan(&relatedChangeSeq); err != nil { + return nil, fmt.Errorf("read related-change boundary: %w", err) + } + if err := inspectRelatedSnapshotColumns(sourceSnapshot); err != nil { + return nil, err + } + // The CSV fallback closes its SQLite transaction during preparation. + // Read journal metadata while that snapshot is still available. + relatedKinds, err = inspectRelatedChangeKinds(sourceSnapshot, + state.LastRelatedChangeSeq, relatedChangeSeq, state.LastMessageID) + if err != nil { + return nil, err + } + if relatedKinds.other { + return nil, fmt.Errorf("%w: unsupported related-row journal dataset", + ErrDerivedRefreshRequiresFullBuild) + } + } + relatedExports := make(map[string]int64) + relatedReplacements := make(map[string]bool) + for dataset, changed := range relatedKinds.datasets() { + if changed { + relatedExports[dataset] = 0 + datasets = append(datasets, dataset) + relatedReplacements[dataset] = true + } + } + if err := sourceSnapshot.PrepareDatasets(datasets...); err != nil { return nil, err } exportDB := sourceSnapshot.DuckDB() @@ -158,7 +231,7 @@ func refreshDerivedDatasetsOnly( return nil, err } - if identityRevision == state.IdentityRevision && + if !repairRelated && identityRevision == state.IdentityRevision && participantIdentifierRevision == state.ParticipantIdentifierRevision && participantDisplayNameRevision == state.ParticipantDisplayNameRevision && personDisplayNameRevision == state.PersonDisplayNameRevision && @@ -170,23 +243,58 @@ func refreshDerivedDatasetsOnly( // it active pagination cursors — for no analytical difference. return &buildResult{OutputDir: analyticsDir, IdentityOnly: true, Skipped: true}, nil } + if repairRelated && !relatedKinds.recipients && + identityRevision == state.IdentityRevision && + participantIdentifierRevision == state.ParticipantIdentifierRevision && + participantDisplayNameRevision == state.ParticipantDisplayNameRevision && + personDisplayNameRevision == state.PersonDisplayNameRevision && + conversationFingerprint == state.ConversationParticipantsFingerprint && + typesFingerprint == state.ConversationTypesFingerprint { + // Labels and attachment metadata do not enter relationship_activity. + // Publish their child rows and marker directly, avoiding a scan of the + // expanded relationship population for a small metadata correction. + if err := exportRelatedDatasets(ctx, exportDB, sourceSnapshot, + state.LastMessageID, staging.root, relatedExports); err != nil { + return nil, err + } + if err := refreshRelatedCacheStats(ctx, exportDB, &state, relatedKinds); err != nil { + return nil, err + } + if err := sourceSnapshot.Close(); err != nil { + return nil, fmt.Errorf("close SQLite related-refresh snapshot: %w", err) + } + state.DerivedDataRevision = derivedDataRevision + state.LastRelatedChangeSeq = relatedChangeSeq + plan := cachePublishPlan{ + Append: map[string]bool{}, + Replace: relatedReplacements, + } + if err := publishDerivedCache(staging, analyticsDir, plan, state, locking); err != nil { + return nil, err + } + reportCacheTextRepairs(os.Stderr, textRepairs) + warnRelatedChangePrune(dbPath, relatedChangeSeq, derivedDataRevision) + return &buildResult{OutputDir: analyticsDir, IdentityOnly: true}, nil + } - if err := exportDerivedOwnerParticipants(ctx, exportDB, staging.root); err != nil { - return nil, err + derivedCopy := func(table, selectSQL string) error { + return copyParquet(ctx, exportDB, filepath.Join(staging.root, table), table+".parquet", selectSQL) + } + if err := derivedCopy(tableOwnerParticipants, ownerParticipantsSelectSQL( + sourceSnapshot.identityPresenceSQL("email_address", "primary_email_present"))); err != nil { + return nil, fmt.Errorf("export derived owner participants: %w", err) } - if err := exportDerivedParticipantClusters(ctx, exportDB, clusters, staging.root); err != nil { + if err := stageParticipantClusters(ctx, exportDB, clusters); err != nil { return nil, err } + if err := derivedCopy(tableParticipantClusters, participantClustersSelectSQL); err != nil { + return nil, fmt.Errorf("export derived participant clusters: %w", err) + } conversationChanged := conversationFingerprint != state.ConversationParticipantsFingerprint if conversationChanged { - if err := exportDerivedConversationParticipants( - ctx, - exportDB, - state.LastMessageID, - staging.root, - ); err != nil { - return nil, err + if err := derivedCopy(tableConversationParticipants, conversationParticipantsSelectSQL(state.LastMessageID)); err != nil { + return nil, fmt.Errorf("export derived conversation participants: %w", err) } } identifiersChanged := @@ -196,8 +304,8 @@ func refreshDerivedDatasetsOnly( // participant_identifiers base dataset (relationship_people search // values and label fallbacks), so a changed mapping must be re-staged // and republished alongside the derived index. - if err := exportDerivedParticipantIdentifiers(ctx, exportDB, staging.root); err != nil { - return nil, err + if err := derivedCopy(tableParticipantIdentifiers, sourceSnapshot.participantIdentifiersExportSelectSQL()); err != nil { + return nil, fmt.Errorf("export derived participant identifiers: %w", err) } } displayNamesChanged := @@ -206,15 +314,15 @@ func refreshDerivedDatasetsOnly( // Participant identifiers can create participant rows, and display-name // mutations change the row already present in participants.parquet. Both // changes must replace that base dataset before rebuilding the directory. - if err := exportDerivedParticipants(ctx, exportDB, staging.root); err != nil { - return nil, err + if err := derivedCopy(tableParticipants, sourceSnapshot.participantsExportSelectSQL()); err != nil { + return nil, fmt.Errorf("export derived participants: %w", err) } } personDisplayNamesChanged := personDisplayNameRevision != state.PersonDisplayNameRevision identityChanged := identityRevision != state.IdentityRevision if personDisplayNamesChanged || identityChanged { - if err := exportDerivedPersonDisplayNames(ctx, exportDB, staging.root); err != nil { - return nil, err + if err := derivedCopy(tablePersonDisplayNames, sourceSnapshot.personDisplayNamesExportSelectSQL()); err != nil { + return nil, fmt.Errorf("export derived person_display_names: %w", err) } } typesChanged := typesFingerprint != state.ConversationTypesFingerprint @@ -223,12 +331,15 @@ func refreshDerivedDatasetsOnly( // base dataset, and the analytical view joins it live — both must // see the current types, so the dataset is re-staged and republished // alongside the derived index. - if err := exportDerivedConversations( - ctx, - exportDB, - state.LastMessageID, - staging.root, - ); err != nil { + if err := derivedCopy(tableConversations, sourceSnapshot.conversationsExportSelectSQL(state.LastMessageID)); err != nil { + return nil, fmt.Errorf("export derived conversations: %w", err) + } + } + if repairRelated { + if err := exportRelatedDatasets(ctx, exportDB, sourceSnapshot, state.LastMessageID, staging.root, relatedExports); err != nil { + return nil, err + } + if err := refreshRelatedCacheStats(ctx, exportDB, &state, relatedKinds); err != nil { return nil, err } } @@ -253,13 +364,19 @@ func refreshDerivedDatasetsOnly( } state.IdentityRevision = identityRevision + if repairRelated { + state.DerivedDataRevision = derivedDataRevision + } state.ParticipantIdentifierRevision = participantIdentifierRevision state.ParticipantDisplayNameRevision = participantDisplayNameRevision state.PersonDisplayNameRevision = personDisplayNameRevision state.ConversationParticipantsFingerprint = conversationFingerprint state.ConversationTypesFingerprint = typesFingerprint - // Stats describe the unchanged committed raw snapshot. Preserve them - // byte-for-byte instead of scanning Parquet again. + if repairRelated { + state.LastRelatedChangeSeq = relatedChangeSeq + } + // The message snapshot remains unchanged; child-row statistics were + // refreshed above when their source rows changed. plan := derivedCachePublishPlan( conversationChanged, typesChanged, @@ -267,9 +384,18 @@ func refreshDerivedDatasetsOnly( identifiersChanged || displayNamesChanged, personDisplayNamesChanged || identityChanged, ) + if repairRelated { + for dataset := range relatedReplacements { + plan.Replace[dataset] = true + } + } if err := publishDerivedCache(staging, analyticsDir, plan, state, locking); err != nil { return nil, err } + reportCacheTextRepairs(os.Stderr, textRepairs) + if repairRelated { + warnRelatedChangePrune(dbPath, relatedChangeSeq, derivedDataRevision) + } return &buildResult{OutputDir: analyticsDir, IdentityOnly: true}, nil } @@ -304,9 +430,9 @@ func fingerprintConversationParticipantsFromSnapshot( // fingerprintConversationTypesFromSnapshot mirrors // sourceConversationTypesFingerprint over the export snapshot, so the stamp // written at publish time describes exactly the type/title metadata the staged -// datasets baked. The normalizations match the staleness query (and the CSV -// snapshot view), not the exported Parquet values; fingerprints only compare -// against each other. +// datasets baked. The query applies the same NULL defaults as the staleness +// query. FingerprintConversationMetadata repairs invalid UTF-8 on both paths +// before hashing. func fingerprintConversationTypesFromSnapshot( ctx context.Context, db sqlRunner, @@ -337,191 +463,48 @@ func fingerprintConversationTypesFromSnapshot( return fingerprint, err } -// exportDerivedConversations re-stages the conversations base dataset with -// the full export query so an index-only refresh triggered by type drift -// rebuilds relationship_activity from current types and republishes the -// dataset the analytical view joins. -func exportDerivedConversations( - ctx context.Context, - db sqlRunner, - lastMessageID int64, - stagingRoot string, -) error { - dir := filepath.Join(stagingRoot, tableConversations) - if err := os.MkdirAll(dir, 0o755); err != nil { - return fmt.Errorf("create derived conversations directory: %w", err) - } - path := filepath.Join(dir, "conversations.parquet") - _, err := db.ExecContext(ctx, fmt.Sprintf(` - COPY ( - %s - ) TO '%s' (FORMAT PARQUET, COMPRESSION 'zstd') - `, conversationsExportSelectSQL(lastMessageID), quoteCacheSQL(path))) - if err != nil { - return fmt.Errorf("export derived conversations: %w", err) - } - return nil -} - -func exportDerivedOwnerParticipants( - ctx context.Context, - db sqlRunner, - stagingRoot string, -) error { - dir := filepath.Join(stagingRoot, tableOwnerParticipants) - if err := os.MkdirAll(dir, 0o755); err != nil { - return fmt.Errorf("create derived owner participants directory: %w", err) +// stageParticipantClusters loads the Go-computed clusters into a DuckDB temp +// table that participantClustersSelectSQL reads. +func stageParticipantClusters(ctx context.Context, db sqlRunner, clusters map[int64]int64) error { + if _, err := db.ExecContext(ctx, + `CREATE TEMP TABLE tmp_participant_clusters (participant_id BIGINT, canonical_id BIGINT)`); err != nil { + return fmt.Errorf("create participant clusters temp table: %w", err) } - path := filepath.Join(dir, "owner_participants.parquet") - _, err := db.ExecContext(ctx, fmt.Sprintf(` - COPY (%s - ) TO '%s' (FORMAT PARQUET, COMPRESSION 'zstd') - `, ownerParticipantsSelectSQL, quoteCacheSQL(path))) - if err != nil { - return fmt.Errorf("export derived owner participants: %w", err) + if len(clusters) == 0 { + return nil } - return nil -} - -// exportDerivedParticipants re-stages the participants base dataset when an -// identifier creates a participant or a display-name mutation changes an -// existing row. The relationship directory reads this dataset directly. -func exportDerivedParticipants( - ctx context.Context, - db sqlRunner, - stagingRoot string, -) error { - dir := filepath.Join(stagingRoot, tableParticipants) - if err := os.MkdirAll(dir, 0o755); err != nil { - return fmt.Errorf("create derived participants directory: %w", err) - } - path := filepath.Join(dir, "participants.parquet") - _, err := db.ExecContext(ctx, fmt.Sprintf(` - COPY ( - %s - ) TO '%s' (FORMAT PARQUET, COMPRESSION 'zstd') - `, participantsExportSelectSQL(), quoteCacheSQL(path))) - if err != nil { - return fmt.Errorf("export derived participants: %w", err) + values := make([]string, 0, len(clusters)) + for participantID, canonicalID := range clusters { + values = append(values, fmt.Sprintf("(%d,%d)", participantID, canonicalID)) } - return nil -} - -func exportDerivedPersonDisplayNames( - ctx context.Context, - db sqlRunner, - stagingRoot string, -) error { - dir := filepath.Join(stagingRoot, tablePersonDisplayNames) - if err := os.MkdirAll(dir, 0o755); err != nil { - return fmt.Errorf("create derived person_display_names directory: %w", err) - } - path := filepath.Join(dir, "person_display_names.parquet") - _, err := db.ExecContext(ctx, fmt.Sprintf(` - COPY ( - %s - ) TO '%s' (FORMAT PARQUET, COMPRESSION 'zstd') - `, personDisplayNamesExportSelectSQL(), quoteCacheSQL(path))) - if err != nil { - return fmt.Errorf("export derived person_display_names: %w", err) + if _, err := db.ExecContext(ctx, `INSERT INTO tmp_participant_clusters VALUES `+strings.Join(values, ",")); err != nil { + return fmt.Errorf("populate participant clusters temp table: %w", err) } return nil } -// exportDerivedParticipantIdentifiers re-stages the participant_identifiers -// base dataset with the full export query so an index-only refresh triggered -// by identifier drift rebuilds the identity directory from current mappings -// and republishes the dataset participant-label fallbacks join. -func exportDerivedParticipantIdentifiers( - ctx context.Context, - db sqlRunner, - stagingRoot string, -) error { - dir := filepath.Join(stagingRoot, tableParticipantIdentifiers) - if err := os.MkdirAll(dir, 0o755); err != nil { - return fmt.Errorf("create derived participant identifiers directory: %w", err) - } - path := filepath.Join(dir, "participant_identifiers.parquet") - _, err := db.ExecContext(ctx, fmt.Sprintf(` - COPY ( - %s - ) TO '%s' (FORMAT PARQUET, COMPRESSION 'zstd') - `, participantIdentifiersExportSelectSQL(), quoteCacheSQL(path))) - if err != nil { - return fmt.Errorf("export derived participant identifiers: %w", err) - } - return nil -} +const participantClustersSelectSQL = `SELECT participant_id, canonical_id FROM tmp_participant_clusters` -func exportDerivedParticipantClusters( - ctx context.Context, - db sqlRunner, - clusters map[int64]int64, - stagingRoot string, -) error { - if _, err := db.ExecContext(ctx, ` - CREATE TEMP TABLE tmp_derived_participant_clusters ( - participant_id BIGINT, - canonical_id BIGINT - ) - `); err != nil { - return fmt.Errorf("create derived participant clusters table: %w", err) - } - if len(clusters) > 0 { - values := make([]string, 0, len(clusters)) - for participantID, canonicalID := range clusters { - values = append(values, fmt.Sprintf("(%d,%d)", participantID, canonicalID)) - } - if _, err := db.ExecContext(ctx, ` - INSERT INTO tmp_derived_participant_clusters - VALUES `+strings.Join(values, ",")); err != nil { - return fmt.Errorf("populate derived participant clusters: %w", err) - } - } - dir := filepath.Join(stagingRoot, tableParticipantClusters) - if err := os.MkdirAll(dir, 0o755); err != nil { - return fmt.Errorf("create derived participant clusters directory: %w", err) - } - path := filepath.Join(dir, "participant_clusters.parquet") - if _, err := db.ExecContext(ctx, fmt.Sprintf(` - COPY ( - SELECT participant_id, canonical_id - FROM tmp_derived_participant_clusters - ) TO '%s' (FORMAT PARQUET, COMPRESSION 'zstd') - `, quoteCacheSQL(path))); err != nil { - return fmt.Errorf("export derived participant clusters: %w", err) - } - return nil +func conversationParticipantsSelectSQL(maxMessageID int64) string { + return fmt.Sprintf(`SELECT cp.conversation_id, cp.participant_id + FROM sqlite_db.conversation_participants cp + WHERE EXISTS ( + SELECT 1 + FROM sqlite_db.messages m + WHERE m.conversation_id = cp.conversation_id + AND %s + AND TRY_CAST(m.id AS BIGINT) <= %d + )`, exportableMessageWhere("m"), maxMessageID) } -func exportDerivedConversationParticipants( - ctx context.Context, - db sqlRunner, - lastMessageID int64, - stagingRoot string, -) error { - dir := filepath.Join(stagingRoot, tableConversationParticipants) +// copyParquet writes selectSQL to dir/file as zstd Parquet, creating dir. +func copyParquet(ctx context.Context, db sqlRunner, dir, file, selectSQL string) error { if err := os.MkdirAll(dir, 0o755); err != nil { - return fmt.Errorf("create derived conversation participants directory: %w", err) - } - path := filepath.Join(dir, "conversation_participants.parquet") - _, err := db.ExecContext(ctx, fmt.Sprintf(` - COPY ( - SELECT cp.conversation_id, cp.participant_id - FROM sqlite_db.conversation_participants cp - WHERE EXISTS ( - SELECT 1 - FROM sqlite_db.messages m - WHERE m.conversation_id = cp.conversation_id - AND %s - AND TRY_CAST(m.id AS BIGINT) <= %d - ) - ) TO '%s' (FORMAT PARQUET, COMPRESSION 'zstd') - `, exportableMessageWhere("m"), lastMessageID, quoteCacheSQL(path))) - if err != nil { - return fmt.Errorf("export derived conversation participants: %w", err) + return err } - return nil + _, err := db.ExecContext(ctx, fmt.Sprintf(`COPY (%s) TO '%s' (FORMAT PARQUET, COMPRESSION 'zstd')`, + selectSQL, quoteCacheSQL(filepath.Join(dir, file)))) + return err } func quoteCacheSQL(value string) string { @@ -543,6 +526,8 @@ func derivedCachePublishPlan( identityindex.DatasetPeople, identityindex.DatasetDomains, identityindex.DatasetRelationshipDaily, + identityindex.DatasetLogicalContributions, + identityindex.DatasetTemperatureContributions, } { plan.Replace[dataset] = true } @@ -584,3 +569,32 @@ func publishDerivedCache( locking, ) } + +// verifyRelatedOnlyAdditions confirms that new sync additions changed only +// child rows of cached messages. A refresh may then record the new addition +// count. Any change to the message population needs a full build instead: +// an old message that became exportable inside the cached boundary, or an +// exportable message above it. The staleness check ignores a message deleted +// at the source when it looks for new messages, so such a message above the +// boundary would otherwise stay out of the cache. +func verifyRelatedOnlyAdditions(snapshot *cacheSourceSnapshot, state syncState) error { + var coveredCount int64 + if err := snapshot.QueryRow(coveredCacheMessageCountSQL(), state.LastMessageID). + Scan(&coveredCount); err != nil { + return fmt.Errorf("check related-refresh message population: %w", err) + } + if coveredCount != state.Stats.TotalMessages { + return fmt.Errorf("%w: cached message population changed", + ErrDerivedRefreshRequiresFullBuild) + } + var uncachedExportable bool + if err := snapshot.QueryRow(`SELECT EXISTS (SELECT 1 FROM messages WHERE id > ? AND `+ + exportableMessageWhere("")+`)`, state.LastMessageID).Scan(&uncachedExportable); err != nil { + return fmt.Errorf("check related-refresh message boundary: %w", err) + } + if uncachedExportable { + return fmt.Errorf("%w: exportable messages above the cached boundary", + ErrDerivedRefreshRequiresFullBuild) + } + return nil +} diff --git a/cmd/msgvault/cmd/cache_export_parity_test.go b/cmd/msgvault/cmd/cache_export_parity_test.go new file mode 100644 index 000000000..473aa5a9d --- /dev/null +++ b/cmd/msgvault/cmd/cache_export_parity_test.go @@ -0,0 +1,121 @@ +package cmd + +import ( + "context" + "database/sql" + "fmt" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/duckdbutil" + "go.kenn.io/msgvault/internal/store" +) + +// TestCacheExportParityFullVersusDerived pins the shared export SQL: a derived +// refresh after identity, identifier, display-name, membership and type +// changes must write the same rows a full rebuild writes. +func TestCacheExportParityFullVersusDerived(t *testing.T) { + require := require.New(t) + tmp := t.TempDir() + dbPath := filepath.Join(tmp, "msgvault.db") + derivedDir := filepath.Join(tmp, "analytics-derived") + fullDir := filepath.Join(tmp, "analytics-full") + st, err := store.Open(dbPath) + require.NoError(err) + require.NoError(st.InitSchema()) + source, err := st.GetOrCreateSource("gmail", "alice@example.com") + require.NoError(err) + require.NoError(st.AddAccountIdentity(source.ID, "alice@example.com", "manual")) + alice, err := st.EnsureParticipant("alice@example.com", "Alice", "example.com") + require.NoError(err) + bob, err := st.EnsureParticipant("bob@example.com", "Bob", "example.com") + require.NoError(err) + carol, err := st.EnsureParticipant("carol@example.com", "Carol", "example.com") + require.NoError(err) + var conversations []int64 + for i, sender := range []int64{alice, bob} { + conv, err := st.EnsureConversationWithType(source.ID, fmt.Sprintf("thread-%d", i), "direct_chat", "Thread") + require.NoError(err) + conversations = append(conversations, conv) + id, err := st.UpsertMessage(&store.Message{ + ConversationID: conv, SourceID: source.ID, SourceMessageID: fmt.Sprintf("message-%d", i), MessageType: "email", + SenderID: sql.NullInt64{Int64: sender, Valid: true}, + SentAt: sql.NullTime{Time: time.Date(2025, 4, 1, 12, 0, 0, 0, time.UTC), Valid: true}, + }) + require.NoError(err) + require.NoError(st.ReplaceMessageRecipients(id, "from", []int64{sender}, []string{""})) + require.NoError(st.ReplaceMessageRecipients(id, "to", []int64{alice + bob - sender}, []string{""})) + _, err = st.DB().Exec(`INSERT INTO conversation_participants (conversation_id, participant_id) VALUES (?, ?), (?, ?)`, + conv, alice, conv, bob) + require.NoError(err) + } + person, _, err := st.CreatePersonFromParticipant(bob) + require.NoError(err) + require.NoError(st.Close()) + _, err = buildCache(dbPath, derivedDir, true) + require.NoError(err) + + st, err = store.Open(dbPath) + require.NoError(err) + name := "Bob Curated" + _, err = st.UpdatePersonDisplayName(person.ID, person.Revision, &name) + require.NoError(err) + _, err = st.LinkParticipants(bob, carol) + require.NoError(err) + _, err = st.DB().Exec(`UPDATE participants SET display_name = NULL WHERE id = ?`, alice) + require.NoError(err) + _, err = st.EnsureParticipantByIdentifier("email", "alice@example.com", "Alice Updated") + require.NoError(err) + require.NoError(st.SetParticipantIdentifier(carol, "phone", "+15550142")) + _, err = st.DB().Exec(`INSERT INTO conversation_participants (conversation_id, participant_id) VALUES (?, ?)`, + conversations[1], carol) + require.NoError(err) + _, err = st.DB().Exec(`UPDATE conversations SET conversation_type = 'group_chat' WHERE id = ?`, conversations[1]) + require.NoError(err) + require.NoError(st.Close()) + + result, err := buildCacheDerivedOnly(dbPath, derivedDir) + require.NoError(err) + require.True(result.IdentityOnly, "the mutations must take the derived refresh path") + require.False(result.Skipped) + _, err = buildCache(dbPath, fullDir, true) + require.NoError(err) + + duckDB, err := duckdbutil.Open(context.Background(), duckdbutil.BuilderPolicy(filepath.Join(tmp, "parity-duckdb-tmp"))) + require.NoError(err) + defer func() { require.NoError(duckDB.Close()) }() + for _, dataset := range []string{ + tableParticipants, tableParticipantIdentifiers, tablePersonDisplayNames, tableOwnerParticipants, + tableParticipantClusters, tableConversations, tableConversationParticipants, + } { + derived := filepath.Join(derivedDir, dataset, "*.parquet") + full := filepath.Join(fullDir, dataset, "*.parquet") + var rows, missing, extra int + require.NoError(duckDB.QueryRow(`SELECT + (SELECT count(*) FROM read_parquet(?)), + (SELECT count(*) FROM (SELECT * FROM read_parquet(?) EXCEPT ALL SELECT * FROM read_parquet(?))), + (SELECT count(*) FROM (SELECT * FROM read_parquet(?) EXCEPT ALL SELECT * FROM read_parquet(?)))`, + full, full, derived, derived, full).Scan(&rows, &missing, &extra), dataset) + require.Positive(rows, "%s must hold rows for the comparison to mean anything", dataset) + require.Zero(missing, "%s rows the derived refresh did not write", dataset) + require.Zero(extra, "%s rows only the derived refresh wrote", dataset) + } +} + +func TestCacheExportParityCopyParquetRoundTrip(t *testing.T) { + require := require.New(t) + db, err := sql.Open("duckdb", "") + require.NoError(err) + defer func() { _ = db.Close() }() + dir := filepath.Join(t.TempDir(), "o'brien", "dataset") + + require.NoError(copyParquet(t.Context(), db, dir, "data.parquet", `SELECT 7 AS id, 'seven' AS name`)) + + var id int + var name string + require.NoError(db.QueryRow(`SELECT id, name FROM read_parquet(?)`, filepath.Join(dir, "data.parquet")).Scan(&id, &name)) + require.Equal(7, id) + require.Equal("seven", name) +} diff --git a/cmd/msgvault/cmd/cache_identity_append_test.go b/cmd/msgvault/cmd/cache_identity_append_test.go new file mode 100644 index 000000000..8fbc8f9e8 --- /dev/null +++ b/cmd/msgvault/cmd/cache_identity_append_test.go @@ -0,0 +1,66 @@ +package cmd + +import ( + "crypto/sha256" + "database/sql" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/identityindex" + "go.kenn.io/msgvault/internal/store" +) + +func TestCacheIdentityDriftWithAppendPreservesMessageShards(t *testing.T) { + assert, require := assert.New(t), require.New(t) + tmp := setupTestSQLite(t) + dbPath := filepath.Join(tmp, "test.db") + analyticsDir := filepath.Join(tmp, "analytics") + _, err := buildCache(dbPath, analyticsDir, true) + require.NoError(err) + before := snapshotMessagesDatasetBytes(t, analyticsDir) + st, err := store.Open(dbPath) + require.NoError(err) + _, err = st.LinkParticipants(3, 4) + require.NoError(err) + _, err = st.DB().Exec(`INSERT INTO messages + (id, source_id, source_message_id, conversation_id, message_type, sent_at, sender_id) + VALUES (6, 1, 'message-6', 101, 'email', '2024-03-02 10:00:00', 3)`) + require.NoError(err) + require.NoError(st.Close()) + stale := cacheNeedsBuild(dbPath, analyticsDir) + assert.True(stale.HasIdentityDrift) + assert.True(stale.HasNew) + assert.False(stale.FullRebuild, "identity links do not change baked message facts") + result, err := buildCacheAuto(dbPath, analyticsDir) + require.NoError(err) + assert.Equal(int64(1), result.StagedCount) + after := snapshotMessagesDatasetBytes(t, analyticsDir) + for path, contents := range before { + assert.Equal(sha256.Sum256([]byte(contents)), sha256.Sum256([]byte(after[path])), "retain message shard %s", path) + } + assert.False(cacheNeedsBuild(dbPath, analyticsDir).NeedsBuild) + + fullDir := filepath.Join(tmp, "full") + _, err = buildCache(dbPath, fullDir, true) + require.NoError(err) + duckDB, err := sql.Open("duckdb", "") + require.NoError(err) + defer func() { require.NoError(duckDB.Close()) }() + for _, dataset := range []string{identityindex.DatasetActivity, identityindex.DatasetLogicalContributions, + identityindex.DatasetTemperatureContributions, identityindex.DatasetRelationshipDaily, identityindex.DatasetDomains} { + pattern := func(root string) string { + return "read_parquet('" + strings.ReplaceAll(filepath.ToSlash(filepath.Join(root, dataset, "**", "*.parquet")), "'", "''") + "')" + } + actual, want := pattern(analyticsDir), pattern(fullDir) + var differences int64 + err := duckDB.QueryRow(`SELECT COUNT(*) FROM ( + (SELECT * FROM ` + actual + ` EXCEPT ALL SELECT * FROM ` + want + `) + UNION ALL + (SELECT * FROM ` + want + ` EXCEPT ALL SELECT * FROM ` + actual + `))`).Scan(&differences) + require.NoError(err, dataset) + assert.Zero(differences, dataset) + } +} diff --git a/cmd/msgvault/cmd/cache_jobs.go b/cmd/msgvault/cmd/cache_jobs.go new file mode 100644 index 000000000..2e160f1d5 --- /dev/null +++ b/cmd/msgvault/cmd/cache_jobs.go @@ -0,0 +1,250 @@ +package cmd + +import ( + "context" + "errors" + "log/slog" + "sync" + "time" + + "github.com/google/uuid" + "go.kenn.io/msgvault/internal/api" +) + +// cacheBuildJobs owns detached builds for one daemon. The builder's file lock +// remains the cross-process serialization boundary; the registry coalesces +// requests arriving at this daemon and gives callers a stable status ID. +type cacheBuildJobs struct { + mu sync.Mutex + ctx context.Context + idle *api.IdleTracker + logger *slog.Logger + run func(context.Context, buildCacheMode) error + current string + pending string + pendingMode buildCacheMode + lastVerification time.Time + verificationRetryAt time.Time + jobs map[string]api.CacheBuildStatus + completed []string + wg sync.WaitGroup +} + +// verifyWhenDue queues a full staleness check in the builder subprocess. +// A clean publication keeps its original timestamp, so remember the last +// check to avoid launching another subprocess on every subsequent query. +func (m *cacheBuildJobs) verifyWhenDue(publishedAt time.Time, interval time.Duration, now time.Time) error { + if publishedAt.IsZero() { + return nil + } + if interval <= 0 { + interval = time.Minute + } + if now.Before(publishedAt.Add(interval)) { + return nil + } + m.mu.Lock() + if m.current != "" { + m.mu.Unlock() + return nil + } + if now.Before(m.verificationRetryAt) { + m.mu.Unlock() + return nil + } + due := m.lastVerification.IsZero() || !now.Before(m.lastVerification.Add(interval)) + if due { + m.lastVerification = now + } + m.mu.Unlock() + if !due { + return nil + } + _, err := m.acceptWithFollowup(buildCacheModeScheduledAuto, false, now) + if err != nil { + m.mu.Lock() + if m.lastVerification.Equal(now) { + m.lastVerification = time.Time{} + } + m.mu.Unlock() + } + return err +} + +func newCacheBuildJobs( + ctx context.Context, idle *api.IdleTracker, + run func(context.Context, buildCacheMode) error, +) *cacheBuildJobs { + if ctx == nil { + ctx = context.Background() + } + if run == nil { + run = buildCacheSubprocessMode + } + return &cacheBuildJobs{ctx: ctx, idle: idle, run: run, jobs: make(map[string]api.CacheBuildStatus)} +} + +func (m *cacheBuildJobs) accept(mode buildCacheMode) (api.CacheBuildStatus, error) { + return m.acceptWithFollowup(mode, false, time.Now()) +} + +// A sync may commit while the active builder is reading an older snapshot. +// Return a separate queued job so callers can wait for the later snapshot. +func (m *cacheBuildJobs) acceptAfterWrite(mode buildCacheMode) (api.CacheBuildStatus, error) { + return m.acceptWithFollowup(mode, true, time.Now()) +} + +func (m *cacheBuildJobs) acceptWithFollowup(mode buildCacheMode, afterWrite bool, now time.Time) (api.CacheBuildStatus, error) { + if m == nil { + return api.CacheBuildStatus{}, errors.New("analytics cache build manager unavailable") + } + m.mu.Lock() + defer m.mu.Unlock() + if m.ctx.Err() != nil { + return api.CacheBuildStatus{}, m.ctx.Err() + } + if mode == buildCacheModeScheduledAuto && now.Before(m.verificationRetryAt) { + return api.CacheBuildStatus{}, nil + } + if m.current != "" && !afterWrite { + return m.jobs[m.current], nil + } + if m.pending != "" { + if m.pendingMode == buildCacheModeScheduledAuto && mode != buildCacheModeScheduledAuto { + m.pendingMode = mode + } + return m.jobs[m.pending], nil + } + job := api.CacheBuildStatus{ + JobID: uuid.NewString(), Status: api.CacheBuildQueued, AcceptedAt: now.UTC(), + } + m.jobs[job.JobID] = job + if m.current != "" { + m.pending, m.pendingMode = job.JobID, mode + return job, nil + } + if err := m.startLocked(job.JobID, mode); err != nil { + delete(m.jobs, job.JobID) + return api.CacheBuildStatus{}, err + } + return job, nil +} + +func (m *cacheBuildJobs) startLocked(id string, mode buildCacheMode) error { + done, ok := m.idle.BeginWorkContext(m.ctx) + if !ok { + return errors.New("daemon is shutting down") + } + m.current = id + m.wg.Add(1) + go m.execute(id, mode, done) + return nil +} + +func (m *cacheBuildJobs) execute(id string, mode buildCacheMode, done func()) { + defer m.wg.Done() + defer done() + m.mu.Lock() + job := m.jobs[id] + job.Status = api.CacheBuildRunning + m.jobs[id] = job + m.mu.Unlock() + err := m.run(m.ctx, mode) + m.mu.Lock() + finishedAt := time.Now().UTC() + var failure string + if err != nil { + failure = "analytics cache build failed; see daemon logs" + if m.ctx.Err() != nil { + failure = "daemon is shutting down" + } else { + repairLogger(m.logger).Error("background analytics cache build failed", "job_id", id, "error", err) + } + if mode == buildCacheModeScheduledAuto { + m.lastVerification = time.Time{} + m.verificationRetryAt = finishedAt.Add(time.Minute) + } + } else if mode == buildCacheModeScheduledAuto { + m.verificationRetryAt = time.Time{} + } + m.finishLocked(id, failure, finishedAt) + if m.current == id { + m.current = "" + } + if m.pending != "" { + pending, pendingMode := m.pending, m.pendingMode + m.pending = "" + m.pendingMode = buildCacheModeDefault + switch { + case m.ctx.Err() != nil: + m.finishLocked(pending, "daemon is shutting down", finishedAt) + case pendingMode == buildCacheModeScheduledAuto && finishedAt.Before(m.verificationRetryAt): + m.finishLocked(pending, "analytics cache retry deferred after build failure", finishedAt) + default: + if startErr := m.startLocked(pending, pendingMode); startErr != nil { + m.finishLocked(pending, startErr.Error(), finishedAt) + if m.ctx.Err() == nil { + repairLogger(m.logger).Error("queue follow-up analytics cache build failed", "error", startErr) + } + } + } + } + m.mu.Unlock() +} + +// Keep the last 100 completions. Running and queued jobs are never evicted. +func (m *cacheBuildJobs) finishLocked(id, failure string, finishedAt time.Time) { + job := m.jobs[id] + job.Status = api.CacheBuildPublished + if failure != "" { + job.Status = api.CacheBuildFailed + } + job.Error = failure + job.FinishedAt = &finishedAt + m.jobs[id] = job + m.completed = append(m.completed, id) + if len(m.completed) > 100 { + delete(m.jobs, m.completed[0]) + m.completed = m.completed[1:] + } +} + +func (m *cacheBuildJobs) waitContext(ctx context.Context) bool { + if m == nil { + return true + } + // The caller cancels m.ctx first. Drain any accept already holding the + // mutex so no WaitGroup Add races the wait. + m.mu.Lock() + done := make(chan struct{}) + go func() { + m.wg.Wait() + close(done) + }() + m.mu.Unlock() + select { + case <-done: + return true + case <-ctx.Done(): + return false + } +} + +func (m *cacheBuildJobs) status(id string) (api.CacheBuildStatus, bool) { + if m == nil { + return api.CacheBuildStatus{}, false + } + m.mu.Lock() + defer m.mu.Unlock() + job, ok := m.jobs[id] + return job, ok +} + +func (m *cacheBuildJobs) active() bool { + if m == nil { + return false + } + m.mu.Lock() + defer m.mu.Unlock() + return m.current != "" +} diff --git a/cmd/msgvault/cmd/cache_jobs_test.go b/cmd/msgvault/cmd/cache_jobs_test.go new file mode 100644 index 000000000..80f87b866 --- /dev/null +++ b/cmd/msgvault/cmd/cache_jobs_test.go @@ -0,0 +1,347 @@ +package cmd + +import ( + "bytes" + "context" + "errors" + "fmt" + "log/slog" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/api" +) + +func TestCacheBuildJobsCoalesceAndRetryAfterFailure(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + started := make(chan struct{}, 2) + release := make(chan struct{}) + var calls atomic.Int32 + jobs := newCacheBuildJobs(t.Context(), nil, func(ctx context.Context, mode buildCacheMode) error { + calls.Add(1) + started <- struct{}{} + select { + case <-release: + return errors.New("synthetic build failure") + case <-ctx.Done(): + return ctx.Err() + } + }) + first, err := jobs.accept(buildCacheModeAuto) + require.NoError(err) + select { + case <-started: + case <-time.After(time.Second): + require.FailNow("first cache job did not start") + } + second, err := jobs.accept(buildCacheModeAuto) + require.NoError(err) + assert.Equal(first.JobID, second.JobID) + assert.Equal(int32(1), calls.Load()) + close(release) + require.Eventually(func() bool { + status, ok := jobs.status(first.JobID) + return ok && status.Status == api.CacheBuildFailed + }, time.Second, 10*time.Millisecond) + retry, err := jobs.accept(buildCacheModeAuto) + require.NoError(err) + assert.NotEqual(first.JobID, retry.JobID) + require.Eventually(func() bool { + status, ok := jobs.status(retry.JobID) + return ok && status.Status == api.CacheBuildFailed + }, time.Second, 10*time.Millisecond) + assert.Equal(int32(2), calls.Load()) +} + +func TestCacheBuildJobsVerifyWhenDue(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + started := make(chan buildCacheMode, 2) + jobs := newCacheBuildJobs(t.Context(), nil, func(_ context.Context, mode buildCacheMode) error { + started <- mode + return nil + }) + now := time.Now().UTC() + interval := time.Hour + require.NoError(jobs.verifyWhenDue(now, interval, now)) + assert.Empty(started) + require.NoError(jobs.verifyWhenDue(now.Add(-interval), interval, now)) + require.Eventually(func() bool { return len(started) == 1 }, time.Second, 10*time.Millisecond) + assert.Equal(buildCacheModeScheduledAuto, <-started) + require.NoError(jobs.verifyWhenDue(now.Add(-interval), interval, now.Add(interval/2))) + assert.Empty(started) + require.NoError(jobs.verifyWhenDue(now.Add(-interval), interval, now.Add(interval))) + require.Eventually(func() bool { return len(started) == 1 }, time.Second, 10*time.Millisecond) + assert.Equal(buildCacheModeScheduledAuto, <-started) +} + +func TestCacheBuildJobsSyncDuringBuildGetsFollowup(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + release := make(chan struct{}) + var modes []buildCacheMode + jobs := newCacheBuildJobs(t.Context(), nil, func(_ context.Context, mode buildCacheMode) error { + modes = append(modes, mode) + <-release + return nil + }) + first, err := jobs.accept(buildCacheModeAuto) + require.NoError(err) + synctest.Wait() + pending, err := jobs.acceptAfterWrite(buildCacheModeScheduledAuto) + require.NoError(err) + assert.NotEqual(first.JobID, pending.JobID) + assert.Equal(api.CacheBuildQueued, pending.Status) + coalesced, err := jobs.acceptAfterWrite(buildCacheModeAuto) + require.NoError(err) + assert.Equal(pending.JobID, coalesced.JobID) + release <- struct{}{} + synctest.Wait() + firstStatus, ok := jobs.status(first.JobID) + require.True(ok) + assert.Equal(api.CacheBuildPublished, firstStatus.Status) + pendingStatus, ok := jobs.status(pending.JobID) + require.True(ok) + assert.Equal(api.CacheBuildRunning, pendingStatus.Status) + assert.Equal([]buildCacheMode{buildCacheModeAuto, buildCacheModeAuto}, modes) + // Once the follow-up has started, another write needs a later snapshot. + next, err := jobs.acceptAfterWrite(buildCacheModeAuto) + require.NoError(err) + assert.NotEqual(pending.JobID, next.JobID) + close(release) + synctest.Wait() + for _, id := range []string{pending.JobID, next.JobID} { + status, ok := jobs.status(id) + require.True(ok) + assert.Equal(api.CacheBuildPublished, status.Status) + } + }) +} + +func TestCacheBuildJobsFailedVerificationRetriesSoon(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + var calls atomic.Int32 + jobs := newCacheBuildJobs(t.Context(), nil, func(_ context.Context, mode buildCacheMode) error { + calls.Add(1) + return errors.New("synthetic verification failure") + }) + now := time.Now().UTC() + interval := time.Hour + require.NoError(jobs.verifyWhenDue(now.Add(-interval), interval, now)) + require.Eventually(func() bool { return calls.Load() == 1 && !jobs.active() }, time.Second, 10*time.Millisecond) + require.NoError(jobs.verifyWhenDue(now.Add(-interval), interval, now.Add(30*time.Second))) + assert.Equal(int32(1), calls.Load()) + require.NoError(jobs.verifyWhenDue(now.Add(-interval), interval, now.Add(2*time.Minute))) + require.Eventually(func() bool { return calls.Load() == 2 }, time.Second, 10*time.Millisecond) +} + +func TestCacheBuildJobsWaitsForShutdown(t *testing.T) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + started := make(chan struct{}) + release := make(chan struct{}) + jobs := newCacheBuildJobs(ctx, nil, func(context.Context, buildCacheMode) error { + close(started) + <-release + return nil + }) + _, err := jobs.accept(buildCacheModeAuto) + require.NoError(err) + <-started + cancel() + waitCtx, stopWait := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer stopWait() + require.False(jobs.waitContext(waitCtx)) + close(release) + completeCtx, stopComplete := context.WithTimeout(context.Background(), time.Second) + defer stopComplete() + require.True(jobs.waitContext(completeCtx)) +} + +func TestManualSyncProbeDoesNotQueueCacheRefresh(t *testing.T) { + assert := assert.New(t) + assert.False(manualSyncCLICommand([]string{"sync-circleback", "--probe"})) + assert.False(manualSyncCLICommand([]string{"sync-notion-meetings", "--probe=true"})) + assert.True(manualSyncCLICommand([]string{"sync-notion-meetings", "--limit", "3"})) +} + +func TestCacheBuildJobsScheduledCooldownCoversAllAccepts(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + var calls int + jobs := newCacheBuildJobs(t.Context(), nil, func(context.Context, buildCacheMode) error { + calls++ + return errors.New("synthetic build failure") + }) + first, err := jobs.accept(buildCacheModeScheduledAuto) + require.NoError(err) + synctest.Wait() + synctest.Sleep(30 * time.Second) + for _, accept := range []func(buildCacheMode) (api.CacheBuildStatus, error){jobs.accept, jobs.acceptAfterWrite} { + skipped, err := accept(buildCacheModeScheduledAuto) + require.NoError(err) + assert.Empty(skipped.JobID) + } + require.NoError(jobs.verifyWhenDue(time.Now().Add(-time.Hour), time.Hour, time.Now())) + synctest.Wait() + assert.Equal(1, calls) + explicit, err := jobs.acceptAfterWrite(buildCacheModeAuto) + require.NoError(err) + assert.NotEmpty(explicit.JobID) + synctest.Wait() + assert.Equal(2, calls) + synctest.Sleep(31 * time.Second) + retry, err := jobs.accept(buildCacheModeScheduledAuto) + require.NoError(err) + assert.NotEmpty(retry.JobID) + assert.NotEqual(first.JobID, retry.JobID) + synctest.Wait() + assert.Equal(3, calls) + }) +} + +func TestCacheBuildJobsFailedBuildDefersOnlyScheduledFollowup(t *testing.T) { + for _, mode := range []buildCacheMode{buildCacheModeScheduledAuto, buildCacheModeAuto} { + t.Run(fmt.Sprint(mode), func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + release := make(chan struct{}) + var calls int + jobs := newCacheBuildJobs(t.Context(), nil, func(context.Context, buildCacheMode) error { + calls++ + <-release + if calls == 1 { + return errors.New("synthetic build failure") + } + return nil + }) + _, err := jobs.accept(buildCacheModeScheduledAuto) + require.NoError(err) + synctest.Wait() + pending, err := jobs.acceptAfterWrite(mode) + require.NoError(err) + close(release) + synctest.Wait() + status, ok := jobs.status(pending.JobID) + require.True(ok) + if mode == buildCacheModeScheduledAuto { + assert.Equal(1, calls) + assert.Equal(api.CacheBuildFailed, status.Status) + assert.Contains(status.Error, "retry") + } else { + assert.Equal(2, calls) + assert.Equal(api.CacheBuildPublished, status.Status) + } + }) + }) + } +} + +func TestCacheBuildJobsRetainsRecentCompletionsAndOutstandingJobs(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + release := make(chan struct{}) + var block bool + jobs := newCacheBuildJobs(t.Context(), nil, func(context.Context, buildCacheMode) error { + if block { + <-release + } + return nil + }) + var completed []string + for range 102 { + job, err := jobs.accept(buildCacheModeAuto) + require.NoError(err) + completed = append(completed, job.JobID) + synctest.Wait() + } + for i, id := range completed { + _, ok := jobs.status(id) + assert.Equal(i >= 2, ok) + } + block = true + active, err := jobs.accept(buildCacheModeAuto) + require.NoError(err) + synctest.Wait() + pending, err := jobs.acceptAfterWrite(buildCacheModeAuto) + require.NoError(err) + assert.NotEqual(active.JobID, pending.JobID) + release <- struct{}{} + synctest.Wait() + _, ok := jobs.status(completed[2]) + assert.False(ok) + status, ok := jobs.status(active.JobID) + require.True(ok) + assert.Equal(api.CacheBuildPublished, status.Status) + status, ok = jobs.status(pending.JobID) + require.True(ok) + assert.Equal(api.CacheBuildRunning, status.Status) + close(release) + synctest.Wait() + }) +} + +func TestCacheBuildJobsShutdownSettlesPendingAndQuietsCancellation(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + var logs bytes.Buffer + var calls int + jobs := newCacheBuildJobs(ctx, nil, func(ctx context.Context, _ buildCacheMode) error { + calls++ + <-ctx.Done() + return errors.New("subprocess terminated") + }) + jobs.logger = slog.New(slog.NewTextHandler(&logs, nil)) + first, err := jobs.accept(buildCacheModeAuto) + require.NoError(err) + synctest.Wait() + pending, err := jobs.acceptAfterWrite(buildCacheModeAuto) + require.NoError(err) + cancel() + synctest.Wait() + require.True(jobs.waitContext(t.Context())) + assert.Equal(1, calls) + for _, id := range []string{first.JobID, pending.JobID} { + status, ok := jobs.status(id) + require.True(ok) + assert.Equal(api.CacheBuildFailed, status.Status) + assert.Contains(status.Error, "shutting down") + assert.NotNil(status.FinishedAt) + } + assert.Empty(logs.String()) + assert.False(jobs.active()) + }) +} + +func TestCacheBuildJobsLogsFailureWhileDaemonRunning(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + var logs bytes.Buffer + jobs := newCacheBuildJobs(t.Context(), nil, func(context.Context, buildCacheMode) error { + return errors.New("synthetic disk failure") + }) + jobs.logger = slog.New(slog.NewTextHandler(&logs, nil)) + job, err := jobs.accept(buildCacheModeAuto) + require.NoError(err) + synctest.Wait() + status, ok := jobs.status(job.JobID) + require.True(ok) + assert.Equal(api.CacheBuildFailed, status.Status) + assert.Contains(logs.String(), "synthetic disk failure") + }) +} diff --git a/cmd/msgvault/cmd/cache_publication.go b/cmd/msgvault/cmd/cache_publication.go index a00d86cf2..23d4f3f8e 100644 --- a/cmd/msgvault/cmd/cache_publication.go +++ b/cmd/msgvault/cmd/cache_publication.go @@ -12,6 +12,7 @@ import ( "strings" "time" + "go.kenn.io/kit/atomicfile" "go.kenn.io/kit/daemon" "go.kenn.io/msgvault/internal/identityindex" "go.kenn.io/msgvault/internal/query" @@ -140,6 +141,8 @@ func cachePublishPlanForMode(replaceAll bool) cachePublishPlan { identityindex.DatasetPeople, identityindex.DatasetDomains, identityindex.DatasetRelationshipDaily, + identityindex.DatasetLogicalContributions, + identityindex.DatasetTemperatureContributions, } { plan.Replace[dataset] = true } @@ -262,8 +265,8 @@ func syncStagedMoveContents(source string) error { return err } for _, directory := range slices.Backward(directories) { - if err := syncDirectory(directory); err != nil { - return err + if err := atomicfile.SyncDir(directory); err != nil { + return fmt.Errorf("sync staged directory: %w", err) } } return nil @@ -278,8 +281,8 @@ func syncStagedMoveContents(source string) error { func syncDestinationParents(destination, analyticsDir string) error { root := filepath.Clean(analyticsDir) for dir := filepath.Dir(filepath.Clean(destination)); ; dir = filepath.Dir(dir) { - if err := syncDirectory(dir); err != nil { - return err + if err := atomicfile.SyncDir(dir); err != nil { + return fmt.Errorf("sync destination parent: %w", err) } if dir == root || dir == filepath.Dir(dir) { return nil @@ -391,24 +394,12 @@ func publishCacheWithBeforeMarker( if err != nil { return fmt.Errorf("encode committed cache sync state: %w", err) } - return commitCacheMarker(analyticsDir, staging.buildID, stateData) + return commitCacheMarker(analyticsDir, stateData) } -func commitCacheMarker(analyticsDir, buildID string, stateData []byte) error { - statePath := query.CacheStatePath(analyticsDir) - tempPath := filepath.Join(analyticsDir, ".last-sync-"+buildID+".tmp") - if err := buildCacheWriteStateFile(tempPath, stateData, 0o600); err != nil { - return fmt.Errorf("write staged cache marker: %w", err) - } - defer func() { _ = os.Remove(tempPath) }() - if err := syncFile(tempPath); err != nil { - return fmt.Errorf("sync staged cache marker: %w", err) - } - if err := os.Rename(tempPath, statePath); err != nil { +func commitCacheMarker(analyticsDir string, stateData []byte) error { + if err := buildCacheWriteStateFile(query.CacheStatePath(analyticsDir), stateData, 0o600); err != nil { return fmt.Errorf("commit cache marker: %w", err) } - if err := syncDirectory(analyticsDir); err != nil { - return fmt.Errorf("sync committed cache marker: %w", err) - } return nil } diff --git a/cmd/msgvault/cmd/cache_publication_sync_nonwindows.go b/cmd/msgvault/cmd/cache_publication_sync_nonwindows.go index 0f17f056d..a2c93c5a0 100644 --- a/cmd/msgvault/cmd/cache_publication_sync_nonwindows.go +++ b/cmd/msgvault/cmd/cache_publication_sync_nonwindows.go @@ -13,13 +13,3 @@ func syncFile(path string) error { defer func() { _ = file.Close() }() return file.Sync() } - -func syncDirectory(path string) error { - // #nosec G703 -- callers pass configured cache parents or validated transaction paths. - directory, err := os.Open(path) - if err != nil { - return err - } - defer func() { _ = directory.Close() }() - return directory.Sync() -} diff --git a/cmd/msgvault/cmd/cache_publication_sync_windows.go b/cmd/msgvault/cmd/cache_publication_sync_windows.go index 522870de5..d3acbb1c8 100644 --- a/cmd/msgvault/cmd/cache_publication_sync_windows.go +++ b/cmd/msgvault/cmd/cache_publication_sync_windows.go @@ -32,13 +32,3 @@ func syncFile(path string) error { } return nil } - -// Windows has no supported equivalent of fsync(2) for directory handles: -// FlushFileBuffers rejects directory handles even when opened with -// FILE_FLAG_BACKUP_SEMANTICS. Regular file bytes are flushed before namespace -// publication. A successful rename is the supported Windows namespace -// boundary available to this transaction. This no-op matches the config -// publication contract without claiming a directory flush occurred. -func syncDirectory(string) error { - return nil -} diff --git a/cmd/msgvault/cmd/cache_publication_test.go b/cmd/msgvault/cmd/cache_publication_test.go index 7791d3a77..39a06eaa2 100644 --- a/cmd/msgvault/cmd/cache_publication_test.go +++ b/cmd/msgvault/cmd/cache_publication_test.go @@ -208,7 +208,7 @@ func TestBuilderLockDoesNotBlockReaders(t *testing.T) { analyticsDir := filepath.Join(t.TempDir(), "analytics") requirements.NoError(os.MkdirAll(analyticsDir, 0o755)) - builderLock, err := acquireCacheBuildLock(analyticsDir) + builderLock, err := acquireCacheBuildLock(context.Background(), analyticsDir) requirements.NoError(err, "acquire builder lock") readCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second) diff --git a/cmd/msgvault/cmd/cache_refresh_test.go b/cmd/msgvault/cmd/cache_refresh_test.go index a4a39d427..c2315a9b2 100644 --- a/cmd/msgvault/cmd/cache_refresh_test.go +++ b/cmd/msgvault/cmd/cache_refresh_test.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "io/fs" + "log/slog" "os" "path/filepath" "strings" @@ -55,6 +56,8 @@ func TestDerivedOnlyRefusesStaleSchemaBeforeCreatingStaging(t *testing.T) { } func TestProductionCacheBuilderOpenSitesUseConfiguredOverrides(t *testing.T) { + cfg := testConfigValue() + savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) @@ -68,8 +71,12 @@ func TestProductionCacheBuilderOpenSitesUseConfiguredOverrides(t *testing.T) { t.Run("full build", func(t *testing.T) { tmp := setupTestSQLite(t) configure(tmp, filepath.Join(tmp, "test.db")) + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) - err := runBuildCacheLocalMode(buildCacheModeFull) + err := runBuildCacheLocalMode( + buildCacheModeFull, + invocationFromContext(testCtx), + ) require.Error(t, err) assert.Contains(t, err.Error(), "memory_limit") @@ -82,8 +89,12 @@ func TestProductionCacheBuilderOpenSitesUseConfiguredOverrides(t *testing.T) { _, err := buildCache(dbPath, analyticsDir, true) require.NoError(t, err) configure(tmp, dbPath) + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) - err = runBuildCacheLocalMode(buildCacheModeDerived) + err = runBuildCacheLocalMode( + buildCacheModeDerived, + invocationFromContext(testCtx), + ) require.Error(t, err) assert.Contains(t, err.Error(), "memory_limit") @@ -137,17 +148,16 @@ func TestDerivedOnlyRefreshCarriesStatsAndRefreshesMembershipRollups(t *testing. defer func() { require.NoError(t, duckDB.Close()) }() var membershipRows int64 requirementsForTest.NoError(duckDB.QueryRow(` - SELECT count(DISTINCT message_id) - FROM read_parquet(?, hive_partitioning = true) - WHERE conversation_id = 102 - AND canonical_id = 3 - AND is_conversation_member + SELECT count(DISTINCT a.message_id) + FROM read_parquet(?, hive_partitioning = true) a + JOIN read_parquet(?) cp ON cp.conversation_id = a.conversation_id + WHERE a.conversation_id = 102 AND cp.participant_id = 3 `, filepath.Join( analyticsDir, identityindex.DatasetActivity, "**", "*.parquet", - )).Scan(&membershipRows)) + ), filepath.Join(analyticsDir, "conversation_participants", "*.parquet")).Scan(&membershipRows)) assertionsForTest.Positive(membershipRows) } @@ -325,6 +335,8 @@ func snapshotMessagesDatasetBytes(t *testing.T, root string) map[string]string { } func TestRebuildCacheAfterWriteReturnsError(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) tmpDir := t.TempDir() dbPath := filepath.Join(tmpDir, "msgvault.db") @@ -336,17 +348,21 @@ func TestRebuildCacheAfterWriteReturnsError(t *testing.T) { savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = &config.Config{HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx sentinel := errors.New("cache export sentinel") buildCacheBeforeMessagesExportHook = func() error { return sentinel } t.Cleanup(func() { buildCacheBeforeMessagesExportHook = nil }) - err = rebuildCacheAfterWrite(dbPath) + err = rebuildCacheAfterWrite(dbPath, invocationFromContext(testCtx)) require.ErrorIs(err, sentinel) require.ErrorContains(err, "refresh analytics cache") } func TestRebuildCacheAfterDerivedRepairRefreshesCurrentCache(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) tmpDir := t.TempDir() @@ -355,7 +371,11 @@ func TestRebuildCacheAfterDerivedRepairRefreshesCurrentCache(t *testing.T) { savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = &config.Config{HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx analyticsDir := cfg.AnalyticsDir() + var logs strings.Builder + invocationFromContext(testCtx).logger = slog.New(slog.NewTextHandler(&logs, nil)) st, err := store.Open(dbPath) require.NoError(err) @@ -437,7 +457,7 @@ func TestRebuildCacheAfterDerivedRepairRefreshesCurrentCache(t *testing.T) { st, err = store.Open(dbPath) require.NoError(err) sum, err := rederive.Run( - context.Background(), st, "beeper", source.Identifier, source.ID, nil, + testCtx, st, "beeper", source.Identifier, source.ID, nil, ) require.NoError(err) require.Zero(sum.Errors) @@ -449,7 +469,9 @@ func TestRebuildCacheAfterDerivedRepairRefreshesCurrentCache(t *testing.T) { assert.True(staleness.FullRebuild, "an incremental append cannot replace already-cached repaired rows") - require.NoError(rebuildCacheAfterWrite(dbPath)) + require.NoError(rebuildCacheAfterWrite(dbPath, invocationFromContext(testCtx))) + assert.Contains(logs.String(), "cache rebuilt") + assert.Contains(logs.String(), "exported=") repairedState, err := query.ReadCacheSyncState(analyticsDir) require.NoError(err) assert.Equal(int64(3), repairedState.DerivedDataRevision, @@ -461,6 +483,8 @@ func TestRebuildCacheAfterDerivedRepairRefreshesCurrentCache(t *testing.T) { } func TestRebuildCacheAfterMixedBeeperMetadataRefreshRebuildsExistingRows(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) tmpDir := t.TempDir() @@ -468,7 +492,10 @@ func TestRebuildCacheAfterMixedBeeperMetadataRefreshRebuildsExistingRows(t *test savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = &config.Config{HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx analyticsDir := cfg.AnalyticsDir() + invocationFromContext(testCtx).logger = testDiscardLogger() st, err := store.Open(dbPath) require.NoError(err) @@ -517,10 +544,10 @@ func TestRebuildCacheAfterMixedBeeperMetadataRefreshRebuildsExistingRows(t *test require.True(staleness.NeedsBuild) assert.True(staleness.HasDerivedDataDrift) assert.True(staleness.FullRebuild) - require.NoError(rebuildCacheAfterWrite(dbPath)) + require.NoError(rebuildCacheAfterWrite(dbPath, invocationFromContext(testCtx))) engine, err := query.NewDuckDBEngine(analyticsDir, "", nil) require.NoError(err) - result, err := engine.QuerySQL(context.Background(), ` + result, err := engine.QuerySQL(testCtx, ` SELECT m.source_message_id, a.attachment_metadata FROM messages m JOIN attachments a ON a.message_id = m.id WHERE m.source_message_id IN ('mixed-old', 'mixed-new') @@ -535,6 +562,8 @@ func TestRebuildCacheAfterMixedBeeperMetadataRefreshRebuildsExistingRows(t *test } func TestScheduledCacheRefreshSkipsWhenAutoBuildCacheDisabled(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) tmpDir := t.TempDir() @@ -548,6 +577,8 @@ func TestScheduledCacheRefreshSkipsWhenAutoBuildCacheDisabled(t *testing.T) { AutoBuildCache: false, }, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx builds := 0 oldRunBuild := runScheduledBuildCacheSubprocess @@ -557,7 +588,7 @@ func TestScheduledCacheRefreshSkipsWhenAutoBuildCacheDisabled(t *testing.T) { } t.Cleanup(func() { runScheduledBuildCacheSubprocess = oldRunBuild }) - err := rebuildCacheAfterScheduledSync(context.Background(), "disabled") + err := rebuildCacheAfterScheduledSync(testCtx, "disabled") require.NoError(err) assert.Zero(builds, "disabled auto_build_cache must not start a cache build") } @@ -643,18 +674,32 @@ func TestScheduledCacheBuildDelay(t *testing.T) { } func TestScheduledCacheRefreshMinimumInterval(t *testing.T) { + cfg := testConfigValue() + now := time.Date(2026, 8, 12, 12, 0, 0, 0, time.UTC) sentinel := errors.New("cache build sentinel") tests := []struct { - name string - publishedAt time.Time - buildErr error - wantBuilds int + name string + publishedAt time.Time + fullRebuildRequired bool + buildErr error + wantBuilds int }{ { name: "recent publication suppresses build", publishedAt: now.Add(-time.Hour), }, + { + name: "recent partial publication suppresses build", + publishedAt: now.Add(-time.Hour), + fullRebuildRequired: true, + }, + { + name: "elapsed partial publication permits repair", + publishedAt: now.Add(-7 * time.Hour), + fullRebuildRequired: true, + wantBuilds: 1, + }, { name: "elapsed interval permits build", publishedAt: now.Add(-7 * time.Hour), @@ -686,6 +731,7 @@ func TestScheduledCacheRefreshMinimumInterval(t *testing.T) { state, err := query.ReadCacheSyncState(analyticsDir) requirements.NoError(err) state.PublishedAt = tt.publishedAt + state.FullRebuildRequired = tt.fullRebuildRequired stateData, err := json.Marshal(state) requirements.NoError(err) requirements.NoError(os.WriteFile(query.CacheStatePath(analyticsDir), stateData, 0o600)) @@ -712,6 +758,7 @@ func TestScheduledCacheRefreshMinimumInterval(t *testing.T) { }, } t.Cleanup(func() { cfg = savedCfg }) + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) oldNow := scheduledCacheBuildNow scheduledCacheBuildNow = func() time.Time { return now } @@ -725,7 +772,7 @@ func TestScheduledCacheRefreshMinimumInterval(t *testing.T) { } t.Cleanup(func() { runScheduledBuildCacheSubprocess = oldRunBuild }) - err = rebuildCacheAfterScheduledSync(context.Background(), "test-source") + err = rebuildCacheAfterScheduledSync(testCtx, "test-source") if tt.buildErr != nil { requirements.ErrorIs(err, tt.buildErr) } else { @@ -741,11 +788,14 @@ func TestScheduledCacheRefreshMinimumInterval(t *testing.T) { } func TestRepairEncodingReturnsCacheRefreshError(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) tmpDir := t.TempDir() savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = &config.Config{HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) stateFile := filepath.Join(cfg.AnalyticsDir(), "_last_sync.json") require.NoError(os.MkdirAll(cfg.AnalyticsDir(), 0o755)) require.NoError(os.WriteFile(stateFile, []byte(`{"schema_version":18}`), 0o600)) @@ -754,7 +804,9 @@ func TestRepairEncodingReturnsCacheRefreshError(t *testing.T) { buildCacheBeforeMessagesExportHook = func() error { return sentinel } t.Cleanup(func() { buildCacheBeforeMessagesExportHook = nil }) - err := runRepairEncodingLocal(&cobra.Command{}) + cmd := &cobra.Command{} + cmd.SetContext(testCtx) + err := runRepairEncodingLocal(cmd) require.ErrorIs(err, sentinel) require.ErrorContains(err, "encoding repair completed") require.ErrorContains(err, "analytics cache refresh failed") @@ -763,6 +815,8 @@ func TestRepairEncodingReturnsCacheRefreshError(t *testing.T) { } func TestScheduledCacheRefreshFailurePreservesCompletedSyncRun(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) tmpDir := t.TempDir() @@ -793,6 +847,8 @@ func TestScheduledCacheRefreshFailurePreservesCompletedSyncRun(t *testing.T) { AutoBuildCache: true, }, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx sentinel := errors.New("scheduled cache sentinel") oldRunBuild := runScheduledBuildCacheSubprocess @@ -802,7 +858,7 @@ func TestScheduledCacheRefreshFailurePreservesCompletedSyncRun(t *testing.T) { getOAuthMgr := func(string) (*oauth.Manager, error) { return nil, errors.New("unexpected Gmail OAuth path") } - err = runScheduledSync(context.Background(), identifier, st, getOAuthMgr) + err = runScheduledSync(testCtx, identifier, st, getOAuthMgr, invocationFromContext(testCtx)) require.ErrorIs(err, sentinel, "cache failure must reach the scheduled job result") require.ErrorContains(err, "refresh analytics cache") @@ -930,6 +986,46 @@ func TestConversationTitleDriftDetectedAndRepairedByDerivedRefresh(t *testing.T) assertions.Equal("Updated cache title", title) } +func TestCSVInvalidConversationTitleKeepsIncrementalCacheFresh(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + t.Setenv("MSGVAULT_FORCE_CSV_SNAPSHOT", "1") + tmp := setupTestSQLite(t) + dbPath := filepath.Join(tmp, "test.db") + analyticsDir := filepath.Join(tmp, "analytics") + db, err := sql.Open("sqlite3", dbPath) + require.NoError(err) + _, err = db.Exec(`UPDATE conversations SET title = CAST(X'4261642080' AS TEXT) WHERE id = 102`) + require.NoError(err) + require.NoError(db.Close()) + + _, err = buildCache(dbPath, analyticsDir, true) + require.NoError(err) + fresh := cacheNeedsBuild(dbPath, analyticsDir) + assert.False(fresh.NeedsBuild, "unchanged invalid title must match the published snapshot: %s", fresh.Reason) + + db, err = sql.Open("sqlite3", dbPath) + require.NoError(err) + _, err = db.Exec(`INSERT INTO messages (id, conversation_id, source_id, source_message_id, sent_at) + VALUES (99, 102, 1, 'msg99', '2026-09-23 10:00:00')`) + require.NoError(err) + require.NoError(db.Close()) + appendState := cacheNeedsBuild(dbPath, analyticsDir) + assert.True(appendState.HasNew) + assert.False(appendState.HasConversationTypeDrift, "unchanged title must not look like metadata drift") + assert.False(appendState.FullRebuild, "new messages alone remain eligible for incremental export") + + db, err = sql.Open("sqlite3", dbPath) + require.NoError(err) + _, err = db.Exec(`UPDATE conversations SET title = 'Changed title' WHERE id = 102`) + require.NoError(err) + require.NoError(db.Close()) + changed := cacheNeedsBuild(dbPath, analyticsDir) + assert.True(changed.NeedsBuild) + assert.True(changed.HasConversationTypeDrift, "a real title change must still be detected") + assert.True(changed.FullRebuild, "metadata drift with a new message requires rewriting committed rows") +} + func TestFullBuildForcedWhenTypeDriftCoincidesWithNewMessages(t *testing.T) { requirements := require.New(t) assertions := assert.New(t) @@ -1242,12 +1338,16 @@ func TestIncrementalBuildRepairsParticipantIdentifierDriftWithNewMessages(t *tes } func TestRepairEncodingRebuildsCacheWithRegeneratedCalendarSnippet(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) tmpDir := t.TempDir() savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = &config.Config{HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx dbPath := cfg.DatabaseDSN() st, err := store.Open(dbPath) @@ -1275,7 +1375,8 @@ func TestRepairEncodingRebuildsCacheWithRegeneratedCalendarSnippet(t *testing.T) require.NoError(st.Close(), "close fixture store") cmd := &cobra.Command{} - cmd.SetContext(context.Background()) + cmd.SetContext(testCtx) + cmd.SetContext(testCtx) require.NoError(runRepairEncodingLocal(cmd), "run local encoding repair") st, err = store.Open(dbPath) @@ -1405,3 +1506,204 @@ func TestDerivedOnlyStaleSchemaRequiresAndCompletesFullRebuild(t *testing.T) { requirements.NoError(err) assertions.Equal(query.CacheReady, readiness) } + +// setupScheduledRefreshFixture publishes a usable cache marker at publishedAt +// and adds one message after it, so the cache is stale. +func setupScheduledRefreshFixture(t *testing.T, now, publishedAt time.Time) (string, context.Context) { + t.Helper() + tmpDir := setupTestSQLiteEmpty(t) + dbPath := filepath.Join(tmpDir, "test.db") + analyticsDir := filepath.Join(tmpDir, "analytics") + writeSyncStateAt(t, analyticsDir, 0, now.Add(-24*time.Hour)) + createFakeParquet(t, analyticsDir) + state, err := query.ReadCacheSyncState(analyticsDir) + require.NoError(t, err) + state.PublishedAt = publishedAt + stateData, err := json.Marshal(state) + require.NoError(t, err) + require.NoError(t, os.WriteFile(query.CacheStatePath(analyticsDir), stateData, 0o600)) + + db, err := sql.Open("sqlite3", dbPath) + require.NoError(t, err) + _, err = db.Exec(`INSERT INTO messages (id, source_id, source_message_id, sent_at) VALUES (1, 1, 'new-message', ?)`, now) + require.NoError(t, err) + require.NoError(t, db.Close()) + + cfg := &config.Config{ + HomeDir: tmpDir, + Data: config.DataConfig{DataDir: tmpDir, DatabaseURL: dbPath}, + Analytics: config.AnalyticsConfig{ + AutoBuildCache: true, + MinRebuildInterval: 6 * time.Hour, + }, + } + oldNow := scheduledCacheBuildNow + scheduledCacheBuildNow = func() time.Time { return now } + t.Cleanup(func() { scheduledCacheBuildNow = oldNow }) + return analyticsDir, testInvocationContext(t.Context(), cfg, invocationOptions{}) +} + +func TestRebuildAfterSyncRunsThrottleCheckOffOperationGate(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + now := time.Date(2026, 8, 12, 12, 0, 0, 0, time.UTC) + analyticsDir, testCtx := setupScheduledRefreshFixture(t, now, now.Add(-time.Hour)) + buildLock, err := cacheBuilderFileLock(analyticsDir) + require.NoError(err) + require.NoError(buildLock.Lock(), "simulate a cache build in progress") + t.Cleanup(func() { _ = buildLock.Unlock() }) + + oldRunBuild := runScheduledBuildCacheSubprocess + runScheduledBuildCacheSubprocess = func(context.Context) error { + return errors.New("unexpected scheduled cache build") + } + t.Cleanup(func() { runScheduledBuildCacheSubprocess = oldRunBuild }) + + checked := make(chan struct{}, 1) + var delays []time.Duration + refresher := newBackgroundCacheRefresher(testCtx, nil, nil) + run := refresher.run + refresher.run = func(ctx context.Context, id string) error { + err := run(ctx, id) + checked <- struct{}{} + return err + } + refresher.afterFunc = func(d time.Duration, fn func()) *time.Timer { + delays = append(delays, d) + return time.AfterFunc(24*time.Hour, fn) + } + oldRefresher := daemonCacheRefresher + daemonCacheRefresher = refresher + t.Cleanup(func() { + require.NoError(shutdownBackgroundCacheRefresher(refresher)) + daemonCacheRefresher = oldRefresher + }) + + done := make(chan error, 1) + go func() { done <- rebuildCacheAfterScheduledSync(testCtx, "test-source") }() + select { + case err := <-done: + require.NoError(err) + case <-time.After(10 * time.Second): + require.FailNow("post-sync check waited on the builder lock") + } + require.NoError(buildLock.Unlock()) + select { + case <-checked: + case <-time.After(10 * time.Second): + require.FailNow("background readiness check did not run") + } + refresher.mu.Lock() + hasDelayedRequest := refresher.delayed != nil + refresher.mu.Unlock() + require.True(hasDelayedRequest, "a throttled authoritative check should retain one delayed retry") + assert.Equal([]time.Duration{5 * time.Hour}, delays, + "a throttled check schedules one refresh for when the interval ends") +} + +func TestRebuildAfterSyncDoesNotThrottleDriftedPublication(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + now := time.Date(2026, 8, 12, 12, 0, 0, 0, time.UTC) + analyticsDir, testCtx := setupScheduledRefreshFixture(t, now, now.Add(-time.Hour)) + // Keep a shard in every dataset while changing the committed file set. + // The old fast check saw only the marker and shard presence, so it could + // incorrectly throttle this damaged publication as recent. + for _, dataset := range query.RequiredParquetDirs { + require.NoError(os.WriteFile(filepath.Join(analyticsDir, dataset, "new-shard.parquet"), []byte("new shard"), 0o600)) + } + + builds := 0 + oldRunBuild := runScheduledBuildCacheSubprocess + runScheduledBuildCacheSubprocess = func(context.Context) error { + builds++ + return nil + } + t.Cleanup(func() { runScheduledBuildCacheSubprocess = oldRunBuild }) + oldRefresher := daemonCacheRefresher + daemonCacheRefresher = nil + t.Cleanup(func() { daemonCacheRefresher = oldRefresher }) + + require.NoError(rebuildCacheAfterScheduledSync(testCtx, "test-source")) + assert.Equal(1, builds, "a changed shard set must bypass the interval throttle") +} + +func TestRebuildAfterSyncSchedulesRetryWhenBuildIsSuperseded(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + now := time.Date(2026, 8, 12, 12, 0, 0, 0, time.UTC) + analyticsDir, testCtx := setupScheduledRefreshFixture(t, now, now.Add(-7*time.Hour)) + + oldRunBuild := runScheduledBuildCacheSubprocess + runScheduledBuildCacheSubprocess = func(context.Context) error { + // The child rechecks staleness under the builder lock and can defer + // inside the interval after a concurrent publisher advances the marker. + state, err := query.ReadCacheSyncState(analyticsDir) + require.NoError(err) + state.PublishedAt = now.Add(-time.Hour) + stateData, err := json.Marshal(state) + require.NoError(err) + require.NoError(os.WriteFile(query.CacheStatePath(analyticsDir), stateData, 0o600)) + return nil // the child deferred its rebuild because the publication is recent + } + t.Cleanup(func() { runScheduledBuildCacheSubprocess = oldRunBuild }) + + var retryDelay time.Duration + var retryID string + require.NoError(rebuildCacheNow(testCtx, "test-source", func(delay time.Duration, id string) { + retryDelay, retryID = delay, id + })) + assert.Equal(5*time.Hour, retryDelay, + "staleness left after the child skips must retry when the newer publication leaves its throttle window") + assert.Equal("test-source", retryID) +} + +func TestRebuildAfterSyncHandsOffToRefresher(t *testing.T) { + now := time.Date(2026, 8, 12, 12, 0, 0, 0, time.UTC) + _, testCtx := setupScheduledRefreshFixture(t, now, now.Add(-7*time.Hour)) + + oldRunBuild := runScheduledBuildCacheSubprocess + runScheduledBuildCacheSubprocess = func(context.Context) error { + return errors.New("post-sync path must not build inline") + } + t.Cleanup(func() { runScheduledBuildCacheSubprocess = oldRunBuild }) + + requested := make(chan string, 1) + refresher := newBackgroundCacheRefresher(testCtx, func(_ context.Context, id string) error { + requested <- id + return nil + }, nil) + oldRefresher := daemonCacheRefresher + daemonCacheRefresher = refresher + t.Cleanup(func() { + daemonCacheRefresher = oldRefresher + require.NoError(t, refresher.Shutdown(context.Background())) + }) + + require.NoError(t, rebuildCacheAfterScheduledSync(testCtx, "test-source")) + select { + case id := <-requested: + assert.Equal(t, "test-source", id) + case <-time.After(10 * time.Second): + require.FailNow(t, "due rebuild was not handed to the refresher") + } +} + +func TestCacheRefresherShutdownWhileWaitingForBuilderLock(t *testing.T) { + require := require.New(t) + now := time.Date(2026, 8, 12, 12, 0, 0, 0, time.UTC) + analyticsDir, testCtx := setupScheduledRefreshFixture(t, now, now.Add(-time.Hour)) + buildLock, err := cacheBuilderFileLock(analyticsDir) + require.NoError(err) + require.NoError(buildLock.Lock()) + refresher := newBackgroundCacheRefresher(testCtx, nil, nil) + t.Cleanup(func() { + require.NoError(buildLock.Unlock()) + require.NoError(refresher.Shutdown(context.Background())) + }) + + require.True(refresher.Request("sync")) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + assert.NoError(t, refresher.Shutdown(ctx), "shutdown cancels a readiness check blocked behind another builder") +} diff --git a/cmd/msgvault/cmd/cache_refresher.go b/cmd/msgvault/cmd/cache_refresher.go new file mode 100644 index 000000000..46dbee1bc --- /dev/null +++ b/cmd/msgvault/cmd/cache_refresher.go @@ -0,0 +1,178 @@ +package cmd + +import ( + "context" + "sync" + "time" + + "go.kenn.io/msgvault/internal/scheduler" +) + +// daemonCacheRefresher runs due analytics cache rebuilds for the daemon. It is +// nil outside the daemon, where post-sync rebuilds run synchronously. +var daemonCacheRefresher *backgroundCacheRefresher + +// backgroundCacheRefresher runs analytics cache rebuilds off the scheduled job +// that requested them, so a sync releases the daemon operation gate as soon as +// its own work is done. It runs one rebuild at a time and collapses requests +// that arrive meanwhile into a single follow-up, so no request is lost. It +// holds no operation gate: builds read a SQLite snapshot and tolerate +// concurrent writes. +type backgroundCacheRefresher struct { + ctx context.Context + cancel context.CancelFunc + run func(context.Context, string) error + work scheduler.WorkTracker + + // afterFunc schedules delayed requests; time.AfterFunc outside tests. + afterFunc func(time.Duration, func()) *time.Timer + + mu sync.Mutex + running bool + pending bool + pendingID string + closed bool + // delayed is the single outstanding RequestAfter timer, due at delayedAt. + delayed *time.Timer + delayedAt time.Time + wg sync.WaitGroup +} + +func newBackgroundCacheRefresher( + ctx context.Context, + run func(context.Context, string) error, + work scheduler.WorkTracker, +) *backgroundCacheRefresher { + ctx, cancel := context.WithCancel(ctx) + r := &backgroundCacheRefresher{ + ctx: ctx, cancel: cancel, run: run, work: work, afterFunc: time.AfterFunc, + } + if r.run == nil { + r.run = func(ctx context.Context, identifier string) error { + return rebuildCacheNow(ctx, identifier, r.RequestAfter) + } + } + return r +} + +// Request asks for a rebuild check on behalf of identifier. It returns false +// once shutdown has begun. +func (r *backgroundCacheRefresher) Request(identifier string) bool { + r.mu.Lock() + defer r.mu.Unlock() + if r.closed { + return false + } + if r.running { + r.pending = true + r.pendingID = identifier + return true + } + r.running = true + r.wg.Add(1) + go r.loop(identifier) + return true +} + +// RequestAfter issues Request(identifier) once delay has elapsed, unless the +// refresher shuts down first. Only the earliest outstanding delayed request +// is kept: every sync inside a throttle window asks for the same refresh, and +// the rebuild it triggers covers all of them. +func (r *backgroundCacheRefresher) RequestAfter(delay time.Duration, identifier string) { + r.mu.Lock() + defer r.mu.Unlock() + if r.closed { + return + } + due := time.Now().Add(delay) + if r.delayed != nil { + if !due.Before(r.delayedAt) { + return + } + r.delayed.Stop() + } + r.delayedAt = due + var timer *time.Timer + timer = r.afterFunc(delay, func() { + r.mu.Lock() + if r.delayed == timer { + r.delayed = nil + } + r.mu.Unlock() + r.Request(identifier) + }) + r.delayed = timer +} + +// Shutdown refuses new requests, cancels the running rebuild, and waits for +// it to stop or for ctx to end. +func (r *backgroundCacheRefresher) Shutdown(ctx context.Context) error { + r.mu.Lock() + r.closed = true + r.pending = false + if r.delayed != nil { + r.delayed.Stop() + r.delayed = nil + } + r.mu.Unlock() + r.cancel() + + done := make(chan struct{}) + go func() { + r.wg.Wait() + close(done) + }() + select { + case <-done: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// shutdownBackgroundCacheRefresher covers startup exits that return before the +// daemon reaches its normal shutdown sequence. Shutdown is safe to call again +// there after the normal path has already drained the worker. +func shutdownBackgroundCacheRefresher(r *backgroundCacheRefresher) error { + if r == nil { + return nil + } + ctx, cancel := context.WithTimeout(context.Background(), serveOperationDrainTimeout) + defer cancel() + if err := r.Shutdown(ctx); err != nil { + loggerFromContext(r.ctx).Warn("analytics cache refresh did not stop during cleanup", "error", err) + return err + } + return nil +} + +func (r *backgroundCacheRefresher) loop(identifier string) { + defer r.wg.Done() + for { + r.runOnce(identifier) + r.mu.Lock() + if !r.pending || r.closed { + r.pending = false + r.running = false + r.mu.Unlock() + return + } + identifier = r.pendingID + r.pending = false + r.mu.Unlock() + } +} + +func (r *backgroundCacheRefresher) runOnce(identifier string) { + if r.work != nil { + done, ok := r.work.BeginWorkContext(r.ctx) + if !ok { + return + } + defer done() + } + if err := r.run(r.ctx, identifier); err != nil && r.ctx.Err() == nil { + loggerFromContext(r.ctx).Warn("background analytics cache refresh failed", + "identifier", identifier, "error", err) + } +} diff --git a/cmd/msgvault/cmd/cache_refresher_test.go b/cmd/msgvault/cmd/cache_refresher_test.go new file mode 100644 index 000000000..d13edf1fa --- /dev/null +++ b/cmd/msgvault/cmd/cache_refresher_test.go @@ -0,0 +1,167 @@ +package cmd + +import ( + "context" + "sync" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRefresherCoalescesRequests(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + var runs atomic.Int32 + release := make(chan struct{}) + var mu sync.Mutex + var seen []string + r := newBackgroundCacheRefresher(context.Background(), func(ctx context.Context, id string) error { + mu.Lock() + seen = append(seen, id) + mu.Unlock() + if runs.Add(1) == 1 { + <-release + } + return nil + }, nil) + + require.True(r.Request("first")) + synctest.Wait() + require.True(r.Request("second")) + require.True(r.Request("third")) + require.True(r.Request("fourth")) + close(release) + synctest.Wait() + + assert.Equal(int32(2), runs.Load(), "requests during a run collapse into one follow-up") + assert.Equal([]string{"first", "fourth"}, seen) + require.NoError(r.Shutdown(context.Background())) + }) +} + +func TestRefresherShutdownCancelsAndWaits(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + started := make(chan struct{}) + var finished atomic.Bool + r := newBackgroundCacheRefresher(context.Background(), func(ctx context.Context, id string) error { + close(started) + <-ctx.Done() + time.Sleep(time.Second) + finished.Store(true) + return ctx.Err() + }, nil) + + require.True(r.Request("sync")) + <-started + require.NoError(r.Shutdown(context.Background())) + assert.True(finished.Load(), "Shutdown returns only after the running build stops") + assert.False(r.Request("late"), "requests after shutdown are refused") + }) +} + +func TestRefresherShutdownStopsDelayedRequests(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + var runs atomic.Int32 + r := newBackgroundCacheRefresher(context.Background(), func(context.Context, string) error { + runs.Add(1) + return nil + }, nil) + r.RequestAfter(time.Hour, "startup") + time.Sleep(30 * time.Minute) + require.NoError(r.Shutdown(context.Background())) + time.Sleep(time.Hour) + synctest.Wait() + assert.Equal(int32(0), runs.Load()) + + r2 := newBackgroundCacheRefresher(context.Background(), func(context.Context, string) error { + runs.Add(1) + return nil + }, nil) + r2.RequestAfter(time.Hour, "startup") + time.Sleep(time.Hour + time.Second) + synctest.Wait() + assert.Equal(int32(1), runs.Load(), "delayed request fires after its delay") + require.NoError(r2.Shutdown(context.Background())) + }) +} + +func TestShutdownBackgroundCacheRefresherStopsDelayedRequests(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + var runs atomic.Int32 + r := newBackgroundCacheRefresher(context.Background(), func(context.Context, string) error { + runs.Add(1) + return nil + }, nil) + r.RequestAfter(time.Hour, "startup") + + require.NoError(shutdownBackgroundCacheRefresher(r)) + time.Sleep(time.Hour) + synctest.Wait() + assert.Zero(runs.Load(), "early startup cleanup cancels the delayed rebuild") + }) +} + +func TestBuildCacheSubprocessWaitHonorsCancellation(t *testing.T) { + require := require.New(t) + buildCacheMu.Lock() + locked := true + unlock := func() { + if locked { + buildCacheMu.Unlock() + locked = false + } + } + t.Cleanup(unlock) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + done := make(chan error, 1) + go func() { + done <- buildCacheSubprocessMode(ctx, buildCacheModeAuto) + }() + + select { + case err := <-done: + require.ErrorIs(err, context.Canceled) + case <-time.After(250 * time.Millisecond): + unlock() + <-done + require.FailNow("canceled cache build must stop waiting for the in-process build lock") + } + unlock() +} + +func TestRefresherRequestAfterKeepsOnlyEarliestTimer(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + var runs atomic.Int32 + r := newBackgroundCacheRefresher(context.Background(), func(context.Context, string) error { + runs.Add(1) + return nil + }, nil) + + r.RequestAfter(2*time.Hour, "later") + r.RequestAfter(time.Hour, "sooner") + r.RequestAfter(3*time.Hour, "latest") + time.Sleep(time.Hour + time.Second) + synctest.Wait() + assert.Equal(int32(1), runs.Load(), "the earliest delayed request fires") + + time.Sleep(3 * time.Hour) + synctest.Wait() + assert.Equal(int32(1), runs.Load(), "later delayed requests were folded into the earliest") + require.NoError(r.Shutdown(context.Background())) + }) +} diff --git a/cmd/msgvault/cmd/cache_related_repair.go b/cmd/msgvault/cmd/cache_related_repair.go new file mode 100644 index 000000000..3677a6c5a --- /dev/null +++ b/cmd/msgvault/cmd/cache_related_repair.go @@ -0,0 +1,184 @@ +package cmd + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "go.kenn.io/msgvault/internal/query" + "go.kenn.io/msgvault/internal/store" +) + +type relatedChangeKinds struct { + recipients bool + labels bool + attachments bool + other bool +} + +func inspectRelatedChangeKinds(snapshot *cacheSourceSnapshot, after, through, lastMessageID int64) (relatedChangeKinds, error) { + var kinds relatedChangeKinds + err := snapshot.QueryRow(`SELECT + count(*) FILTER (WHERE dataset = 'message_recipients') > 0, + count(*) FILTER (WHERE dataset IN ('message_labels', 'labels')) > 0, + count(*) FILTER (WHERE dataset = 'attachments') > 0, + count(*) FILTER (WHERE dataset NOT IN + ('message_recipients', 'message_labels', 'labels', 'attachments')) > 0 + FROM cache_related_change_journal WHERE seq > ? AND seq <= ? AND message_id <= ?`, + after, through, lastMessageID).Scan(&kinds.recipients, &kinds.labels, &kinds.attachments, &kinds.other) + if err != nil { + return relatedChangeKinds{}, fmt.Errorf("inspect related change kinds: %w", err) + } + return kinds, nil +} + +func refreshRelatedCacheStats(ctx context.Context, db sqlRunner, state *query.CacheSyncState, kinds relatedChangeKinds) error { + if kinds.recipients { + statement := `SELECT count(DISTINCT p.email_address), count(DISTINCT p.domain) + FROM sqlite_db.message_recipients mr + JOIN sqlite_db.messages m ON m.id = mr.message_id + JOIN sqlite_db.participants p ON p.id = mr.participant_id + WHERE mr.recipient_type = 'from' AND m.id <= ? AND ` + exportableMessageWhere("m") + if err := db.QueryRowContext(ctx, statement, state.LastMessageID). + Scan(&state.Stats.UniqueSenders, &state.Stats.UniqueDomains); err != nil { + return fmt.Errorf("refresh related sender statistics: %w", err) + } + } + if kinds.attachments { + statement := `SELECT coalesce(sum(try_cast(a.size AS BIGINT)), 0) + FROM sqlite_db.attachments a JOIN sqlite_db.messages m ON m.id = a.message_id + WHERE m.id <= ? AND ` + exportableMessageWhere("m") + if err := db.QueryRowContext(ctx, statement, state.LastMessageID). + Scan(&state.Stats.AttachmentSizeBytes); err != nil { + return fmt.Errorf("refresh related attachment statistics: %w", err) + } + } + return nil +} + +// The marker is published before pruning. A failed publication keeps all +// journal entries available for replay; a failed prune only leaves redundant +// entries, since sqlite_sequence preserves the acknowledged high watermark. +func pruneAcknowledgedRelatedChanges(dbPath string, relatedSeq, derivedRevision int64) error { + st, err := store.Open(dbPath) + if err != nil { + return fmt.Errorf("open store to prune related changes: %w", err) + } + defer func() { _ = st.Close() }() + tx, err := st.DB().Begin() + if err != nil { + return fmt.Errorf("begin related-change prune: %w", err) + } + defer func() { _ = tx.Rollback() }() + if _, err := tx.Exec(`DELETE FROM cache_related_change_journal WHERE seq <= ?`, relatedSeq); err != nil { + return fmt.Errorf("prune related changes: %w", err) + } + if _, err := tx.Exec(`DELETE FROM cache_related_revision_journal WHERE revision <= ?`, derivedRevision); err != nil { + return fmt.Errorf("prune related revisions: %w", err) + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit related-change prune: %w", err) + } + return nil +} + +func warnRelatedChangePrune(dbPath string, relatedSeq, derivedRevision int64) { + if err := pruneAcknowledgedRelatedChanges(dbPath, relatedSeq, derivedRevision); err != nil { + fmt.Fprintf(os.Stderr, "Warning: %v\n", err) + } +} + +func inspectRelatedSnapshotColumns(snapshot *cacheSourceSnapshot) error { + for _, column := range []struct { + table string + name string + present *bool + }{ + {"message_recipients", "email_address", &snapshot.hasRecipientEnvelope}, + {"attachments", "mime_type", &snapshot.hasAttachmentMIME}, + {"attachments", "attachment_metadata", &snapshot.hasAttachmentMetadata}, + } { + var count int + statement := fmt.Sprintf("SELECT COUNT(*) FROM pragma_table_info('%s') WHERE name = '%s'", + column.table, column.name) + if err := snapshot.QueryRow(statement).Scan(&count); err != nil { + return fmt.Errorf("inspect %s.%s for related repair: %w", column.table, column.name, err) + } + *column.present = count > 0 + } + return nil +} + +// datasets lists replacements needed for edits to already-cached messages. +func (k relatedChangeKinds) datasets() map[string]bool { + return map[string]bool{ + "message_recipients": k.recipients, + tableLabels: k.labels, + "message_labels": k.labels, + tableAttachments: k.attachments, + } +} + +// exportRelatedDatasets exports the requested child datasets from one snapshot. +// Each lower message-ID boundary is zero for a replacement, or the committed +// watermark for an append. Label definitions are always replaced. +func exportRelatedDatasets( + ctx context.Context, db sqlRunner, snapshot *cacheSourceSnapshot, + lastMessageID int64, stagingRoot string, afterMessageIDs map[string]int64, +) error { + parentFilter := func(dataset string) string { + return fmt.Sprintf(`SELECT CAST(m.id AS BIGINT) FROM sqlite_db.messages m + WHERE %s AND TRY_CAST(m.id AS BIGINT) <= %d AND TRY_CAST(m.id AS BIGINT) > %d`, + exportableMessageWhere("m"), lastMessageID, afterMessageIDs[dataset]) + } + // Missing envelopes fall back to the participant address. A damaged but + // present envelope stays unknown and must still suppress that fallback. + envelope := "NULL::VARCHAR" + presence := "FALSE" + participant := "CASE WHEN " + cacheIdentityTextSQL("p.email_address") + + " = '' THEN NULL ELSE " + snapshot.identityExportSQL("p.email_address") + " END" + if snapshot.hasRecipientEnvelope { + envelope = "CASE WHEN " + cacheIdentityTextSQL("mr.email_address") + + " = '' THEN NULL ELSE " + snapshot.identityExportSQL("mr.email_address") + " END" + presence = snapshot.identityPresenceSQL("mr.email_address", "mr.envelope_present") + } + mimeType := "'' AS mime_type" + if snapshot.hasAttachmentMIME { + mimeType = "COALESCE(" + snapshot.textSQL("mime_type") + ", '') AS mime_type" + } + metadata := "NULL::VARCHAR AS attachment_metadata" + if snapshot.hasAttachmentMetadata { + metadata = snapshot.textSQL("attachment_metadata") + " AS attachment_metadata" + } + exports := []struct { + dataset string + selectSQL string + }{ + {tableLabels, `SELECT id, COALESCE(` + snapshot.textSQL("name") + `, '') AS name + FROM sqlite_db.labels`}, + {"message_recipients", fmt.Sprintf(`SELECT mr.message_id, mr.participant_id, + %[4]s AS recipient_type, + COALESCE(%[5]s, '') AS display_name, + CASE WHEN %[6]s THEN %[1]s ELSE %[7]s END AS email_address, + %[1]s AS envelope_address + FROM sqlite_db.message_recipients mr + LEFT JOIN sqlite_db.participants p ON p.id = mr.participant_id + WHERE mr.message_id > %[3]d AND TRY_CAST(mr.message_id AS BIGINT) IN (%[2]s)`, envelope, parentFilter("message_recipients"), afterMessageIDs["message_recipients"], snapshot.identityExportSQL("mr.recipient_type"), snapshot.textSQL("mr.display_name"), presence, participant)}, + {"message_labels", fmt.Sprintf(`SELECT message_id, label_id + FROM sqlite_db.message_labels WHERE message_id > %d AND TRY_CAST(message_id AS BIGINT) IN (%s)`, afterMessageIDs["message_labels"], parentFilter("message_labels"))}, + {tableAttachments, fmt.Sprintf(`SELECT id AS attachment_id, message_id, size, + COALESCE(%s, '') AS filename, + %s, %s FROM sqlite_db.attachments + WHERE message_id > %d AND TRY_CAST(message_id AS BIGINT) IN (%s)`, snapshot.textSQL("filename"), mimeType, metadata, afterMessageIDs[tableAttachments], parentFilter(tableAttachments))}, + } + for _, item := range exports { + if _, ok := afterMessageIDs[item.dataset]; !ok { + continue + } + if err := copyParquet(ctx, db, filepath.Join(stagingRoot, item.dataset), "data.parquet", item.selectSQL); err != nil { + return fmt.Errorf("export related dataset %s: %w", item.dataset, query.HintRepairEncoding(err)) + } + } + return nil +} diff --git a/cmd/msgvault/cmd/cache_snapshot_population_test.go b/cmd/msgvault/cmd/cache_snapshot_population_test.go new file mode 100644 index 000000000..92b4d00ff --- /dev/null +++ b/cmd/msgvault/cmd/cache_snapshot_population_test.go @@ -0,0 +1,121 @@ +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCacheCatchUpRepairsNewlyExportableOldMessage(t *testing.T) { + for _, withAppend := range []bool{false, true} { + t.Run(map[bool]string{false: "within_boundary", true: "with_append"}[withAppend], func(t *testing.T) { + assert, require := assert.New(t), require.New(t) + c, st := openTestDaemonAnalyticsStore(t) + _, err := st.DB().Exec(` + INSERT INTO sources (id, source_type, identifier) VALUES (1, 'gmail', 'owner@example.com'); + INSERT INTO conversations (id, source_id, source_conversation_id, conversation_type) + VALUES (1, 1, 'thread', 'email_thread'); + INSERT INTO messages (id, source_id, source_message_id, conversation_id, message_type, sent_at) + VALUES (1, 1, 'pending', 1, 'email', NULL), + (2, 1, 'message-2', 1, 'email', '2024-01-02 00:00:00'); + INSERT INTO labels (id, name) VALUES (1, 'synthetic'); + `) + require.NoError(err) + _, err = buildCache(c.DatabaseDSN(), c.AnalyticsDir(), true) + require.NoError(err) + // Finishing this parent changes the message population below the + // cache's ID boundary. Its journaled labels alone cannot repair it. + _, err = st.DB().Exec(` + UPDATE messages SET sent_at = '2024-01-01 00:00:00' WHERE id = 1; + INSERT INTO message_labels (message_id, label_id) VALUES (1, 1); + INSERT INTO sync_runs (source_id, started_at, completed_at, status, messages_added) + VALUES (1, datetime('now'), datetime('now'), 'completed', 1); + `) + require.NoError(err) + wantCount := int64(2) + if withAppend { + _, err = st.DB().Exec(`INSERT INTO messages + (id, source_id, source_message_id, conversation_id, message_type, sent_at) + VALUES (3, 1, 'message-3', 1, 'email', '2024-01-03 00:00:00')`) + require.NoError(err) + wantCount++ + } else { + // Serving defers the population check to the builder. Background + // inspection still detects the missing old message facts. + light, err := cacheNeedsBuildForServing(t.Context(), c.DatabaseDSN(), c.AnalyticsDir()) + require.NoError(err) + assert.True(light.NeedsBuild) + assert.True(light.HasRelatedRowDrift) + assert.False(light.FullRebuild, "builder verifies the population before repair") + assert.True(cacheNeedsBuild(c.DatabaseDSN(), c.AnalyticsDir()).FullRebuild) + } + built, err := buildCache(c.DatabaseDSN(), c.AnalyticsDir(), false) + require.NoError(err) + assert.Equal(wantCount, built.StagedCount, "the old missing parent requires a full facts repair") + assert.False(cacheNeedsBuild(c.DatabaseDSN(), c.AnalyticsDir()).NeedsBuild) + engine, err := openDaemonDuckDBEngine(c, st) + require.NoError(err) + t.Cleanup(func() { require.NoError(engine.Close()) }) + result, err := engine.QuerySQL(t.Context(), "SELECT COUNT(*) FROM messages") + require.NoError(err) + assert.EqualValues(wantCount, result.Rows[0][0]) + }) + } +} + +// The staleness check ignores messages deleted at the source when it looks for +// new messages. A child-row refresh must not record the new addition count while +// such an exportable message sits above the cached boundary. +func TestCacheCatchUpExportsSourceDeletedMessageAboveBoundary(t *testing.T) { + tests := []struct { + name string + csv string + }{ + {name: "scanner"}, + {name: "csv", csv: "1"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert, require := assert.New(t), require.New(t) + t.Setenv("MSGVAULT_FORCE_CSV_SNAPSHOT", tt.csv) + c, st := openTestDaemonAnalyticsStore(t) + _, err := st.DB().Exec(` + INSERT INTO sources (id, source_type, identifier) VALUES (1, 'gmail', 'owner@example.com'); + INSERT INTO conversations (id, source_id, source_conversation_id, conversation_type) + VALUES (1, 1, 'thread', 'email_thread'); + INSERT INTO messages (id, source_id, source_message_id, conversation_id, message_type, sent_at) + VALUES (1, 1, 'message-1', 1, 'email', '2024-01-01 00:00:00'); + INSERT INTO labels (id, name) VALUES (1, 'synthetic'); + `) + require.NoError(err) + _, err = buildCache(c.DatabaseDSN(), c.AnalyticsDir(), true) + require.NoError(err) + // The deletion time predates the cache build, so the deletion check + // does not force a full rebuild on its own. + _, err = st.DB().Exec(` + INSERT INTO messages (id, source_id, source_message_id, conversation_id, message_type, + sent_at, deleted_from_source_at) + VALUES (2, 1, 'message-2', 1, 'email', '2024-01-02 00:00:00', '2024-01-03 00:00:00'); + INSERT INTO message_labels (message_id, label_id) VALUES (1, 1); + INSERT INTO sync_runs (source_id, started_at, completed_at, status, messages_added) + VALUES (1, datetime('now'), datetime('now'), 'completed', 1); + `) + require.NoError(err) + stale := cacheNeedsBuild(c.DatabaseDSN(), c.AnalyticsDir()) + require.False(stale.HasNew, "a source-deleted message is not live") + require.True(stale.HasRelatedRowDrift) + + _, err = buildCacheAuto(c.DatabaseDSN(), c.AnalyticsDir()) + require.NoError(err) + assert.False(cacheNeedsBuild(c.DatabaseDSN(), c.AnalyticsDir()).NeedsBuild) + engine, err := openDaemonDuckDBEngine(c, st) + require.NoError(err) + t.Cleanup(func() { require.NoError(engine.Close()) }) + result, err := engine.QuerySQL(t.Context(), "SELECT id FROM messages ORDER BY id") + require.NoError(err) + require.Len(result.Rows, 2) + assert.EqualValues(2, result.Rows[1][0]) + }) + } +} diff --git a/cmd/msgvault/cmd/cache_staleness.go b/cmd/msgvault/cmd/cache_staleness.go index ad0069f3d..20dd201f1 100644 --- a/cmd/msgvault/cmd/cache_staleness.go +++ b/cmd/msgvault/cmd/cache_staleness.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "database/sql" "fmt" "strings" @@ -19,9 +20,12 @@ type cacheStaleness struct { // must rebuild immediately. PublishedAt is valid only when this is true. HasUsablePublication bool PublishedAt time.Time - HasNew bool // new messages since last build - HasDeleted bool // deletions since last build - HasUpdated bool // updates or additions within the cached ID boundary require repair + Generation string + PendingAdditions int64 // positive cache addition counter delta, when known + HasNew bool // new messages since last build + HasDeleted bool // deletions since last build + HasUpdated bool // updates or additions within the cached ID boundary require repair + HasRelatedRowDrift bool // journaled child rows changed within the committed message boundary // HasIdentityDrift signals participant_links or account_identities // changed since the last build. Also set whenever // HasAccountIdentityDrift is set (AddAccountIdentity/RemoveAccountIdentity @@ -87,6 +91,20 @@ func deletedSinceBuildCountSQL() string { AND ` + sentCacheExportMessageWhere("") } +// Bound freshness work by the published message IDs, not by the number of +// unpublished rows a sync has added. SQLite otherwise prefers the seq key. +func coveredRelatedChangesSQL() string { + return `SELECT COUNT(*) > 0, COALESCE(MAX(dataset = 'message_facts'), 0) + FROM cache_related_change_journal INDEXED BY idx_cache_related_change_message + WHERE seq > ? AND message_id <= ?` +} + +// Pending rows can become exportable below the committed ID boundary. A child +// journal alone cannot repair those missing message facts. +func coveredCacheMessageCountSQL() string { + return "SELECT COUNT(*) FROM messages WHERE id <= ? AND " + exportableMessageWhere("") +} + // hiddenSinceBuildCountSQL counts exportable messages dedup-hidden since the // last cache build. Same cold-start constraint as deletedSinceBuildCountSQL: // it must be served by idx_messages_deleted_at. @@ -108,32 +126,90 @@ func hiddenSinceBuildCountSQL() string { // SQLite-shaped queries against pgx (which would fail on the ? // placeholders and the sqlite_master probe). func cacheNeedsBuild(dbPath, analyticsDir string) cacheStaleness { + return cacheNeedsBuildContext(context.Background(), dbPath, analyticsDir) +} + +func cacheNeedsBuildContext(ctx context.Context, dbPath, analyticsDir string) cacheStaleness { + if ctx.Err() != nil { + return cacheStaleness{} + } if store.IsPostgresURL(dbPath) { return cacheStaleness{} } - buildLock, err := acquireCacheBuildLock(analyticsDir) + buildLock, err := acquireCacheBuildLock(ctx, analyticsDir) if err != nil { - return cacheStaleness{ - NeedsBuild: true, FullRebuild: true, - Reason: "cannot acquire cache recovery lock", - } + return cacheStalenessFailure(ctx, "cannot acquire cache recovery lock") } defer func() { _ = buildLock.Unlock() }() - return cacheNeedsBuildLocked(dbPath, analyticsDir) + return cacheNeedsBuildLocked(ctx, dbPath, analyticsDir) +} + +func cacheStalenessFailure(ctx context.Context, reason string) cacheStaleness { + if ctx.Err() != nil { + return cacheStaleness{} + } + return cacheStaleness{ + NeedsBuild: true, + FullRebuild: true, + Reason: reason, + } +} + +// cacheNeedsBuildForQuery inspects the committed publication without waiting +// for a builder that is staging the next generation. The shared lock excludes +// only the brief publication step and destructive cache maintenance. +func cacheNeedsBuildForQuery(ctx context.Context, dbPath, analyticsDir string) (cacheStaleness, error) { + return inspectCacheForQuery(ctx, dbPath, analyticsDir, true, false) +} + +// cacheNeedsBuildForServing omits the two archive-wide conversation hashes. +// The scheduled background check runs the full inspection when the minimum +// rebuild interval expires; requests still see indexed sync and revision +// signals immediately, without scanning millions of membership rows. +func cacheNeedsBuildForServing(ctx context.Context, dbPath, analyticsDir string) (cacheStaleness, error) { + return inspectCacheForQuery(ctx, dbPath, analyticsDir, false, true) +} + +func inspectCacheForQuery(ctx context.Context, dbPath, analyticsDir string, full, markerOnly bool) (cacheStaleness, error) { + if store.IsPostgresURL(dbPath) { + return cacheStaleness{}, nil + } + release, err := query.AcquireCacheReadLock(ctx, analyticsDir) + if err != nil { + return cacheStaleness{}, err + } + defer release() + result := cacheNeedsBuildLockedWithOptions(ctx, dbPath, analyticsDir, full, markerOnly) + return result, ctx.Err() } // cacheNeedsBuildLocked performs readiness inspection while the caller holds -// the exclusive cache builder lock (publications also run under it, so the -// committed marker cannot change mid-inspection). Incomplete marker-last +// either the builder lock or the publication read lock, so the committed +// marker cannot change mid-inspection. Incomplete marker-last // publication is detected as drift and rebuilt; publication does not // maintain a recovery journal. -func cacheNeedsBuildLocked(dbPath, analyticsDir string) cacheStaleness { - readiness, err := query.InspectCacheReadiness(analyticsDir) +func cacheNeedsBuildLocked(ctx context.Context, dbPath, analyticsDir string) cacheStaleness { + return cacheNeedsBuildLockedWithConversationHashes(ctx, dbPath, analyticsDir, true) +} + +func cacheNeedsBuildLockedWithConversationHashes(ctx context.Context, dbPath, analyticsDir string, full bool) cacheStaleness { + return cacheNeedsBuildLockedWithOptions(ctx, dbPath, analyticsDir, full, false) +} + +func cacheNeedsBuildLockedWithOptions(ctx context.Context, dbPath, analyticsDir string, full, markerOnly bool) cacheStaleness { + if ctx.Err() != nil { + return cacheStaleness{} + } + inspect := query.InspectCacheReadiness + if markerOnly { + inspect = query.InspectCacheMarkerReadiness + } + readiness, err := inspect(analyticsDir) + if ctx.Err() != nil { + return cacheStaleness{} + } if err != nil { - return cacheStaleness{ - NeedsBuild: true, FullRebuild: true, - Reason: "cannot inspect cache status", - } + return cacheStalenessFailure(ctx, "cannot inspect cache status") } switch readiness { case query.CacheAbsent: @@ -159,12 +235,15 @@ func cacheNeedsBuildLocked(dbPath, analyticsDir string) cacheStaleness { case query.CacheReady: } + if ctx.Err() != nil { + return cacheStaleness{} + } state, err := query.ReadCacheSyncState(analyticsDir) + if ctx.Err() != nil { + return cacheStaleness{} + } if err != nil { - return cacheStaleness{ - NeedsBuild: true, FullRebuild: true, - Reason: "cannot read cache state", - } + return cacheStalenessFailure(ctx, "cannot read cache state") } // A cache written under a different Parquet schema layout is stale even @@ -179,24 +258,26 @@ func cacheNeedsBuildLocked(dbPath, analyticsDir string) cacheStaleness { } } - db, err := store.Open(dbPath) + if ctx.Err() != nil { + return cacheStaleness{} + } + // Inspection must not checkpoint the WAL on close: an active export can + // hold a read snapshot, making that checkpoint wait for the busy timeout. + db, err := store.OpenReadOnlyContext(ctx, dbPath) if err != nil { - return cacheStaleness{ - NeedsBuild: true, FullRebuild: true, - Reason: "cannot verify cache status", - } + return cacheStalenessFailure(ctx, "cannot verify cache status") } defer func() { _ = db.Close() }() + if ctx.Err() != nil { + return cacheStaleness{} + } var maxLiveID int64 - err = db.DB().QueryRow(` + err = db.DB().QueryRowContext(ctx, ` SELECT COALESCE(MAX(id), 0) FROM messages - WHERE ` + cacheLiveMessageWhere("")).Scan(&maxLiveID) + WHERE `+cacheLiveMessageWhere("")).Scan(&maxLiveID) if err != nil { - return cacheStaleness{ - NeedsBuild: true, FullRebuild: true, - Reason: "cannot verify cache status", - } + return cacheStalenessFailure(ctx, "cannot verify cache status") } // Collect staleness signals without short-circuiting so a mixed @@ -205,23 +286,31 @@ func cacheNeedsBuildLocked(dbPath, analyticsDir string) cacheStaleness { result := cacheStaleness{ HasUsablePublication: true, PublishedAt: state.PublishedAt, + Generation: state.DatasetFingerprint, + } + + if state.FullRebuildRequired { + result.HasUpdated = true + result.FullRebuild = true + reasons = append(reasons, "previous build published a partial snapshot") } if maxLiveID > state.LastMessageID { newCount := maxLiveID - state.LastMessageID result.HasNew = true + result.PendingAdditions = newCount reasons = append(reasons, fmt.Sprintf("%d new messages", newCount)) } syncAtStr := state.LastSyncAt.UTC().Format("2006-01-02 15:04:05") var deletedSinceBuild int64 - err = db.DB().QueryRow(deletedSinceBuildCountSQL(), syncAtStr).Scan(&deletedSinceBuild) + if ctx.Err() != nil { + return cacheStaleness{} + } + err = db.DB().QueryRowContext(ctx, deletedSinceBuildCountSQL(), syncAtStr).Scan(&deletedSinceBuild) if err != nil { - return cacheStaleness{ - NeedsBuild: true, FullRebuild: true, - Reason: "cannot verify deletion state", - } + return cacheStalenessFailure(ctx, "cannot verify deletion state") } if deletedSinceBuild > 0 { result.HasDeleted = true @@ -239,12 +328,12 @@ func cacheNeedsBuildLocked(dbPath, analyticsDir string) cacheStaleness { // both source-deleted and dedup-hidden after LastSyncAt is reported // once (as a deletion), not double-counted in the reason string. var hiddenSinceBuild int64 - err = db.DB().QueryRow(hiddenSinceBuildCountSQL(), syncAtStr).Scan(&hiddenSinceBuild) + if ctx.Err() != nil { + return cacheStaleness{} + } + err = db.DB().QueryRowContext(ctx, hiddenSinceBuildCountSQL(), syncAtStr).Scan(&hiddenSinceBuild) if err != nil { - return cacheStaleness{ - NeedsBuild: true, FullRebuild: true, - Reason: "cannot verify dedup state", - } + return cacheStalenessFailure(ctx, "cannot verify dedup state") } if hiddenSinceBuild > 0 { result.HasDeleted = true @@ -254,24 +343,25 @@ func cacheNeedsBuildLocked(dbPath, analyticsDir string) cacheStaleness { } var hasSyncRunsTable int - err = db.DB().QueryRow(` + var coveredAdditionReason string + if ctx.Err() != nil { + return cacheStaleness{} + } + err = db.DB().QueryRowContext(ctx, ` SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'sync_runs' `).Scan(&hasSyncRunsTable) if err != nil { - return cacheStaleness{ - NeedsBuild: true, FullRebuild: true, - Reason: "cannot verify sync history", - } + return cacheStalenessFailure(ctx, "cannot verify sync history") } if hasSyncRunsTable > 0 { - counters, counterErr := readCacheSyncCounters(db.DB()) + if ctx.Err() != nil { + return cacheStaleness{} + } + counters, counterErr := readCacheSyncCountersContext(ctx, db.DB()) err = counterErr if err != nil { - return cacheStaleness{ - NeedsBuild: true, FullRebuild: true, - Reason: "cannot verify sync history", - } + return cacheStalenessFailure(ctx, "cannot verify sync history") } if counters.updates != state.LastCacheUpdateCount { result.HasUpdated = true @@ -295,32 +385,108 @@ func cacheNeedsBuildLocked(dbPath, analyticsDir string) cacheStaleness { counters.failedRunCount, counters.failedRunIDSum)) } if counters.additions != state.LastCacheAdditionCount { + if delta := counters.additions - state.LastCacheAdditionCount; delta > 0 { + result.PendingAdditions = delta + } // A larger message ID gives the incremental exporter an exact lower // boundary for ordinary append-only syncs. If the ID boundary did not // move (or history moved backwards), the changed addition counter may // describe related rows for a parent already present in Parquet, so a - // full rebuild is the only safe repair. - if counters.additions < state.LastCacheAdditionCount || maxLiveID <= state.LastMessageID { + // full rebuild is needed unless the child-row journal covers the + // pending repair. Classify that case after inspecting the journal. + if counters.additions < state.LastCacheAdditionCount { result.HasUpdated = true result.FullRebuild = true reasons = append(reasons, fmt.Sprintf( "cache addition watermark changed from %d to %d within message boundary %d", state.LastCacheAdditionCount, counters.additions, state.LastMessageID)) + } else if maxLiveID <= state.LastMessageID { + coveredAdditionReason = fmt.Sprintf( + "cache addition watermark changed from %d to %d within message boundary %d", + state.LastCacheAdditionCount, counters.additions, state.LastMessageID) } } } - - derivedDataRevision, err := db.DerivedDataRevision() + if ctx.Err() != nil { + return cacheStaleness{} + } + var hasRelatedChangeJournal int + err = db.DB().QueryRowContext(ctx, ` + SELECT COUNT(*) FROM sqlite_master + WHERE type = 'table' AND name = 'cache_related_change_journal' + `).Scan(&hasRelatedChangeJournal) if err != nil { - return cacheStaleness{ - NeedsBuild: true, FullRebuild: true, - Reason: "cannot verify derived-data revision", + return cacheStalenessFailure(ctx, "cannot inspect related-change journal") + } + if hasRelatedChangeJournal == 0 && state.LastRelatedChangeSeq != 0 { + return cacheStalenessFailure(ctx, "related-change journal is missing") + } + if hasRelatedChangeJournal > 0 { + var latestSeq int64 + err = db.DB().QueryRowContext(ctx, ` + SELECT COALESCE((SELECT seq FROM sqlite_sequence + WHERE name = 'cache_related_change_journal'), 0) + `).Scan(&latestSeq) + if err != nil { + return cacheStalenessFailure(ctx, "cannot inspect related-change sequence") + } + if latestSeq < state.LastRelatedChangeSeq { + result.FullRebuild = true + reasons = append(reasons, "related-change journal moved backwards") + } else if latestSeq > state.LastRelatedChangeSeq { + var coveredChanges, messageFactsChanged bool + err = db.DB().QueryRowContext(ctx, coveredRelatedChangesSQL(), + state.LastRelatedChangeSeq, state.LastMessageID).Scan(&coveredChanges, &messageFactsChanged) + if err != nil { + return cacheStalenessFailure(ctx, "cannot inspect related-row changes") + } + if coveredChanges { + result.HasRelatedRowDrift = true + reasons = append(reasons, "related rows changed") + } + if messageFactsChanged { + result.HasDerivedDataDrift = true + result.FullRebuild = true + reasons = append(reasons, "cached message facts changed") + } } } + + if ctx.Err() != nil { + return cacheStaleness{} + } + if coveredAdditionReason != "" { + reasons = append(reasons, coveredAdditionReason) + var coveredCount int64 + // Serving stays on indexed staleness signals. The builder verifies + // population from its read snapshot before repairing child rows. + if result.HasRelatedRowDrift && !markerOnly { + if err := db.DB().QueryRowContext(ctx, coveredCacheMessageCountSQL(), state.LastMessageID). + Scan(&coveredCount); err != nil { + return cacheStalenessFailure(ctx, "cannot verify cached message population") + } + } + if !result.HasRelatedRowDrift || (!markerOnly && coveredCount != state.Stats.TotalMessages) { + result.HasUpdated = true + result.FullRebuild = true + } + } + derivedDataRevision, err := db.DerivedDataRevisionContext(ctx) + if err != nil { + return cacheStalenessFailure(ctx, "cannot verify derived-data revision") + } if derivedDataRevision != state.DerivedDataRevision { - result.HasDerivedDataDrift = true - result.FullRebuild = true - reasons = append(reasons, "derived message data changed") + relatedOnly, relatedErr := db.RelatedDerivedRevisionsOnly(ctx, + state.DerivedDataRevision, derivedDataRevision) + if relatedErr != nil { + return cacheStalenessFailure(ctx, "cannot classify derived-data revision") + } + // Appends already export related rows above the published message ID. + if !relatedOnly { + result.HasDerivedDataDrift = true + result.FullRebuild = true + reasons = append(reasons, "derived message data changed") + } } // Account-identity drift covers identity mutations that invalidate baked @@ -335,12 +501,12 @@ func cacheNeedsBuildLocked(dbPath, analyticsDir string) cacheStaleness { // independently of it, so derivedDriftOnly (build_cache.go) never // mistakes this for the cheap-refresh case even though the same // mutation also bumps identity_revision below. - accountIdentityRevision, err := db.AccountIdentityRevision() + if ctx.Err() != nil { + return cacheStaleness{} + } + accountIdentityRevision, err := db.AccountIdentityRevisionContext(ctx) if err != nil { - return cacheStaleness{ - NeedsBuild: true, FullRebuild: true, - Reason: "cannot verify account identity revision", - } + return cacheStalenessFailure(ctx, "cannot verify account identity revision") } if accountIdentityRevision != state.AccountIdentityRevision { result.HasAccountIdentityDrift = true @@ -354,90 +520,92 @@ func cacheNeedsBuildLocked(dbPath, analyticsDir string) cacheStaleness { // rebuild: the index-only refresh (refreshDerivedDatasetsOnly) handles // it, and a full rebuild triggered by any other signal (including // HasAccountIdentityDrift above) refreshes it naturally. - identityRevision, err := db.IdentityRevision() + if ctx.Err() != nil { + return cacheStaleness{} + } + identityRevision, err := db.IdentityRevisionContext(ctx) if err != nil { - return cacheStaleness{ - NeedsBuild: true, FullRebuild: true, - Reason: "cannot verify identity revision", - } + return cacheStalenessFailure(ctx, "cannot verify identity revision") } if identityRevision != state.IdentityRevision { result.HasIdentityDrift = true reasons = append(reasons, "identity revision changed") } - participantIdentifierRevision, err := db.ParticipantIdentifierRevision() + if ctx.Err() != nil { + return cacheStaleness{} + } + participantIdentifierRevision, err := db.ParticipantIdentifierRevisionContext(ctx) if err != nil { - return cacheStaleness{ - NeedsBuild: true, FullRebuild: true, - Reason: "cannot verify participant identifier revision", - } + return cacheStalenessFailure(ctx, "cannot verify participant identifier revision") } if participantIdentifierRevision != state.ParticipantIdentifierRevision { result.HasParticipantIdentifierDrift = true reasons = append(reasons, "participant identifiers changed") } - participantDisplayNameRevision, err := db.ParticipantDisplayNameRevision() + if ctx.Err() != nil { + return cacheStaleness{} + } + participantDisplayNameRevision, err := db.ParticipantDisplayNameRevisionContext(ctx) if err != nil { - return cacheStaleness{ - NeedsBuild: true, FullRebuild: true, - Reason: "cannot verify participant display-name revision", - } + return cacheStalenessFailure(ctx, "cannot verify participant display-name revision") } if participantDisplayNameRevision != state.ParticipantDisplayNameRevision { result.HasParticipantDisplayNameDrift = true reasons = append(reasons, "participant display names changed") } - personDisplayNameRevision, err := db.PersonDisplayNameRevision() + if ctx.Err() != nil { + return cacheStaleness{} + } + personDisplayNameRevision, err := db.PersonDisplayNameRevisionContext(ctx) if err != nil { - return cacheStaleness{ - NeedsBuild: true, FullRebuild: true, - Reason: "cannot verify person display-name revision", - } + return cacheStalenessFailure(ctx, "cannot verify person display-name revision") } if personDisplayNameRevision != state.PersonDisplayNameRevision { result.HasPersonDisplayNameDrift = true reasons = append(reasons, "person display names changed") } - conversationFingerprint, err := sourceConversationParticipantsFingerprint( - db.DB(), - state.LastMessageID, - ) - if err != nil { - return cacheStaleness{ - NeedsBuild: true, FullRebuild: true, - Reason: "cannot verify conversation participants", + if full { + if ctx.Err() != nil { + return cacheStaleness{} + } + conversationFingerprint, err := sourceConversationParticipantsFingerprint( + ctx, + db.DB(), + state.LastMessageID, + ) + if err != nil { + return cacheStalenessFailure(ctx, "cannot verify conversation participants") + } + if conversationFingerprint != state.ConversationParticipantsFingerprint { + result.HasConversationParticipantDrift = true + reasons = append(reasons, "conversation participants changed") } - } - if conversationFingerprint != state.ConversationParticipantsFingerprint { - result.HasConversationParticipantDrift = true - reasons = append(reasons, "conversation participants changed") - } - typesFingerprint, err := sourceConversationTypesFingerprint( - db.DB(), - state.LastMessageID, - ) - if err != nil { - return cacheStaleness{ - NeedsBuild: true, FullRebuild: true, - Reason: "cannot verify conversation metadata", + if ctx.Err() != nil { + return cacheStaleness{} + } + typesFingerprint, err := sourceConversationTypesFingerprint( + ctx, + db.DB(), + state.LastMessageID, + ) + if err != nil { + return cacheStalenessFailure(ctx, "cannot verify conversation metadata") + } + if typesFingerprint != state.ConversationTypesFingerprint { + result.HasConversationTypeDrift = true + reasons = append(reasons, "conversation metadata changed") } - } - if typesFingerprint != state.ConversationTypesFingerprint { - result.HasConversationTypeDrift = true - reasons = append(reasons, "conversation metadata changed") } - // An incremental build can append only new activity rows. If canonical - // links, conversation membership, or conversation types also changed, - // existing rows need to be rewritten under the new relationship - // dimensions, so rebuild the base generation and relationship index - // together. + // Membership and conversation types still require a full build on append. + // Canonical links can instead rebuild the relationship index from retained + // message facts plus the appended shards, without re-exporting old messages. if result.HasNew && - (result.HasIdentityDrift || result.HasConversationParticipantDrift || + (result.HasConversationParticipantDrift || result.HasConversationTypeDrift) { result.FullRebuild = true } @@ -452,14 +620,15 @@ func cacheNeedsBuildLocked(dbPath, analyticsDir string) cacheStaleness { // sourceConversationTypesFingerprint hashes (id, conversation_type, title) for // conversations with exportable messages inside the committed watermark. The -// NULL normalization must match the conversations Parquet export and -// fingerprintConversationTypesFromSnapshot so an unchanged database always -// reproduces the stamped fingerprint. +// NULL normalization matches fingerprintConversationTypesFromSnapshot. +// FingerprintConversationMetadata repairs invalid UTF-8 from either source +// before hashing, so unchanged data reproduces the stamped fingerprint. func sourceConversationTypesFingerprint( + ctx context.Context, db *sql.DB, lastMessageID int64, ) (string, error) { - rows, err := db.Query(` + rows, err := db.QueryContext(ctx, ` SELECT c.id, COALESCE(c.conversation_type, 'email_thread'), COALESCE(c.title, '') FROM conversations c @@ -484,10 +653,11 @@ func sourceConversationTypesFingerprint( } func sourceConversationParticipantsFingerprint( + ctx context.Context, db *sql.DB, lastMessageID int64, ) (string, error) { - rows, err := db.Query(` + rows, err := db.QueryContext(ctx, ` SELECT cp.conversation_id, cp.participant_id FROM conversation_participants cp WHERE EXISTS ( diff --git a/cmd/msgvault/cmd/cache_staleness_test.go b/cmd/msgvault/cmd/cache_staleness_test.go index f5cbc0b43..4a4e3439a 100644 --- a/cmd/msgvault/cmd/cache_staleness_test.go +++ b/cmd/msgvault/cmd/cache_staleness_test.go @@ -1,7 +1,11 @@ package cmd import ( + "context" "database/sql" + "errors" + "log/slog" + "os" "path/filepath" "strings" "testing" @@ -9,10 +13,78 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/query" "go.kenn.io/msgvault/internal/store" "go.kenn.io/msgvault/internal/testutil" ) +func TestCacheNeedsBuildContextCanceledDoesNotRequestBuild(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + got := cacheNeedsBuildContext(ctx, filepath.Join(t.TempDir(), "msgvault.db"), t.TempDir()) + + assert.False(t, got.NeedsBuild, "cancellation is not evidence that the cache is stale: %+v", got) +} + +func TestCacheStalenessQueriesRespectCancellation(t *testing.T) { + st, err := store.OpenForTest(filepath.Join(t.TempDir(), "msgvault.db")) + require.NoError(t, err) + t.Cleanup(func() { _ = st.Close() }) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err = readCacheSyncCountersContext(ctx, st.DB()) + require.ErrorIs(t, err, context.Canceled) + _, err = sourceConversationParticipantsFingerprint(ctx, st.DB(), 0) + require.ErrorIs(t, err, context.Canceled) +} + +// Cancel after the cache lock is held and a real database revision query has +// started. A cancellation before entry cannot catch a dropped scan context. +func TestCacheStalenessCanceledDuringInspection(t *testing.T) { + tmp := setupTestSQLite(t) + dbPath := filepath.Join(tmp, "test.db") + analyticsDir := filepath.Join(tmp, "analytics") + _, err := buildCache(dbPath, analyticsDir, true) + require.NoError(t, err) + + checks := []struct { + name string + inspect func(context.Context, string, string) (cacheStaleness, error) + }{ + {"background", func(ctx context.Context, dbPath, analyticsDir string) (cacheStaleness, error) { + return cacheNeedsBuildContext(ctx, dbPath, analyticsDir), ctx.Err() + }}, + {"query", cacheNeedsBuildForQuery}, + {"serving", cacheNeedsBuildForServing}, + } + for _, check := range checks { + t.Run(check.name, func(t *testing.T) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + previous := slog.Default() + t.Cleanup(func() { slog.SetDefault(previous) }) + slog.SetDefault(slog.New(slog.NewTextHandler(&strings.Builder{}, &slog.HandlerOptions{ + Level: slog.LevelDebug, + ReplaceAttr: func(_ []string, attr slog.Attr) slog.Attr { + if attr.Key == "stmt" && strings.Contains(attr.Value.String(), "SELECT value FROM archive_metadata") { + cancel() + } + return attr + }, + }))) + + got, err := check.inspect(ctx, dbPath, analyticsDir) + + require.ErrorIs(ctx.Err(), context.Canceled, "inspection must reach the database before cancellation") + require.ErrorIs(err, context.Canceled) + assert.Equal(t, cacheStaleness{}, got, "a canceled inspection must not publish a partial staleness result") + }) + } +} + func TestCacheNeedsBuildInterruptedStateOnlyCache(t *testing.T) { tmpDir := setupTestSQLiteEmpty(t) dbPath := filepath.Join(tmpDir, "test.db") @@ -25,6 +97,251 @@ func TestCacheNeedsBuildInterruptedStateOnlyCache(t *testing.T) { assert.Contains(t, got.Reason, "interrupted") } +func TestCacheNeedsBuildTracksCoveredRelatedRows(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + tmp := t.TempDir() + dbPath := filepath.Join(tmp, "msgvault.db") + analyticsDir := filepath.Join(tmp, "analytics") + st, err := store.Open(dbPath) + require.NoError(err) + require.NoError(st.InitSchema()) + src, err := st.GetOrCreateSource("test", "synthetic@example.com") + require.NoError(err) + conversationID, err := st.EnsureConversationWithType(src.ID, "thread", "email_thread", "Synthetic") + require.NoError(err) + messageID, err := st.UpsertMessage(&store.Message{ + ConversationID: conversationID, SourceID: src.ID, SourceMessageID: "message", + MessageType: "email", SentAt: sql.NullTime{ + Time: time.Date(2025, 1, 1, 10, 0, 0, 0, time.UTC), Valid: true, + }, + }) + require.NoError(err) + _, err = st.DB().Exec(`INSERT INTO labels (id, name) VALUES (1, 'synthetic'), (2, 'changed')`) + require.NoError(err) + _, err = st.DB().Exec(`INSERT INTO participants (id, email_address, domain) + VALUES (1, 'one@example.com', 'example.com'), (2, 'two@example.com', 'example.com')`) + require.NoError(err) + _, err = st.DB().Exec(`INSERT INTO message_recipients + (message_id, participant_id, recipient_type) VALUES (?, 1, 'to')`, messageID) + require.NoError(err) + _, err = st.DB().Exec(`INSERT INTO attachments + (id, message_id, storage_path, filename, size) VALUES (1, ?, 'synthetic', 'old.txt', 1)`, messageID) + require.NoError(err) + _, err = st.DB().Exec(`UPDATE messages SET has_attachments = TRUE, attachment_count = 1 WHERE id = ?`, messageID) + require.NoError(err) + _, err = st.DB().Exec(`INSERT INTO message_labels (message_id, label_id) VALUES (?, 1)`, messageID) + require.NoError(err) + require.NoError(st.Close()) + _, err = buildCache(dbPath, analyticsDir, true) + require.NoError(err) + state, err := query.ReadCacheSyncState(analyticsDir) + require.NoError(err) + assert.Positive(state.LastRelatedChangeSeq) + assert.False(cacheNeedsBuild(dbPath, analyticsDir).NeedsBuild) + + st, err = store.Open(dbPath) + require.NoError(err) + var acknowledgedRows int64 + require.NoError(st.DB().QueryRow(`SELECT COUNT(*) FROM cache_related_change_journal`).Scan(&acknowledgedRows)) + assert.Zero(acknowledgedRows, "published journal entries should be pruned") + require.NoError(st.AddMessageLabels(messageID, []int64{2})) + require.NoError(st.ReplaceMessageRecipients(messageID, "to", []int64{2}, []string{"Recipient Two"})) + _, err = st.DB().Exec(`UPDATE attachments SET filename = 'new.txt', size = 2 WHERE id = 1`) + require.NoError(err) + require.NoError(st.Close()) + got := cacheNeedsBuild(dbPath, analyticsDir) + assert.True(got.NeedsBuild) + assert.True(got.HasRelatedRowDrift) + assert.Contains(got.Reason, "related rows changed") + st, err = store.Open(dbPath) + require.NoError(err) + var seq int64 + require.NoError(st.DB().QueryRow(`SELECT MAX(seq) FROM cache_related_change_journal`).Scan(&seq)) + require.NoError(st.Close()) + assert.Greater(seq, state.LastRelatedChangeSeq) + previousHook := buildCacheBeforeMessagesExportHook + buildCacheBeforeMessagesExportHook = func() error { + return errors.New("related-row repair attempted a message export") + } + t.Cleanup(func() { buildCacheBeforeMessagesExportHook = previousHook }) + result, err := buildCacheAuto(dbPath, analyticsDir) + require.NoError(err) + assert.True(result.IdentityOnly) + assert.Zero(result.ExportedCount) + assert.False(cacheNeedsBuild(dbPath, analyticsDir).NeedsBuild) + repairedState, err := query.ReadCacheSyncState(analyticsDir) + require.NoError(err) + assert.Equal(seq, repairedState.LastRelatedChangeSeq) + duckDB, err := sql.Open("duckdb", "") + require.NoError(err) + defer func() { _ = duckDB.Close() }() + labelsPath := strings.ReplaceAll(filepath.Join(analyticsDir, "message_labels", "*.parquet"), "'", "''") + var labelCount int64 + require.NoError(duckDB.QueryRow("SELECT COUNT(*) FROM read_parquet('" + labelsPath + "')").Scan(&labelCount)) + assert.Equal(int64(2), labelCount) + recipientsPath := strings.ReplaceAll(filepath.Join(analyticsDir, "message_recipients", "*.parquet"), "'", "''") + var recipientID int64 + require.NoError(duckDB.QueryRow("SELECT participant_id FROM read_parquet('" + recipientsPath + "')").Scan(&recipientID)) + assert.Equal(int64(2), recipientID) + attachmentsPath := strings.ReplaceAll(filepath.Join(analyticsDir, "attachments", "*.parquet"), "'", "''") + var filename string + var attachmentSize int64 + require.NoError(duckDB.QueryRow("SELECT filename, size FROM read_parquet('"+attachmentsPath+"')").Scan(&filename, &attachmentSize)) + assert.Equal("new.txt", filename) + assert.Equal(int64(2), attachmentSize) + assert.Equal(int64(2), repairedState.Stats.AttachmentSizeBytes) + + buildCacheBeforeMessagesExportHook = nil + unchangedFiles := make(map[string]os.FileInfo) + for _, pattern := range []string{ + "message_recipients/*.parquet", "attachments/*.parquet", "relationship_activity/occurred_year=*/*.parquet", + } { + paths, err := filepath.Glob(filepath.Join(analyticsDir, filepath.FromSlash(pattern))) + require.NoError(err) + require.NotEmpty(paths) + for _, path := range paths { + unchangedFiles[path], err = os.Stat(path) + require.NoError(err) + } + } + st, err = store.Open(dbPath) + require.NoError(err) + newMessageID, err := st.UpsertMessage(&store.Message{ + ConversationID: conversationID, SourceID: src.ID, SourceMessageID: "new-message", + MessageType: "email", SentAt: sql.NullTime{ + Time: time.Date(2025, 1, 2, 10, 0, 0, 0, time.UTC), Valid: true, + }, + }) + require.NoError(err) + require.NoError(st.ReplaceMessageRecipients(newMessageID, "from", []int64{1}, []string{"New Sender"})) + _, err = st.DB().Exec(`INSERT INTO labels (id, name) VALUES (3, 'second-change')`) + require.NoError(err) + require.NoError(st.AddMessageLabels(messageID, []int64{3})) + require.NoError(st.Close()) + result, err = buildCacheAuto(dbPath, analyticsDir) + require.NoError(err) + assert.Equal(int64(1), result.StagedCount) + assert.False(cacheNeedsBuild(dbPath, analyticsDir).NeedsBuild) + require.NoError(duckDB.QueryRow("SELECT COUNT(*) FROM read_parquet('" + labelsPath + "')").Scan(&labelCount)) + assert.Equal(int64(3), labelCount) + for path, before := range unchangedFiles { + after, err := os.Stat(path) + require.NoError(err) + assert.True(os.SameFile(before, after), "label repair with an append must retain %s", path) + } + + activityFiles, err := filepath.Glob(filepath.Join(analyticsDir, + "relationship_activity", "occurred_year=*", "*.parquet")) + require.NoError(err) + require.NotEmpty(activityFiles) + activityBefore, err := os.Stat(activityFiles[0]) + require.NoError(err) + st, err = store.Open(dbPath) + require.NoError(err) + _, err = st.DB().Exec(`INSERT INTO labels (id, name) VALUES (4, 'label-only')`) + require.NoError(err) + require.NoError(st.AddMessageLabels(messageID, []int64{4})) + require.NoError(st.Close()) + _, err = buildCacheAuto(dbPath, analyticsDir) + require.NoError(err) + activityAfter, err := os.Stat(activityFiles[0]) + require.NoError(err) + assert.Equal(activityBefore.ModTime(), activityAfter.ModTime(), + "label-only repair must reuse relationship activity") + assert.Equal(activityBefore.Size(), activityAfter.Size()) + var labelDefinitionCount int64 + definitionsPath := strings.ReplaceAll(filepath.Join(analyticsDir, "labels", "*.parquet"), "'", "''") + require.NoError(duckDB.QueryRow("SELECT COUNT(*) FROM read_parquet('" + definitionsPath + "')").Scan(&labelDefinitionCount)) + assert.Equal(int64(4), labelDefinitionCount) + + // A display-name change has no message_labels mutation, but must still + // republish the label definitions used by analytical views. + st, err = store.Open(dbPath) + require.NoError(err) + _, err = st.DB().Exec(`UPDATE labels SET name = 'renamed' WHERE id = 4`) + require.NoError(err) + require.NoError(st.Close()) + got = cacheNeedsBuild(dbPath, analyticsDir) + assert.True(got.HasRelatedRowDrift) + assert.False(got.FullRebuild) + _, err = buildCacheAuto(dbPath, analyticsDir) + require.NoError(err) + var labelName string + require.NoError(duckDB.QueryRow("SELECT name FROM read_parquet('" + definitionsPath + "') WHERE id = 4").Scan(&labelName)) + assert.Equal("renamed", labelName) + for path, before := range unchangedFiles { + after, err := os.Stat(path) + require.NoError(err) + assert.True(os.SameFile(before, after), "label-only repair must retain %s", path) + } + + // Attachment metadata repairs must replace their totals, not add the old + // size again, while keeping old relationship activity in place. + st, err = store.Open(dbPath) + require.NoError(err) + _, err = st.DB().Exec(`UPDATE attachments SET size = 3 WHERE id = 1`) + require.NoError(err) + _, err = st.UpsertMessage(&store.Message{ + ConversationID: conversationID, SourceID: src.ID, SourceMessageID: "attachment-append", + MessageType: "email", SentAt: sql.NullTime{Time: time.Date(2025, 1, 3, 10, 0, 0, 0, time.UTC), Valid: true}, + }) + require.NoError(err) + require.NoError(st.Close()) + result, err = buildCacheAuto(dbPath, analyticsDir) + require.NoError(err) + assert.Equal(int64(1), result.StagedCount) + repairedState, err = query.ReadCacheSyncState(analyticsDir) + require.NoError(err) + assert.Equal(int64(3), repairedState.Stats.AttachmentSizeBytes) + activityAfter, err = os.Stat(activityFiles[0]) + require.NoError(err) + assert.True(os.SameFile(activityBefore, activityAfter)) + + // An analytics directory may be a link to a separate storage location. + // Repair old recipients while appending messages through that link. + realAnalytics := filepath.Join(t.TempDir(), "analytics") + require.NoError(os.Rename(analyticsDir, realAnalytics)) + if err := os.Symlink(realAnalytics, analyticsDir); err != nil { + t.Skipf("symlink unavailable: %v", err) + } + st, err = store.Open(dbPath) + require.NoError(err) + require.NoError(st.ReplaceMessageRecipients(messageID, "to", []int64{1}, []string{"Recipient One"})) + _, err = st.UpsertMessage(&store.Message{ + ConversationID: conversationID, SourceID: src.ID, SourceMessageID: "third-message", + MessageType: "email", SentAt: sql.NullTime{Time: time.Date(2025, 1, 3, 10, 0, 0, 0, time.UTC), Valid: true}, + }) + require.NoError(err) + require.NoError(st.Close()) + result, err = buildCacheAuto(dbPath, analyticsDir) + require.NoError(err) + assert.Equal(int64(1), result.StagedCount) + require.NoError(duckDB.QueryRow("SELECT participant_id FROM read_parquet('"+recipientsPath+"') WHERE message_id = ?", messageID).Scan(&recipientID)) + assert.Equal(int64(1), recipientID) + repairedState, err = query.ReadCacheSyncState(analyticsDir) + require.NoError(err) + assert.Equal(int64(4), repairedState.Stats.TotalMessages) + assert.Equal(int64(3), repairedState.Stats.AttachmentSizeBytes) + var canonicalID int64 + require.NoError(duckDB.QueryRow(`SELECT canonical_id FROM read_parquet(?, hive_partitioning=true) + WHERE message_id = ? AND canonical_id IS NOT NULL`, + filepath.Join(analyticsDir, "relationship_activity", "**", "*.parquet"), messageID).Scan(&canonicalID)) + assert.Equal(int64(1), canonicalID) + assert.False(cacheNeedsBuild(dbPath, analyticsDir).NeedsBuild) + + // Attachment or From-recipient edits may also alter facts baked into old + // message shards; those cannot use the child-only repair path. + st, err = store.Open(dbPath) + require.NoError(err) + _, err = st.DB().Exec(`UPDATE messages SET attachment_count = attachment_count + 1 WHERE id = ?`, messageID) + require.NoError(err) + require.NoError(st.Close()) + got = cacheNeedsBuild(dbPath, analyticsDir) + assert.True(got.FullRebuild) + assert.True(got.HasDerivedDataDrift) +} + func TestCacheNeedsBuild_MeetingMutation(t *testing.T) { tests := []struct { name string @@ -89,11 +406,12 @@ func TestCacheNeedsBuild_MeetingMutation(t *testing.T) { } } -func TestCacheNeedsBuildMixedNewMessagesAndRelationshipDriftForcesFullRebuild(t *testing.T) { +func TestCacheNeedsBuildMixedNewMessagesAndRelationshipDrift(t *testing.T) { tests := []struct { name string mutate func(*testing.T, *store.Store) assert func(*testing.T, cacheStaleness) + full bool }{ { name: "participant links", @@ -109,6 +427,7 @@ func TestCacheNeedsBuildMixedNewMessagesAndRelationshipDriftForcesFullRebuild(t }, { name: "conversation membership", + full: true, mutate: func(t *testing.T, st *store.Store) { t.Helper() _, err := st.DB().Exec(` @@ -152,8 +471,8 @@ func TestCacheNeedsBuildMixedNewMessagesAndRelationshipDriftForcesFullRebuild(t got := cacheNeedsBuild(dbPath, analyticsDir) assertions.True(got.HasNew) tt.assert(t, got) - assertions.True(got.FullRebuild, - "mixed new-message and relationship drift must not append stale index rows") + assertions.Equal(tt.full, got.FullRebuild, + "only membership drift requires re-exporting message facts on append") }) } } @@ -228,3 +547,94 @@ func TestCacheStalenessCounts_UseDeletionIndexes(t *testing.T) { assert.Equal(int64(1), deleted) assert.Equal(int64(1), hidden) } + +func TestCoveredRelatedChangesUseMessageBoundary(t *testing.T) { + s := testutil.NewSQLiteTestStore(t) + _, err := s.DB().Exec(`DELETE FROM cache_related_change_journal; + INSERT INTO cache_related_change_journal (seq, dataset, message_id) + VALUES (1, 'message_facts', 1), (2, 'labels', 0), + (3, 'message_labels', 100), (4, 'message_facts', 50), (5, 'message_facts', 101)`) + require.NoError(t, err) + plan := explainQueryPlan(t, s, coveredRelatedChangesSQL(), 1, 100) + assert.Contains(t, plan, "SEARCH cache_related_change_journal USING INDEX idx_cache_related_change_message (message_id= 0 { + qualifier, name = column[:i+1], column[i+1:] + } + return "contains(" + qualifier + "__invalid_utf8, '|" + name + "|')" +} + +// cacheIdentityPresenceSQL reports whether an identity column recorded a +// non-empty value at the byte level, mirroring SQLite's guard that the raw +// column is non-NULL and non-blank after TRIM. Presence must not depend on +// UTF-8 validity: a damaged envelope or primary email must still suppress +// attribution through another address. cacheIdentityTextSQL turns invalid +// bytes into NULL, so testing that key alone would misread them as absent. TRIM +// is unsafe on invalid bytes in DuckDB, so the NULL (invalid) key falls +// back to the encoded byte length (octet_length is the BLOB-safe length; +// length does not bind BLOBs in DuckDB); invalid strings are never +// all-spaces, so non-empty bytes are equivalent to a non-empty TRIM result, +// and TRIM never sees invalid bytes. +func cacheIdentityPresenceSQL(column string) string { + v := "TRY_CAST(" + column + " AS VARCHAR)" + return "CASE WHEN TRY(decode(encode(" + v + "))) IS NULL " + + "THEN octet_length(encode(COALESCE(" + v + ", ''))) > 0 " + + "ELSE TRIM(TRY(decode(encode(" + v + ")))) <> '' END" +} + +// reportCacheTextRepairs explains the replacement count and archive repair limits. +func reportCacheTextRepairs(w io.Writer, repairs *cacheTextRepairs) { + if repairs == nil { + return + } + n := repairs.Count() + if n == 0 && repairs.identities.Load() == 0 { + return + } + if n > 0 { + _, _ = fmt.Fprintf(w, "Warning: %d invalid UTF-8 repair(s) applied while building the analytics cache; affected cache text uses U+FFFD.\n", n) + } + if n := repairs.identities.Load(); n > 0 { + _, _ = fmt.Fprintf(w, "Warning: %d identity value(s) exported as unknown because of invalid UTF-8.\n", n) + } + _, _ = fmt.Fprintln(w, "Run 'msgvault repair-encoding' to repair supported archive fields and rebuild the cache. RFC 822 Message-IDs, source message IDs, source identifiers, recorded envelope addresses, account identity addresses, and participant identifiers require separate recovery if damaged.") + _, _ = fmt.Fprintln(w, "For damaged List-IDs, run 'msgvault repair-list-ids --apply' to re-derive them from stored raw MIME; recovery requires an intact original List-Id header.") +} diff --git a/cmd/msgvault/cmd/cache_text_test.go b/cmd/msgvault/cmd/cache_text_test.go new file mode 100644 index 000000000..8e1551d02 --- /dev/null +++ b/cmd/msgvault/cmd/cache_text_test.go @@ -0,0 +1,213 @@ +package cmd + +import ( + "context" + "database/sql" + "fmt" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCacheTextRepairsSanitize(t *testing.T) { + tests := []struct { + name, in, want string + counted int64 + }{ + {"valid text is unchanged", "Müller 🎉", "Müller 🎉", 0}, + {"lone continuation byte", "\x80", "�", 1}, + {"truncated emoji", "Calendar: lunch \xf0\x9f", "Calendar: lunch ��", 1}, + {"following valid bytes survive", "é\xe9té", "é�té", 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repairs := &cacheTextRepairs{} + assert.Equal(t, tt.want, repairs.sanitize(tt.in)) + assert.Equal(t, tt.counted, repairs.Count()) + }) + } + var nilRepairs *cacheTextRepairs + assert.Equal(t, int64(0), nilRepairs.Count()) +} + +func requireSQLiteScanner(t *testing.T, duckDB *sql.DB) { + t.Helper() + if _, err := duckDB.Exec("INSTALL sqlite; LOAD sqlite;"); err != nil { + t.Skipf("DuckDB sqlite extension unavailable: %v", err) + } +} + +func TestCacheIdentityTextSQLValidOrNull(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + sqlitePath := filepath.Join(t.TempDir(), "identity.db") + sqliteDB, err := sql.Open("sqlite3", sqlitePath) + require.NoError(err) + _, err = sqliteDB.Exec(`CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)`) + require.NoError(err) + for id, raw := range map[int][]byte{ + 1: []byte("ok"), + 2: []byte("a\x80b"), + 3: []byte("\x80"), + 4: []byte("primary\uFFFD@example.com"), + } { + _, err = sqliteDB.Exec(`INSERT INTO t VALUES (?, CAST(? AS TEXT))`, id, raw) + require.NoError(err) + } + _, err = sqliteDB.Exec(`INSERT INTO t VALUES (5, NULL)`) + require.NoError(err) + require.NoError(sqliteDB.Close()) + + duckDB, err := sql.Open("duckdb", "") + require.NoError(err) + defer func() { _ = duckDB.Close() }() + duckDB.SetMaxOpenConns(1) + requireSQLiteScanner(t, duckDB) + _, err = duckDB.Exec(fmt.Sprintf("ATTACH '%s' AS src (TYPE sqlite, READ_ONLY)", + strings.ReplaceAll(sqlitePath, "'", "''"))) + require.NoError(err) + + // cacheIdentityTextSQL is pure SQL: it needs no registered fallback + // function, only the sqlite scanner passing stored bytes through. + rows, err := duckDB.Query(`SELECT id, ` + cacheIdentityTextSQL("v") + ` FROM src.t ORDER BY id`) + require.NoError(err) + defer func() { _ = rows.Close() }() + got := map[int]sql.NullString{} + for rows.Next() { + var id int + var v sql.NullString + require.NoError(rows.Scan(&id, &v)) + got[id] = v + } + require.NoError(rows.Err()) + + assert.Equal(sql.NullString{String: "ok", Valid: true}, got[1], "valid value passes through unchanged") + assert.Equal(sql.NullString{}, got[2], "invalid value becomes NULL") + assert.Equal(sql.NullString{}, got[3], "invalid value becomes NULL") + assert.Equal(sql.NullString{String: "primary\uFFFD@example.com", Valid: true}, got[4], + "a literal U+FFFD rune is valid UTF-8 and passes through byte-identical") + assert.Equal(sql.NullString{}, got[5], "NULL stays NULL") +} + +func TestCacheIdentityPresenceSQLMatchesSQLiteTrimSemantics(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + cases := map[int][]byte{ + 1: []byte("owner@example.com"), + 2: []byte("owner\x80@example.com"), + 3: []byte("\x80"), + 4: []byte(" "), + 5: []byte(""), + 6: []byte("owner\uFFFD@example.com"), + } + sqlitePath := filepath.Join(t.TempDir(), "envelope.db") + sqliteDB, err := sql.Open("sqlite3", sqlitePath) + require.NoError(err) + _, err = sqliteDB.Exec(`CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)`) + require.NoError(err) + for id, raw := range cases { + _, err = sqliteDB.Exec(`INSERT INTO t VALUES (?, CAST(? AS TEXT))`, id, raw) + require.NoError(err) + } + _, err = sqliteDB.Exec(`INSERT INTO t VALUES (7, NULL)`) + require.NoError(err) + + // The store decides envelope presence with SQLite over the raw bytes; + // the cache expression must agree row for row. + want := map[int]bool{} + storeRows, err := sqliteDB.Query(`SELECT id, v IS NOT NULL AND TRIM(v) <> '' FROM t`) + require.NoError(err) + defer func() { require.NoError(storeRows.Close()) }() + for storeRows.Next() { + var id int + var present bool + require.NoError(storeRows.Scan(&id, &present)) + want[id] = present + } + require.NoError(storeRows.Err()) + require.NoError(sqliteDB.Close()) + + duckDB, err := sql.Open("duckdb", "") + require.NoError(err) + defer func() { _ = duckDB.Close() }() + duckDB.SetMaxOpenConns(1) + requireSQLiteScanner(t, duckDB) + _, err = duckDB.Exec(fmt.Sprintf("ATTACH '%s' AS src (TYPE sqlite, READ_ONLY)", + strings.ReplaceAll(sqlitePath, "'", "''"))) + require.NoError(err) + + rows, err := duckDB.Query(`SELECT id, ` + cacheIdentityPresenceSQL("v") + ` FROM src.t ORDER BY id`) + require.NoError(err) + defer func() { _ = rows.Close() }() + got := map[int]bool{} + for rows.Next() { + var id int + var present bool + require.NoError(rows.Scan(&id, &present)) + got[id] = present + } + require.NoError(rows.Err()) + + for id := range want { + assert.Equal(want[id], got[id], + "row %d: cache envelope presence must match the store's raw-column guard", id) + } + assert.True(got[2], "invalid bytes are still a recorded envelope") + assert.True(got[3], "a single invalid byte is still a recorded envelope") + assert.False(got[4], "an all-spaces envelope counts as absent, like SQLite TRIM") + assert.False(got[5], "an empty envelope counts as absent") + assert.False(got[7], "a NULL envelope counts as absent") +} + +func TestCacheTextSQLRepairsSQLiteScannerValues(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + ctx := context.Background() + + sqlitePath := filepath.Join(t.TempDir(), "text.db") + sqliteDB, err := sql.Open("sqlite3", sqlitePath) + require.NoError(err) + _, err = sqliteDB.Exec(`CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)`) + require.NoError(err) + for id, raw := range map[int][]byte{1: []byte("ok"), 2: []byte("ab\xf0\x9fcd"), 3: []byte("\x80")} { + _, err = sqliteDB.Exec(`INSERT INTO t VALUES (?, CAST(? AS TEXT))`, id, raw) + require.NoError(err) + } + _, err = sqliteDB.Exec(`INSERT INTO t VALUES (4, NULL)`) + require.NoError(err) + require.NoError(sqliteDB.Close()) + + duckDB, err := sql.Open("duckdb", "") + require.NoError(err) + defer func() { _ = duckDB.Close() }() + duckDB.SetMaxOpenConns(1) + repairs := &cacheTextRepairs{} + require.NoError(registerCacheTextFunctions(ctx, duckDB, repairs)) + requireSQLiteScanner(t, duckDB) + _, err = duckDB.Exec(fmt.Sprintf("ATTACH '%s' AS src (TYPE sqlite, READ_ONLY)", + strings.ReplaceAll(sqlitePath, "'", "''"))) + require.NoError(err) + + rows, err := duckDB.Query(`SELECT id, ` + cacheTextSQL("v") + ` FROM src.t ORDER BY id`) + require.NoError(err) + defer func() { _ = rows.Close() }() + got := map[int]sql.NullString{} + for rows.Next() { + var id int + var v sql.NullString + require.NoError(rows.Scan(&id, &v)) + got[id] = v + } + require.NoError(rows.Err()) + + assert.Equal(sql.NullString{String: "ok", Valid: true}, got[1]) + assert.Equal(sql.NullString{String: "ab��cd", Valid: true}, got[2]) + assert.Equal(sql.NullString{String: "�", Valid: true}, got[3]) + assert.Equal(sql.NullString{}, got[4], "NULL must stay NULL") + assert.Equal(int64(2), repairs.Count(), "only invalid values reach the Go fallback") +} diff --git a/cmd/msgvault/cmd/calendar.go b/cmd/msgvault/cmd/calendar.go index 029f72369..4cbd24784 100644 --- a/cmd/msgvault/cmd/calendar.go +++ b/cmd/msgvault/cmd/calendar.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "log/slog" "os" "slices" "strings" @@ -16,6 +17,7 @@ import ( "golang.org/x/oauth2" "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/calcontrol" "go.kenn.io/msgvault/internal/calsync" "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/daemonclient" @@ -30,6 +32,7 @@ const calScopeEscalationConfirmedFlag = "scope-escalation-confirmed" var ( calAddOAuthApp string calAddHeadless bool + calAddWrite bool calAddAll bool calAddMinRole string calAddCalendars []string @@ -46,7 +49,7 @@ var ( func init() { rootCmd.AddCommand(newAddCalendarCmd()) - rootCmd.AddCommand(newSyncCalendarCmd()) + rootCmd.AddCommand(addManualSyncCacheFlags(newSyncCalendarCmd())) } func interactiveStdin() bool { @@ -70,11 +73,17 @@ func newAddCalendarLocalCmd() *cobra.Command { Use: "add-calendar ", Short: "Authorize Google Calendar access and register calendars for an account", Long: "Grants read-only Calendar access (calendar.readonly) to an account and " + - "registers its calendars for sync. If the account already has a Gmail token, " + + "registers its calendars for sync. Use --write to also grant calendar.events. " + + "Source write_calendars and invite_calendars remain explicit opt-ins. If the account already has a Gmail token, " + "re-consent bundles Gmail + Calendar together; keep BOTH checked on the consent " + "screen so Gmail access is not dropped.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg email := normalizeCalendarAccountEmail(args[0]) ctx := cmd.Context() oauthAppExplicit := cmd.Flags().Changed("oauth-app") @@ -85,7 +94,7 @@ func newAddCalendarLocalCmd() *cobra.Command { return usageErr(cmd, err) } - st, cleanup, err := openWritableStoreAndInit() + st, cleanup, err := openWritableStoreAndInitForInvocation(state) if err != nil { return err } @@ -104,7 +113,7 @@ func newAddCalendarLocalCmd() *cobra.Command { // sources for these apps; RegisterCalendars is then the live proof // that the Calendar scope was actually granted. if cfg.OAuth.ServiceAccountKeyFor(oauthApp) != "" { - client, err := buildCalendarClient(ctx, email, oauthApp, false) + client, err := buildCalendarClient(ctx, email, oauthApp, false, calAddWrite) if err != nil { return err } @@ -117,12 +126,12 @@ func newAddCalendarLocalCmd() *cobra.Command { if err != nil { return err } - mgr, err := newCalendarOAuthManager(secretsPath, email) + mgr, err := newCalendarOAuthManager(secretsPath, email, state, calAddWrite) if err != nil { - return wrapOAuthError(fmt.Errorf("create oauth manager: %w", err)) + return wrapOAuthError(fmt.Errorf("create oauth manager: %w", err), cfg) } hasToken := mgr.HasToken(email) - hasCalendarScope := mgr.HasScope(email, oauth.ScopeCalendarReadonly) + hasCalendarScope := mgr.HasScope(email, oauth.ScopeCalendarReadonly) && (!calAddWrite || mgr.HasScope(email, oauth.ScopeCalendarEvents)) tokenReusable := calendarAddTokenReusable(mgr, email, appDecision) // A token that exists, carries the calendar scope, and matches the @@ -142,7 +151,7 @@ func newAddCalendarLocalCmd() *cobra.Command { // token is copied in, re-running add-calendar --headless skips this // and registers the calendars (an API call that needs no browser). if calAddHeadless && (!hasToken || !hasCalendarScope || !tokenReusable || tokenExpiredOrRevoked) { - oauth.PrintCalendarHeadlessInstructions(email, cfg.TokensDir(), oauthApp) + oauth.PrintCalendarHeadlessInstructions(email, cfg.TokensDir(), oauthApp, calAddWrite) return nil } @@ -150,18 +159,18 @@ func newAddCalendarLocalCmd() *cobra.Command { case !hasToken: fmt.Printf("Authorizing %s for Calendar...\n", email) if err := mgr.Authorize(ctx, email); err != nil { - return wrapOAuthError(err) + return wrapOAuthError(err, cfg) } case tokenExpiredOrRevoked: fmt.Printf("Calendar token for %s is expired or revoked. Re-authorizing...\n", email) if err := mgr.AuthorizePreservingGrantedScopes(ctx, email); err != nil { - return wrapOAuthError(err) + return wrapOAuthError(err, cfg) } case !hasCalendarScope: - headline, body, cancelHint := calendarScopeEscalationPrompt() + headline, body, cancelHint := calendarScopeEscalationPrompt(calAddWrite) existingScopes := mgr.GrantedScopes(email) requiredScopes := calendarEscalationScopes(existingScopes, - calendarShouldPreserveGmail(hasToken, mgr.HasScopeMetadata(email), existingScopes)) + calendarShouldPreserveGmail(hasToken, mgr.HasScopeMetadata(email), existingScopes), calAddWrite) confirmed, err := cmd.Flags().GetBool(calScopeEscalationConfirmedFlag) if err != nil { return fmt.Errorf("read --%s flag: %w", calScopeEscalationConfirmedFlag, err) @@ -182,11 +191,11 @@ func newAddCalendarLocalCmd() *cobra.Command { case !tokenReusable: fmt.Printf("OAuth app for %s requires reauthorization. Authorizing...\n", email) if err := mgr.Authorize(ctx, email); err != nil { - return wrapOAuthError(err) + return wrapOAuthError(err, cfg) } } - client, err := buildCalendarClient(ctx, email, oauthApp, interactiveStdin()) + client, err := buildCalendarClient(ctx, email, oauthApp, interactiveStdin(), calAddWrite) if err != nil { return err } @@ -196,6 +205,7 @@ func newAddCalendarLocalCmd() *cobra.Command { }, } cmd.Flags().StringVar(&calAddOAuthApp, "oauth-app", "", "named OAuth app to use") + cmd.Flags().BoolVar(&calAddWrite, "write", false, "also authorize calendar.events; source configuration still controls writable calendar IDs") cmd.Flags().BoolVar(&calAddHeadless, "headless", false, "headless host: print token-copy instructions instead of opening a browser") cmd.Flags().BoolVar(&calAddAll, "all-calendars", false, "include reader/freeBusyReader calendars (default: owner+writer)") cmd.Flags().StringVar(&calAddMinRole, "min-access-role", "", "minimum access role: owner|writer|reader") @@ -212,6 +222,11 @@ func newAddCalendarLocalCmd() *cobra.Command { // rows, and reports what was registered plus the follow-up sync command. It is // shared by add-calendar's user-consent and service-account paths. func registerCalendarsAndReport(ctx context.Context, out io.Writer, st *store.Store, client gcal.API, email, oauthApp string, oauthAppSet bool) error { + state := invocationFromContext(ctx) + var logger *slog.Logger + if state != nil { + logger = state.logger + } syncer := calsync.New(client, st, calsync.Options{ AccountEmail: email, OAuthApp: oauthApp, @@ -219,7 +234,10 @@ func registerCalendarsAndReport(ctx context.Context, out io.Writer, st *store.St Calendars: calAddCalendars, AllCalendars: calAddAll, MinAccessRole: calAddMinRole, - }).WithLogger(logger) + }) + if logger != nil { + syncer = syncer.WithLogger(logger) + } cals, err := syncer.RegisterCalendars(ctx) if err != nil { @@ -261,6 +279,7 @@ func runAddCalendarHTTP(cmd *cobra.Command, args []string) error { OAuthApp: calAddOAuthApp, OAuthAppExplicit: cmd.Flags().Changed("oauth-app"), Headless: calAddHeadless, + Write: calAddWrite, }) if err != nil { return err @@ -321,9 +340,14 @@ func preflightAddCalendarAuthorize( requestedApp string, requestedExplicit bool, ) error { - if IsRemoteMode() || calAddHeadless || plan == nil { + state := invocationFromContext(ctx) + if IsRemoteMode(state) || calAddHeadless || plan == nil { return nil } + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg oauthApp, needsClientCheck, ok := preflightCalendarOAuthApp(plan, requestedApp, requestedExplicit) if !ok { return nil @@ -335,12 +359,12 @@ func preflightAddCalendarAuthorize( if err != nil { return err } - mgr, err := newCalendarOAuthManager(secretsPath, email) + mgr, err := newCalendarOAuthManager(secretsPath, email, state, calAddWrite) if err != nil { - return wrapOAuthError(fmt.Errorf("create oauth manager: %w", err)) + return wrapOAuthError(fmt.Errorf("create oauth manager: %w", err), cfg) } hasToken := mgr.HasToken(email) - hasCalendarScope := mgr.HasScope(email, oauth.ScopeCalendarReadonly) + hasCalendarScope := mgr.HasScope(email, oauth.ScopeCalendarReadonly) && (!calAddWrite || mgr.HasScope(email, oauth.ScopeCalendarEvents)) tokenReusable := hasToken && (!needsClientCheck || mgr.TokenMatchesClient(email)) tokenExpiredOrRevoked := hasToken && hasCalendarScope && tokenReusable && calendarTokenExpiredOrRevoked(ctx, mgr, email) @@ -349,12 +373,12 @@ func preflightAddCalendarAuthorize( case !hasToken: fmt.Printf("Authorizing %s for Calendar...\n", email) if err := mgr.Authorize(ctx, email); err != nil { - return wrapOAuthError(err) + return wrapOAuthError(err, cfg) } case tokenExpiredOrRevoked: fmt.Printf("Calendar token for %s is expired or revoked. Re-authorizing...\n", email) if err := mgr.AuthorizePreservingGrantedScopes(ctx, email); err != nil { - return wrapOAuthError(err) + return wrapOAuthError(err, cfg) } case !hasCalendarScope: // Scope escalation replaces the granted scope set, so it only runs @@ -364,14 +388,14 @@ func preflightAddCalendarAuthorize( } existingScopes := mgr.GrantedScopes(email) requiredScopes := calendarEscalationScopes(existingScopes, - calendarShouldPreserveGmail(hasToken, mgr.HasScopeMetadata(email), existingScopes)) + calendarShouldPreserveGmail(hasToken, mgr.HasScopeMetadata(email), existingScopes), calAddWrite) if err := authorizeScopeEscalation(ctx, email, requiredScopes, secretsPath); err != nil { return err } case !tokenReusable: fmt.Printf("OAuth app for %s requires reauthorization. Authorizing...\n", email) if err := mgr.Authorize(ctx, email); err != nil { - return wrapOAuthError(err) + return wrapOAuthError(err, cfg) } } return nil @@ -408,6 +432,12 @@ func newSyncCalendarLocalCmd() *cobra.Command { "email) or used directly as an email. --after/--before bound a full sync only.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger ctx := cmd.Context() email := normalizeCalendarAccountEmail(args[0]) @@ -444,7 +474,7 @@ func newSyncCalendarLocalCmd() *cobra.Command { return usageErr(cmd, err) } - st, cleanup, err := openWritableStoreAndInit() + st, cleanup, err := openWritableStoreAndInitForInvocation(state) if err != nil { return err } @@ -477,7 +507,10 @@ func newSyncCalendarLocalCmd() *cobra.Command { TimeMax: timeMax, Limit: calSyncLimit, NoResume: calSyncNoResume, - }).WithLogger(logger) + }) + if logger != nil { + syncer = syncer.WithLogger(logger) + } var res calsync.Result if calendarSyncShouldRunFullForSources(existing, calSyncFull, calSyncAll, calSyncMinRole, calendars, hasFullOnlyOptions) { @@ -490,7 +523,7 @@ func newSyncCalendarLocalCmd() *cobra.Command { } fmt.Printf("Calendar sync complete: %d calendar(s), %d event(s) added, %d cancelled\n", res.CalendarsSynced, res.EventsAdded, res.EventsCancelled) - return rebuildCacheAfterWrite(cfg.DatabaseDSN()) + return rebuildCacheAfterManualSync(cfg.DatabaseDSN(), state) }, } cmd.Flags().StringVar(&calSyncOAuthApp, "oauth-app", "", "named OAuth app to use") @@ -505,21 +538,31 @@ func newSyncCalendarLocalCmd() *cobra.Command { return cmd } -func calendarAddOAuthScopes(preserveGmail bool) []string { +func calendarAddOAuthScopes(preserveGmail bool, write ...bool) []string { + scopes := append([]string(nil), oauth.ScopesCalendar...) if preserveGmail { - return append([]string(nil), oauth.ScopesGmailCalendar...) + scopes = append([]string(nil), oauth.ScopesGmailCalendar...) + } + if len(write) > 0 && write[0] { + scopes = appendScopeIfMissing(scopes, oauth.ScopeCalendarEvents) } - return append([]string(nil), oauth.ScopesCalendar...) + return scopes } -func newCalendarOAuthManager(clientSecretsPath, account string) (*oauth.Manager, error) { +func newCalendarOAuthManager(clientSecretsPath, account string, state *invocation, write ...bool) (*oauth.Manager, error) { + state = invocationState(context.Background(), state) + if state == nil || state.cfg == nil || state.logger == nil { + return nil, errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger account = normalizeCalendarAccountEmail(account) probe, err := oauth.NewManagerWithScopes(clientSecretsPath, cfg.TokensDir(), logger, oauth.ScopesCalendar) if err != nil { return nil, err } existingScopes := probe.GrantedScopes(account) - scopes := calendarOAuthScopesForAccount(probe.HasToken(account), probe.HasScopeMetadata(account), existingScopes) + scopes := calendarOAuthScopesForAccount(probe.HasToken(account), probe.HasScopeMetadata(account), existingScopes, write...) if slices.Equal(scopes, oauth.ScopesCalendar) { return probe, nil } @@ -572,9 +615,9 @@ func calendarRegisteredIDs(sources []*store.Source) map[string]struct{} { return ids } -func calendarEscalationScopes(existingScopes []string, preserveGmail bool) []string { +func calendarEscalationScopes(existingScopes []string, preserveGmail bool, write ...bool) []string { scopes := append([]string(nil), existingScopes...) - required := calendarAddOAuthScopes(preserveGmail) + required := calendarAddOAuthScopes(preserveGmail, write...) // Preserving Gmail must not mean re-widening it. calendarAddOAuthScopes // returns the full Gmail bundle, so an account narrowed to read-only via // `add-account --readonly` would silently regain write access just by @@ -588,7 +631,15 @@ func calendarEscalationScopes(existingScopes []string, preserveGmail bool) []str return scopes } -func calendarScopeEscalationPrompt() (string, []string, string) { +func calendarScopeEscalationPrompt(write ...bool) (string, []string, string) { + if len(write) > 0 && write[0] { + return "CALENDAR EVENT WRITE ACCESS REQUIRED", []string{ + "Calendar control needs calendar.events permission to create or change events.", + "Source write_calendars and invite_calendars still restrict allowed calendars.", + "Re-consent preserves Gmail and other granted Google scopes. Keep every existing permission checked.", + }, "Cancelled. Calendar write access was not granted." + } + return "CALENDAR ACCESS REQUIRED", []string{ "Calendar sync needs read-only Calendar access.", "", @@ -601,10 +652,15 @@ func calendarScopeEscalationPrompt() (string, []string, string) { } func planCLIAddCalendar( - _ context.Context, + ctx context.Context, st *store.Store, req api.CLIAddCalendarPlanRequest, ) (api.CLIAddCalendarPlanResponse, error) { + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil { + return api.CLIAddCalendarPlanResponse{}, errors.New("configuration is unavailable") + } + cfg := state.cfg email := normalizeCalendarAccountEmail(req.Email) if email == "" { return api.CLIAddCalendarPlanResponse{}, errors.New("account email is required") @@ -630,9 +686,9 @@ func planCLIAddCalendar( if err != nil { return api.CLIAddCalendarPlanResponse{}, err } - mgr, err := newCalendarOAuthManager(secretsPath, email) + mgr, err := newCalendarOAuthManager(secretsPath, email, state, req.Write) if err != nil { - return api.CLIAddCalendarPlanResponse{}, wrapOAuthError(fmt.Errorf("create oauth manager: %w", err)) + return api.CLIAddCalendarPlanResponse{}, wrapOAuthError(fmt.Errorf("create oauth manager: %w", err), cfg) } // The resolved app binding is returned even when no escalation is // needed: the frontend uses it to run any required browser @@ -643,12 +699,12 @@ func planCLIAddCalendar( NeedsClientCheck: appDecision.NeedsClientCheck, } hasToken := mgr.HasToken(email) - hasCalendarScope := mgr.HasScope(email, oauth.ScopeCalendarReadonly) + hasCalendarScope := mgr.HasScope(email, oauth.ScopeCalendarReadonly) && (!req.Write || mgr.HasScope(email, oauth.ScopeCalendarEvents)) if req.Headless || !hasToken || hasCalendarScope { return plan, nil } - headline, body, cancelHint := calendarScopeEscalationPrompt() + headline, body, cancelHint := calendarScopeEscalationPrompt(req.Write) plan.NeedsScopeEscalation = true plan.Headline = headline plan.BodyLines = body @@ -656,9 +712,9 @@ func planCLIAddCalendar( return plan, nil } -func calendarOAuthScopesForAccount(hasToken bool, hasScopeMetadata bool, existingScopes []string) []string { +func calendarOAuthScopesForAccount(hasToken bool, hasScopeMetadata bool, existingScopes []string, write ...bool) []string { return calendarEscalationScopes(existingScopes, - calendarShouldPreserveGmail(hasToken, hasScopeMetadata, existingScopes)) + calendarShouldPreserveGmail(hasToken, hasScopeMetadata, existingScopes), write...) } func calendarShouldPreserveGmail(hasToken bool, hasScopeMetadata bool, existingScopes []string) bool { @@ -677,7 +733,13 @@ func hasAnyScope(scopes []string, candidates []string) bool { return false } -func calendarEscalationScopesForAccount(account string, clientSecretsPath string) ([]string, error) { +func calendarEscalationScopesForAccount(account string, clientSecretsPath string, state *invocation) ([]string, error) { + state = invocationState(context.Background(), state) + if state == nil || state.cfg == nil || state.logger == nil { + return nil, errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger account = normalizeCalendarAccountEmail(account) mgr, err := oauth.NewManagerWithScopes(clientSecretsPath, cfg.TokensDir(), logger, oauth.ScopesGmailCalendar) if err != nil { @@ -864,12 +926,18 @@ func calendarSyncNextCommand(email, oauthApp string, opts calendarSyncNextOption // the account email (never a calendar source identifier). If reauth is needed, // it preserves Gmail only for existing Gmail/legacy tokens; Calendar-only tokens // stay Calendar-only. The limiter is sized for the Calendar per-user budget. -func buildCalendarClient(ctx context.Context, accountEmail, oauthApp string, interactive bool) (gcal.API, error) { +func buildCalendarClient(ctx context.Context, accountEmail, oauthApp string, interactive bool, write ...bool) (gcal.API, error) { + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil || state.logger == nil { + return nil, errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger accountEmail = normalizeCalendarAccountEmail(accountEmail) var tokenSource oauth2.TokenSource if saKeyPath := cfg.OAuth.ServiceAccountKeyFor(oauthApp); saKeyPath != "" { - saMgr, err := oauth.NewServiceAccountManager(saKeyPath, oauth.ScopesCalendar) + saMgr, err := oauth.NewServiceAccountManager(saKeyPath, calendarAddOAuthScopes(false, write...)) if err != nil { return nil, fmt.Errorf("service account: %w", err) } @@ -882,13 +950,16 @@ func buildCalendarClient(ctx context.Context, accountEmail, oauthApp string, int if err != nil { return nil, err } - mgr, err := newCalendarOAuthManager(secretsPath, accountEmail) + mgr, err := newCalendarOAuthManager(secretsPath, accountEmail, state, write...) if err != nil { - return nil, wrapOAuthError(fmt.Errorf("create oauth manager: %w", err)) + return nil, wrapOAuthError(fmt.Errorf("create oauth manager: %w", err), cfg) } if err := requireCalendarTokenForSync(mgr, accountEmail); err != nil { return nil, err } + if len(write) > 0 && write[0] && !mgr.HasScope(accountEmail, oauth.ScopeCalendarEvents) { + return nil, fmt.Errorf("%w: calendar event writes are not authorized; run 'msgvault add-calendar %s --write'", calcontrol.ErrDenied, accountEmail) + } tokenSource, err = getTokenSourceWithReauth(ctx, mgr, accountEmail, interactive, calendarReauthHint) if err != nil { return nil, err @@ -913,7 +984,8 @@ func requireCalendarTokenForSync(mgr *oauth.Manager, accountEmail string) error func calendarTokenActionError(accountEmail string) error { return fmt.Errorf( - "calendar access for %s is not authorized; run 'msgvault add-calendar %s' to grant %s", + "%w: calendar access for %s is not authorized; run 'msgvault add-calendar %s' to grant %s", + calcontrol.ErrDenied, accountEmail, accountEmail, oauth.ScopeCalendarReadonly, @@ -925,6 +997,11 @@ func calendarTokenActionError(accountEmail string) error { // Store. The first run full-syncs (and registers calendars); later runs are // incremental. Embedding is picked up later by scan-and-fill via embed_gen=NULL. func runConfiguredGCalSync(ctx context.Context, st *store.Store, src config.GCalSource) error { + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil || state.logger == nil { + return errors.New("configuration is unavailable") + } + logger := state.logger email := normalizeCalendarAccountEmail(src.Email) if email == "" { return fmt.Errorf("gcal source %q email is required", src.Name) @@ -950,7 +1027,10 @@ func runConfiguredGCalSync(ctx context.Context, st *store.Store, src config.GCal OAuthApp: appDecision.OAuthApp, OAuthAppSet: appDecision.OAuthAppSet, Calendars: src.Calendars, - }).WithLogger(logger) + }) + if logger != nil { + syncer = syncer.WithLogger(logger) + } if calendarSyncShouldRunFullForSources(existing, false, false, "", src.Calendars, false) { _, err = syncer.Full(ctx) diff --git a/cmd/msgvault/cmd/calendar_control.go b/cmd/msgvault/cmd/calendar_control.go new file mode 100644 index 000000000..c1530b428 --- /dev/null +++ b/cmd/msgvault/cmd/calendar_control.go @@ -0,0 +1,359 @@ +package cmd + +import ( + "context" + "encoding/json/jsontext" + "encoding/json/v2" + "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/agentgrant" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/calcontrol" + "go.kenn.io/msgvault/internal/calsync" + "go.kenn.io/msgvault/internal/gcal" +) + +func init() { rootCmd.AddCommand(newCalendarControlCmd(nil)) } + +type calendarRunner func(context.Context, calcontrol.Request) (*calcontrol.Result, error) + +func newCalendarControlCmd(run calendarRunner) *cobra.Command { + var account, sendUpdates string + var dryRun, readOnly, jsonOutput bool + root := &cobra.Command{Use: "calendar", Short: "Control live calendar events and query availability", Long: "Create, update, delete, move, or respond to live Google Calendar events through the daemon.\nRequires an explicit --account; calendar IDs and configured aliases select the target calendar.\nWrites require add-calendar --write plus write_calendars in the source configuration.\nGuest changes also require invite_calendars. Notifications default to none."} + root.PersistentFlags().StringVar(&account, "account", "", "configured calendar source name or OAuth account (required)") + root.PersistentFlags().StringVar(&sendUpdates, "send-updates", "none", "guest notifications: none, all, or externalOnly") + root.PersistentFlags().BoolVar(&dryRun, "dry-run", false, "verify access and print the proposed changes without writing") + root.PersistentFlags().BoolVar(&readOnly, "read-only", false, "forbid every event mutation") + root.PersistentFlags().BoolVar(&jsonOutput, "json", false, "print the complete result and archive receipts as JSON") + for _, action := range []string{"create", "update", "delete", "move", "respond", "freebusy", "conflicts"} { + var summary, description, location, from, to, tz, scope, original, response, destination string + var allDay bool + var attendees, addAttendees, rules, reminders, calendars []string + use := action + " " + argCount := 1 + if action != "create" && !calcontrol.IsRead(action) { + use += " " + argCount = 2 + } + command := &cobra.Command{Use: use, Short: "Calendar " + action, Args: cobra.ExactArgs(argCount)} + if action == "move" { + command.Use += " [destination-calendar-id]" + command.Args = cobra.RangeArgs(2, 3) + } + command.RunE = func(cmd *cobra.Command, args []string) error { + request := calcontrol.Request{Action: action, Account: account, CalendarID: args[0], DryRun: dryRun, ReadOnly: readOnly} + if len(args) > 1 { + request.EventID = args[1] + } + if !calcontrol.IsRead(action) { + request.SendUpdates = sendUpdates + request.Scope = scope + request.OriginalStart = original + request.Destination = destination + if action == "move" && len(args) == 3 { + if cmd.Flags().Changed("destination") { + return usageErr(cmd, errors.New("use a positional destination or --destination, not both")) + } + request.Destination = args[2] + } + request.Response = response + } else { + if cmd.Flags().Changed("send-updates") { + return usageErr(cmd, errors.New("--send-updates is only valid for event changes")) + } + request.CalendarIDs = calendars + request.TimeZone = tz + } + if cmd.Flags().Changed("summary") { + request.Event.Summary = &summary + } + if cmd.Flags().Changed("description") { + request.Event.Description = &description + } + if cmd.Flags().Changed("location") { + request.Event.Location = &location + } + for _, bound := range []struct { + flag, value string + target **gcal.EventDateTime + }{{"from", from, &request.Event.Start}, {"to", to, &request.Event.End}} { + if !cmd.Flags().Changed(bound.flag) { + continue + } + dt, err := parseCalendarBound(bound.value, tz, allDay) + if err != nil { + return usageErr(cmd, err) + } + if calcontrol.IsRead(action) { + instant, ok := dt.Instant() + if !ok { + return usageErr(cmd, errors.New("invalid availability bound")) + } + if bound.flag == "from" { + request.TimeMin = instant + } else { + request.TimeMax = instant + } + } else { + *bound.target = &dt + } + } + if cmd.Flags().Changed("attendees") { + values := []gcal.Attendee{} + for _, email := range attendees { + if email != "" { + values = append(values, gcal.Attendee{Email: strings.TrimSpace(email)}) + } + } + request.Event.Attendees = &values + } + for _, email := range addAttendees { + request.AddAttendees = append(request.AddAttendees, strings.TrimSpace(email)) + } + if cmd.Flags().Changed("rrule") { + values := []string{} + for _, line := range rules { + if line != "" { + if !strings.HasPrefix(line, "RRULE:") { + line = "RRULE:" + line + } + values = append(values, line) + } + } + request.Event.Recurrence = &values + } + if cmd.Flags().Changed("reminder") { + value, err := parseCalendarReminders(reminders) + if err != nil { + return usageErr(cmd, err) + } + request.Event.Reminders = value + } + if cmd.Flags().Changed("tz") && !calcontrol.IsRead(action) && request.Event.Start == nil && request.Event.End == nil { + return usageErr(cmd, errors.New("--tz requires --from or --to")) + } + if err := request.Validate(); err != nil { + return usageErr(cmd, err) + } + runner := run + if runner == nil { + runner = func(ctx context.Context, r calcontrol.Request) (*calcontrol.Result, error) { + client, _, err := OpenHTTPStore(ctx) + if err != nil { + return nil, err + } + defer func() { _ = client.Close() }() + return client.ControlCalendar(ctx, r) + } + } + result, err := runner(cmd.Context(), request) + if err != nil { + return err + } + if result == nil { + return errors.New("calendar operation returned no result") + } + if jsonOutput || result.DryRun || calcontrol.IsRead(action) { + if err := json.MarshalEncode(jsontext.NewEncoder(cmd.OutOrStdout()), result, json.Deterministic(true)); err != nil { + return err + } + } else { + for _, receipt := range result.Writes { + archiveStatus := fmt.Sprintf("archive message %d", receipt.MessageID) + if !receipt.Archived { + archiveStatus = "not archived: " + receipt.ArchiveError + } + if _, err := fmt.Fprintf(cmd.OutOrStdout(), "%s %s on %s (%s)\n", receipt.Action, receipt.Event.ID, receipt.CalendarID, archiveStatus); err != nil { + return fmt.Errorf("print calendar receipt: %w", err) + } + } + } + if result.Error != "" { + return errors.New(result.Error) + } + for _, receipt := range result.Writes { + if !receipt.Archived { + return fmt.Errorf("google change completed for %s; archive write failed: %s; sync-calendar to reconcile, do not repeat the mutation", receipt.Event.ID, receipt.ArchiveError) + } + } + return nil + } + if action == "create" || action == "update" { + command.Flags().StringVar(&summary, "summary", "", "event title (explicit empty clears on update)") + command.Flags().StringVar(&description, "description", "", "event description") + command.Flags().StringVar(&location, "location", "", "event location") + command.Flags().BoolVar(&allDay, "all-day", false, "date-only event; --to is the exclusive end date") + command.Flags().StringSliceVar(&attendees, "attendees", nil, "replace guest email list (comma-separated; empty clears)") + command.Flags().StringArrayVar(&rules, "rrule", nil, "recurrence RRULE (repeatable; empty clears)") + command.Flags().StringArrayVar(&reminders, "reminder", nil, "popup:minutes, email:minutes, default, or none (repeatable)") + } + if action == "create" || action == "update" || calcontrol.IsRead(action) { + command.Flags().StringVar(&from, "from", "", "start: RFC3339, or local YYYY-MM-DDTHH:MM with --tz") + command.Flags().StringVar(&to, "to", "", "end: RFC3339, or local YYYY-MM-DDTHH:MM with --tz") + command.Flags().StringVar(&tz, "tz", "", "IANA time zone for local times and recurrence") + } + if action == "update" { + command.Flags().StringSliceVar(&addAttendees, "add-attendee", nil, "add guests while preserving existing attendees and RSVP state") + } + if action == "update" || action == "delete" || action == "respond" { + scopeHelp := "recurrence scope: single (default), or all" + if action == "update" || action == "delete" { + scopeHelp = "recurrence scope: single (default), future, or all" + } + command.Flags().StringVar(&scope, "scope", "", scopeHelp) + command.Flags().StringVar(&original, "original-start", "", "original occurrence start (RFC3339 or all-day date)") + } + if action == "move" { + command.Flags().StringVar(&destination, "destination", "", "destination calendar ID or configured alias") + } + if action == "respond" { + command.Flags().StringVar(&response, "status", "", "self RSVP: accepted, declined, or tentative") + } + if calcontrol.IsRead(action) { + command.Flags().StringSliceVar(&calendars, "calendars", nil, "calendar IDs or aliases to include (default: positional calendar)") + } + root.AddCommand(command) + } + return root +} +func parseCalendarBound(value, tz string, allDay bool) (gcal.EventDateTime, error) { + dt := gcal.EventDateTime{TimeZone: tz} + if allDay { + if _, err := time.Parse("2006-01-02", value); err != nil { + return dt, fmt.Errorf("all-day bounds must be YYYY-MM-DD: %w", err) + } + dt.Date = value + return dt, nil + } + if parsed, err := time.Parse(time.RFC3339, value); err == nil { + dt.DateTime = parsed + return dt, nil + } + if tz == "" { + return dt, errors.New("timed bounds require RFC3339 with an offset, or --tz for local times") + } + loc, err := time.LoadLocation(tz) + if err != nil { + return dt, fmt.Errorf("invalid IANA time zone: %w", err) + } + for _, layout := range []string{"2006-01-02T15:04:05", "2006-01-02T15:04"} { + if parsed, err := time.ParseInLocation(layout, value, loc); err == nil { + dt.DateTime = parsed + return dt, nil + } + } + return dt, errors.New("invalid calendar time; expected RFC3339 or YYYY-MM-DDTHH:MM") +} +func parseCalendarReminders(values []string) (*gcal.Reminders, error) { + r := &gcal.Reminders{Overrides: []gcal.Reminder{}} + for _, value := range values { + if value == "default" || value == "none" { + if len(values) != 1 { + return nil, errors.New("default/none reminder cannot be combined with overrides") + } + r.UseDefault = value == "default" + return r, nil + } + method, minutesText, ok := strings.Cut(value, ":") + if !ok { + return nil, errors.New("reminder must be popup:minutes, email:minutes, default, or none") + } + minutes, err := strconv.Atoi(minutesText) + if err != nil { + return nil, fmt.Errorf("invalid reminder minutes: %w", err) + } + r.Overrides = append(r.Overrides, gcal.Reminder{Method: method, Minutes: minutes}) + } + return r, nil +} + +var _ api.CalendarController = (*storeAPIAdapter)(nil) + +func (a *storeAPIAdapter) ControlCalendar(ctx context.Context, request calcontrol.Request, grant *agentgrant.Grant, acquireWrite func(context.Context) (func(), error)) (*calcontrol.Result, error) { + if err := request.Validate(); err != nil { + return nil, err + } + if grant != nil { + for _, permission := range calcontrol.RequestPermissions(request) { + if !grant.HasPermission(permission) { + return nil, calcontrol.ErrDenied + } + } + } + setupError := func(err error) error { + if grant == nil { + return err + } + if a.logger != nil { + a.logger.Warn("calendar setup failed", "error", err) + } + return calcontrol.ErrDenied + } + if a.config == nil { + return nil, setupError(fmt.Errorf("%w: calendar configuration is unavailable", calcontrol.ErrDenied)) + } + source := a.config.GetGCalSource(request.Account) + if source == nil || !source.Enabled { + return nil, setupError(fmt.Errorf("%w: enabled calendar source is required", calcontrol.ErrDenied)) + } + source.Email = normalizeCalendarAccountEmail(source.Email) + if err := calcontrol.AuthorizeSourceRequest(*source, request, grant); err != nil { + return nil, setupError(err) + } + existing, err := a.store.GetSourcesByTypeAndAccount(sourceTypeCalendar, source.Email) + if err != nil { + return nil, setupError(fmt.Errorf("%w: load calendar OAuth binding: %w", calcontrol.ErrInternal, err)) + } + appDecision, err := calendarSyncOAuthAppDecision(a.store, source.Email, existing, source.OAuthApp, source.OAuthApp != "") + if err != nil { + return nil, setupError(fmt.Errorf("%w: resolve calendar OAuth binding: %w", calcontrol.ErrInternal, err)) + } + source.OAuthApp = appDecision.OAuthApp + ctx = a.invocationContext(ctx) + var client gcal.ControlAPI + if a.calendarClientFactory != nil { + client, err = a.calendarClientFactory(ctx, *source, !calcontrol.IsRead(request.Action)) + } else { + var reader gcal.API + reader, err = buildCalendarClient(ctx, source.Email, source.OAuthApp, false, !calcontrol.IsRead(request.Action)) + if err == nil { + var ok bool + client, ok = reader.(gcal.ControlAPI) + if !ok { + _ = reader.Close() + return nil, setupError(fmt.Errorf("%w: calendar client does not support event control", calcontrol.ErrInternal)) + } + } + } + if err != nil { + if errors.Is(err, calcontrol.ErrDenied) { + return nil, setupError(err) + } + return nil, setupError(fmt.Errorf("%w: create calendar client: %w", calcontrol.ErrInternal, err)) + } + defer func() { _ = client.Close() }() + syncer := calsync.New(client, a.store, calsync.Options{AccountEmail: source.Email, OAuthApp: source.OAuthApp, OAuthAppSet: appDecision.OAuthAppSet}) + if a.logger != nil { + syncer.WithLogger(a.logger) + } + service := calcontrol.Service{Source: *source, Client: client, Persist: syncer.PersistEvent, AcquireWrite: acquireWrite} + result, err := service.Execute(ctx, request, grant) + if grant != nil && errors.Is(err, calcontrol.ErrDenied) { + return nil, calcontrol.ErrDenied + } + if err == nil && result != nil && len(result.Writes) > 0 && a.draftCacheRefresh != nil { + refreshCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + refreshErr := a.draftCacheRefresh(refreshCtx, "calendar event changed") + cancel() + if refreshErr != nil && a.logger != nil { + a.logger.Warn("calendar analytics refresh failed", "error", refreshErr) + } + } + return result, err +} diff --git a/cmd/msgvault/cmd/calendar_control_auth_test.go b/cmd/msgvault/cmd/calendar_control_auth_test.go new file mode 100644 index 000000000..42558a1fe --- /dev/null +++ b/cmd/msgvault/cmd/calendar_control_auth_test.go @@ -0,0 +1,21 @@ +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "go.kenn.io/msgvault/internal/oauth" +) + +func TestCalendarWriteConsentPreservesNarrowedGmailAndOtherServices(t *testing.T) { + assertions := assert.New(t) + existing := []string{oauth.ScopeGmailReadonly, "https://www.googleapis.com/auth/drive.readonly", "openid"} + scopes := calendarEscalationScopes(existing, true, true) + assertions.Contains(scopes, oauth.ScopeCalendarEvents) + assertions.Contains(scopes, oauth.ScopeCalendarReadonly) + assertions.Contains(scopes, "https://www.googleapis.com/auth/drive.readonly") + assertions.Contains(scopes, "openid") + assertions.NotContains(scopes, oauth.ScopeGmailModify) + assertions.NotContains(scopes, oauth.ScopeGmailCompose) + assertions.NotContains(calendarEscalationScopes(existing, true), oauth.ScopeCalendarEvents) +} diff --git a/cmd/msgvault/cmd/calendar_control_cli_test.go b/cmd/msgvault/cmd/calendar_control_cli_test.go new file mode 100644 index 000000000..0a9b66fd1 --- /dev/null +++ b/cmd/msgvault/cmd/calendar_control_cli_test.go @@ -0,0 +1,112 @@ +package cmd + +import ( + "context" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/calcontrol" + "go.kenn.io/msgvault/internal/gcal" +) + +func TestCalendarControlCLIParsesSharedTargetAndDryRun(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + var got calcontrol.Request + cmd := newCalendarControlCmd(func(_ context.Context, r calcontrol.Request) (*calcontrol.Result, error) { + got = r + return &calcontrol.Result{DryRun: true}, nil + }) + cmd.SetArgs([]string{"create", "team@example.com", "--account", "person@example.com", "--summary", "Planning", "--from", "2026-10-02T09:00:00", "--to", "2026-10-02T10:00:00", "--tz", "Europe/London", "--attendees", "guest@example.com", "--rrule", "RRULE:FREQ=WEEKLY;COUNT=4", "--reminder", "popup:15", "--dry-run", "--json"}) + requirements.NoError(cmd.Execute()) + assertions.Equal("person@example.com", got.Account) + assertions.Equal("team@example.com", got.CalendarID) + assertions.Equal("none", got.SendUpdates) + assertions.True(got.DryRun) + requirements.NotNil(got.Event.Start) + assertions.Equal("Europe/London", got.Event.Start.TimeZone) + requirements.NotNil(got.Event.Attendees) + assertions.Equal("guest@example.com", (*got.Event.Attendees)[0].Email) + requirements.NotNil(got.Event.Reminders) + assertions.Equal(15, got.Event.Reminders.Overrides[0].Minutes) +} +func TestCalendarControlCLIAllDayAndPartialPatch(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + var got calcontrol.Request + run := func(_ context.Context, r calcontrol.Request) (*calcontrol.Result, error) { + got = r + return &calcontrol.Result{}, nil + } + cmd := newCalendarControlCmd(run) + cmd.SetArgs([]string{"create", "primary", "--account", "person@example.com", "--summary", "Day off", "--from", "2026-10-02", "--to", "2026-10-03", "--all-day"}) + requirements.NoError(cmd.Execute()) + assertions.Equal("2026-10-02", got.Event.Start.Date) + cmd = newCalendarControlCmd(run) + cmd.SetArgs([]string{"update", "team", "event", "--account", "person@example.com", "--summary", "", "--add-attendee", "new@example.com, other@example.com"}) + requirements.NoError(cmd.Execute()) + requirements.NotNil(got.Event.Summary) + assertions.Empty(*got.Event.Summary) + assertions.Nil(got.Event.Start) + assertions.Nil(got.Event.End) + assertions.Equal([]string{"new@example.com", "other@example.com"}, got.AddAttendees) +} +func TestCalendarControlCLIRejectsInvalidInputBeforeBackend(t *testing.T) { + for _, flags := range [][]string{{"--from", "invalid", "--to", "2026-10-02T10:00:00Z"}, {"--from", "2026-10-02T09:00:00Z", "--to", "2026-10-02T08:00:00Z"}, {"--from", "2026-10-02T09:00:00Z", "--to", "2026-10-02T10:00:00Z", "--send-updates", "invalid"}} { + calls := 0 + cmd := newCalendarControlCmd(func(context.Context, calcontrol.Request) (*calcontrol.Result, error) { + calls++ + return &calcontrol.Result{}, nil + }) + cmd.SetArgs(append([]string{"create", "team", "--account", "person@example.com", "--summary", "Planning"}, flags...)) + require.Error(t, cmd.Execute()) + assert.Zero(t, calls) + } +} + +func TestCalendarControlCLIMovePositionalDestination(t *testing.T) { + var got calcontrol.Request + cmd := newCalendarControlCmd(func(_ context.Context, r calcontrol.Request) (*calcontrol.Result, error) { + got = r + return &calcontrol.Result{}, nil + }) + cmd.SetArgs([]string{"move", "team", "event", "other", "--account", "person@example.com"}) + require.NoError(t, cmd.Execute()) + assert.Equal(t, "other", got.Destination) +} + +func TestCalendarControlCLIScopeHelpMatchesSupportedActions(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + root := newCalendarControlCmd(nil) + respond, _, err := root.Find([]string{"respond"}) + requirements.NoError(err) + respondScope := respond.Flags().Lookup("scope") + requirements.NotNil(respondScope) + assertions.NotContains(respondScope.Usage, "future") + + update, _, err := root.Find([]string{"update"}) + requirements.NoError(err) + updateScope := update.Flags().Lookup("scope") + requirements.NotNil(updateScope) + assertions.Contains(updateScope.Usage, "future") +} + +func TestCalendarControlCLIReportsArchiveFailureWithoutMessageID(t *testing.T) { + var output strings.Builder + cmd := newCalendarControlCmd(func(context.Context, calcontrol.Request) (*calcontrol.Result, error) { + return &calcontrol.Result{Writes: []calcontrol.WriteReceipt{{ + Action: "update", CalendarID: "team", Event: gcal.Event{ID: "event"}, ArchiveError: "archive store unavailable", + }}}, nil + }) + cmd.SetArgs([]string{"update", "team", "event", "--account", "person@example.com", "--summary", "Planning"}) + cmd.SetOut(&output) + + err := cmd.Execute() + + require.ErrorContains(t, err, "archive write failed: archive store unavailable") + assert.Contains(t, output.String(), "(not archived: archive store unavailable)") + assert.NotContains(t, output.String(), "archive message 0") +} diff --git a/cmd/msgvault/cmd/calendar_control_integration_test.go b/cmd/msgvault/cmd/calendar_control_integration_test.go new file mode 100644 index 000000000..1c083c7d8 --- /dev/null +++ b/cmd/msgvault/cmd/calendar_control_integration_test.go @@ -0,0 +1,304 @@ +package cmd + +import ( + "bytes" + "context" + "encoding/json/v2" + "fmt" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/apiprotocol" + "go.kenn.io/msgvault/internal/calcontrol" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/daemonclient" + "go.kenn.io/msgvault/internal/gcal" + "go.kenn.io/msgvault/internal/testutil" + "golang.org/x/oauth2" +) + +func TestCalendarControlClassifiesDaemonSetupFailures(t *testing.T) { + for _, tc := range []struct { + name, token, code string + missingToken, closedStore bool + status int + }{ + {name: "missing write consent", token: gmailCalendarDriveTokenJSON, status: http.StatusForbidden, code: "calendar_denied"}, + {name: "missing calendar consent", token: gmailOnlyTokenJSON, status: http.StatusForbidden, code: "calendar_denied"}, + {name: "missing token", token: gmailCalendarDriveTokenJSON, missingToken: true, status: http.StatusForbidden, code: "calendar_denied"}, + {name: "store failure", token: gmailCalendarDriveTokenJSON, closedStore: true, status: http.StatusInternalServerError, code: "calendar_internal"}, + } { + t.Run(tc.name, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + tokenPath, restore := seedTokenEnv(t, tc.token) + defer restore() + if tc.missingToken { + requirements.NoError(os.Remove(tokenPath)) + } + home := filepath.Dir(filepath.Dir(tokenPath)) + cfg := config.NewDefaultConfig() + cfg.HomeDir, cfg.Data.DataDir = home, home + cfg.OAuth.ClientSecrets = filepath.Join(home, "client_secret.json") + cfg.GCal = []config.GCalSource{{Email: scopeEscalationAccount, Enabled: true, WriteCalendars: []string{"team@example.com"}}} + st := testutil.NewTestStore(t) + if tc.closedStore { + requirements.NoError(st.Close()) + } + adapter := &storeAPIAdapter{store: st, config: cfg, logger: slog.New(slog.DiscardHandler)} + start := gcal.EventDateTime{DateTime: time.Date(2026, 10, 2, 9, 0, 0, 0, time.UTC)} + end := gcal.EventDateTime{DateTime: start.DateTime.Add(time.Hour)} + title := "Planning" + control := calcontrol.Request{Action: "create", Account: scopeEscalationAccount, CalendarID: "team@example.com", Event: gcal.EventInput{Summary: &title, Start: &start, End: &end}} + requirements.NoError(control.Validate()) + body, err := json.Marshal(control) + requirements.NoError(err) + request := httptest.NewRequest(http.MethodPost, "/api/v1/calendar/control", bytes.NewReader(body)) + request.Header.Set("Content-Type", "application/json") + response := httptest.NewRecorder() + api.NewServer(cfg, adapter, nil, slog.New(slog.DiscardHandler)).Router().ServeHTTP(response, request) + assertions.Equal(tc.status, response.Code, response.Body.String()) + assertions.Contains(response.Body.String(), tc.code) + assertions.NotContains(response.Body.String(), "calendar_failed") + }) + } +} + +func TestCalendarControlDaemonClientArchiveAndDelegatedGrants(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + st := testutil.NewTestStore(t) + gate := api.NewSerialOperationGate() + var creates atomic.Int64 + var reader atomic.Bool + var writeClient atomic.Bool + provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "Bearer synthetic-calendar-token", r.Header.Get("Authorization")) + w.Header().Set("Content-Type", "application/json") + switch r.Method + " " + r.URL.Path { + case "GET /users/me/calendarList": + role := "owner" + if reader.Load() { + role = "reader" + } + assertions.NoError(json.MarshalWrite(w, gcal.CalendarListPage{Items: []gcal.Calendar{{ID: "team@example.com", AccessRole: role, Primary: true}, {ID: "other@example.com", AccessRole: "writer"}}})) + case "POST /freeBusy": + assertions.NoError(json.MarshalWrite(w, gcal.FreeBusyResponse{Calendars: map[string]gcal.CalendarBusy{"team@example.com": {Busy: []gcal.BusyPeriod{}}}})) + case "POST /calendars/team@example.com/events": + _, _, held := gate.Holder() + assertions.True(held, "calendar writes must hold the archive operation gate") + creates.Add(1) + assert.Equal(t, "none", r.URL.Query().Get("sendUpdates")) + var input gcal.EventInput + if !assertions.NoError(json.UnmarshalRead(r.Body, &input)) { + w.WriteHeader(http.StatusBadRequest) + return + } + assert.Nil(t, input.Attendees) + if !assertions.NotNil(input.Summary) { + w.WriteHeader(http.StatusBadRequest) + return + } + assertions.NoError(json.MarshalWrite(w, gcal.Event{ID: "created", Status: "confirmed", Summary: *input.Summary, Start: *input.Start, End: *input.End, Organizer: gcal.Person{Email: "team@example.com"}})) + default: + assert.Fail(t, "unexpected Google request", r.Method+" "+r.URL.Path) + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(provider.Close) + cfg := &config.Config{Server: config.ServerConfig{APIKey: "synthetic-owner-key", AgentAccess: true}, GCal: []config.GCalSource{{Email: "Person@Example.COM", Enabled: true, WriteCalendars: []string{"team@example.com", "other@example.com"}}}} + adapter := &storeAPIAdapter{store: st, config: cfg, logger: slog.New(slog.DiscardHandler), calendarClientFactory: func(_ context.Context, source config.GCalSource, write bool) (gcal.ControlAPI, error) { + assert.Equal(t, "person@example.com", source.Email) + writeClient.Store(write) + return gcal.NewClient(oauth2.StaticTokenSource(&oauth2.Token{AccessToken: "synthetic-calendar-token"}), gcal.WithBaseURL(provider.URL)), nil + }} + server := api.NewServerWithOptions(api.ServerOptions{Config: cfg, Store: adapter, Logger: slog.New(slog.DiscardHandler), OperationGate: gate}) + daemon := httptest.NewServer(server.Router()) + t.Cleanup(daemon.Close) + owner, err := daemonclient.New(daemonclient.Config{URL: daemon.URL, APIKey: cfg.Server.APIKey, AllowInsecure: true, HTTPClient: daemon.Client()}) + requirements.NoError(err) + t.Cleanup(func() { assert.NoError(t, owner.Close()) }) + title := "Planning" + start := gcal.EventDateTime{DateTime: time.Date(2026, 10, 2, 9, 0, 0, 0, time.UTC)} + end := gcal.EventDateTime{DateTime: start.DateTime.Add(time.Hour)} + request := calcontrol.Request{Action: "create", Account: "person@example.com", CalendarID: "team@example.com", Event: gcal.EventInput{Summary: &title, Start: &start, End: &end}} + result, err := owner.ControlCalendar(t.Context(), request) + requirements.NoError(err) + assertions.NotEmpty(result.PlanFingerprint) + fingerprint := result.PlanFingerprint + requirements.Len(result.Writes, 1) + assertions.True(result.Writes[0].Archived) + requirements.Positive(result.Writes[0].MessageID) + source, err := st.GetSourceByIdentifier("person@example.com/team@example.com") + requirements.NoError(err) + meta, err := st.GetMessageMetadata(result.Writes[0].MessageID) + requirements.NoError(err) + var archivedMetadata struct { + OrganizerEmail string `json:"organizer_email"` + AccountEmail string `json:"account_email"` + } + requirements.NoError(json.Unmarshal([]byte(meta.String), &archivedMetadata)) + assertions.Equal("team@example.com", archivedMetadata.OrganizerEmail) + assertions.Equal("person@example.com", archivedMetadata.AccountEmail) + issued, err := owner.IssueAgentToken(t.Context(), "calendar writer", []string{"calendar.write"}, []int64{source.ID}, nil) + requirements.NoError(err) + delegated, err := daemonclient.New(daemonclient.Config{URL: daemon.URL, AgentToken: issued.Secret, AllowInsecure: true, HTTPClient: daemon.Client()}) + requirements.NoError(err) + t.Cleanup(func() { assert.NoError(t, delegated.Close()) }) + releaseGate, ok := gate.BeginLabeledWorkContext(t.Context(), "scheduled source sync") + requirements.True(ok) + releaseGate = sync.OnceFunc(releaseGate) + defer releaseGate() + for _, action := range []string{"freebusy", "conflicts"} { + availability, err := owner.ControlCalendar(t.Context(), calcontrol.Request{ + Action: action, Account: request.Account, CalendarID: request.CalendarID, + TimeMin: start.DateTime, TimeMax: end.DateTime, + }) + requirements.NoError(err) + assertions.NotNil(availability.FreeBusy) + assertions.False(writeClient.Load()) + } + request.DryRun = true + result, err = delegated.ControlCalendar(t.Context(), request) + requirements.NoError(err) + assertions.Equal(fingerprint, result.PlanFingerprint) + assertions.True(result.DryRun) + assertions.True(writeClient.Load()) + assertions.Equal(int64(1), creates.Load()) + request.CalendarID = "primary" + _, err = delegated.ControlCalendar(t.Context(), request) + requirements.NoError(err, "primary is authorized against its live canonical ID") + otherSource, err := st.GetOrCreateSource(sourceTypeCalendar, "person@example.com/other@example.com") + requirements.NoError(err) + otherGrant, err := owner.IssueAgentToken(t.Context(), "other calendar", []string{"calendar.write"}, []int64{otherSource.ID}, nil) + requirements.NoError(err) + primaryBody, err := json.Marshal(request) + requirements.NoError(err) + primaryRequest := httptest.NewRequest(http.MethodPost, "/api/v1/calendar/control", bytes.NewReader(primaryBody)) + primaryRequest.Header.Set("Content-Type", "application/json") + primaryRequest.Header.Set(apiprotocol.AgentTokenHeader, otherGrant.Secret) + primaryResponse := httptest.NewRecorder() + server.Router().ServeHTTP(primaryResponse, primaryRequest) + assertions.Equal(http.StatusForbidden, primaryResponse.Code) + var primaryError api.ErrorResponse + requirements.NoError(json.Unmarshal(primaryResponse.Body.Bytes(), &primaryError)) + assertions.Equal(calcontrol.ErrDenied.Error(), primaryError.Message) + request.CalendarID = "other@example.com" + _, err = delegated.ControlCalendar(t.Context(), request) + requirements.Error(err) + assertions.Equal(int64(1), creates.Load()) + request.CalendarID = "team@example.com" + request.Event.Attendees = &[]gcal.Attendee{{Email: "guest@example.com"}} + _, err = delegated.ControlCalendar(t.Context(), request) + requirements.Error(err) + assertions.Equal(int64(1), creates.Load()) + request.Event.Attendees = nil + request.DryRun = false + body, err := json.Marshal(request) + requirements.NoError(err) + waitCtx, cancel := context.WithCancel(t.Context()) + blocked := httptest.NewRequestWithContext(waitCtx, http.MethodPost, "/api/v1/calendar/control", bytes.NewReader(body)) + blocked.Header.Set("Content-Type", "application/json") + blocked.Header.Set("X-Api-Key", cfg.Server.APIKey) + response := httptest.NewRecorder() + blockedDone := make(chan struct{}) + go func() { + defer close(blockedDone) + server.Router().ServeHTTP(response, blocked) + }() + assertions.Eventually(gate.HasRequestWaiters, time.Second, time.Millisecond, "validated mutation queues behind the held gate") + cancel() + <-blockedDone + assertions.Equal(http.StatusServiceUnavailable, response.Code, response.Body.String()) + assertions.Contains(response.Body.String(), "operation_in_progress") + assertions.Equal(int64(1), creates.Load(), "blocked mutation never reaches the provider") + releaseGate() + reader.Store(true) + _, err = owner.ControlCalendar(t.Context(), request) + requirements.Error(err) + assertions.Contains(err.Error(), "owner or writer") + assertions.Equal(int64(1), creates.Load()) +} + +func TestCalendarControlDelegatedSetupHidesConfiguration(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + st := testutil.NewTestStore(t) + team, err := st.GetOrCreateSource(sourceTypeCalendar, "person@example.com/team@example.com") + requirements.NoError(err) + other, err := st.GetOrCreateSource(sourceTypeCalendar, "person@example.com/other@example.com") + requirements.NoError(err) + cfg := &config.Config{Server: config.ServerConfig{APIKey: "synthetic-owner-key", AgentAccess: true}} + var logs strings.Builder + logger := slog.New(slog.NewTextHandler(&logs, nil)) + const credentialPath = "/configured/calendar-credentials.json" + var clientCalls atomic.Int64 + adapter := &storeAPIAdapter{store: st, config: cfg, logger: logger, calendarClientFactory: func(context.Context, config.GCalSource, bool) (gcal.ControlAPI, error) { + clientCalls.Add(1) + return nil, fmt.Errorf("read OAuth credentials %s: unavailable", credentialPath) + }} + server := api.NewServer(cfg, adapter, nil, logger) + daemon := httptest.NewServer(server.Router()) + t.Cleanup(daemon.Close) + owner, err := daemonclient.New(daemonclient.Config{URL: daemon.URL, APIKey: cfg.Server.APIKey, AllowInsecure: true, HTTPClient: daemon.Client()}) + requirements.NoError(err) + t.Cleanup(func() { assert.NoError(t, owner.Close()) }) + configured := []config.GCalSource{{Name: "work", Email: "Person@Example.COM", Enabled: true, + WriteCalendars: []string{"team@example.com"}, CalendarAliases: map[string]string{"team": "team@example.com"}}} + const body = `{"action":"create","account":"work","calendar_id":"team","event":{"summary":"Planning","start":{"dateTime":"2026-10-02T09:00:00Z"},"end":{"dateTime":"2026-10-02T10:00:00Z"}}}` + for _, tc := range []struct { + name, permission, calendar string + sourceID int64 + configured bool + clientCalls int64 + }{ + {"missing capability and configuration", "calendar.read", "team", team.ID, false, 0}, + {"missing capability with configuration", "calendar.read", "team", team.ID, true, 0}, + {"unrelated calendar grant", "calendar.write", "team", other.ID, true, 0}, + {"authorized setup failure", "calendar.write", "team", team.ID, true, 1}, + {"primary setup failure", "calendar.write", "primary", other.ID, true, 1}, + } { + t.Run(tc.name, func(t *testing.T) { + assertions := assert.New(t) + requirements := require.New(t) + if tc.configured { + cfg.GCal = configured + } else { + cfg.GCal = nil + } + issued, err := owner.IssueAgentToken(t.Context(), tc.name, []string{tc.permission}, []int64{tc.sourceID}, nil) + requirements.NoError(err) + request := httptest.NewRequest(http.MethodPost, "/api/v1/calendar/control", strings.NewReader(strings.Replace(body, `"calendar_id":"team"`, `"calendar_id":"`+tc.calendar+`"`, 1))) + request.Header.Set("Content-Type", "application/json") + request.Header.Set(apiprotocol.AgentTokenHeader, issued.Secret) + before := clientCalls.Load() + response := httptest.NewRecorder() + server.Router().ServeHTTP(response, request) + assertions.Equal(http.StatusForbidden, response.Code, response.Body.String()) + var result api.ErrorResponse + requirements.NoError(json.Unmarshal(response.Body.Bytes(), &result)) + assertions.Equal(calcontrol.ErrDenied.Error(), result.Message) + assertions.NotContains(response.Body.String(), credentialPath) + assertions.Equal(before+tc.clientCalls, clientCalls.Load()) + }) + } + assertions.Contains(logs.String(), credentialPath, "operator log retains delegated setup diagnostics") + request := httptest.NewRequest(http.MethodPost, "/api/v1/calendar/control", strings.NewReader(body)) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("X-Api-Key", cfg.Server.APIKey) + response := httptest.NewRecorder() + server.Router().ServeHTTP(response, request) + assertions.Equal(http.StatusInternalServerError, response.Code) + assertions.Contains(response.Body.String(), credentialPath, "owner retains setup diagnostics") +} diff --git a/cmd/msgvault/cmd/calendar_register_test.go b/cmd/msgvault/cmd/calendar_register_test.go index 536bff1ac..fee9ba55c 100644 --- a/cmd/msgvault/cmd/calendar_register_test.go +++ b/cmd/msgvault/cmd/calendar_register_test.go @@ -59,6 +59,8 @@ func TestRegisterCalendarsAndReport_NoMatchIsNotAnError(t *testing.T) { // configured with a service account: it resolves the binding and reports that // no consent/escalation round trip is needed. func TestPlanCLIAddCalendar_ServiceAccountAppNeedsNoConsent(t *testing.T) { + cfg := testConfigValue() + st := testutil.NewTestStore(t) savedCfg := cfg defer func() { cfg = savedCfg }() @@ -70,8 +72,10 @@ func TestPlanCLIAddCalendar_ServiceAccountAppNeedsNoConsent(t *testing.T) { }, }, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx - plan, err := planCLIAddCalendar(context.Background(), st, api.CLIAddCalendarPlanRequest{ + plan, err := planCLIAddCalendar(testCtx, st, api.CLIAddCalendarPlanRequest{ Email: "bob@example.com", OAuthApp: "sa", OAuthAppExplicit: true, }) require.NoError(t, err, "service-account apps have no client_secrets and must not be asked for one") diff --git a/cmd/msgvault/cmd/calendar_test.go b/cmd/msgvault/cmd/calendar_test.go index 471b0e2d4..512190dd7 100644 --- a/cmd/msgvault/cmd/calendar_test.go +++ b/cmd/msgvault/cmd/calendar_test.go @@ -107,11 +107,17 @@ func TestBuildCalendarClientRejectsLegacyTokenWithoutCalendarScope(t *testing.T) tokenPath, restore := seedTokenEnv(t, legacyTokenJSON) defer restore() + configHome := filepath.Dir(filepath.Dir(tokenPath)) + cfg := config.NewDefaultConfig() + cfg.HomeDir = configHome + cfg.Data.DataDir = configHome + cfg.OAuth.ClientSecrets = filepath.Join(configHome, "client_secret.json") + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) before, err := os.ReadFile(tokenPath) require.NoError(err, "read seeded token") - client, err := buildCalendarClient(context.Background(), scopeEscalationAccount, "", false) + client, err := buildCalendarClient(testCtx, scopeEscalationAccount, "", false) if client != nil { defer func() { _ = client.Close() }() } @@ -161,11 +167,11 @@ func TestCalendarAddOAuthAppDecisionFallsBackToGmailBinding(t *testing.T) { require := require.New(t) st := newCalendarDecisionStore(t) - src, err := st.GetOrCreateSource(sourceTypeGmail, "User@Acme.com") + src, err := st.GetOrCreateSource(sourceTypeGmail, "User@example.com") require.NoError(err) require.NoError(st.UpdateSourceOAuthApp(src.ID, sql.NullString{String: "acme", Valid: true})) - decision, err := calendarAddOAuthAppDecision(st, "user@acme.com", "", false) + decision, err := calendarAddOAuthAppDecision(st, "user@example.com", "", false) require.NoError(err) assert.Equal("acme", decision.OAuthApp) @@ -195,6 +201,9 @@ func TestCalendarAddOAuthAppDecisionKeepsCalendarDefaultOverGmailBinding(t *test } func TestAddCalendarHeadlessNormalizesAccountEmail(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + assert := assert.New(t) require := require.New(t) @@ -213,13 +222,15 @@ func TestAddCalendarHeadlessNormalizesAccountEmail(t *testing.T) { Data: config.DataConfig{DataDir: tmpDir}, OAuth: config.OAuthConfig{ClientSecrets: secretsPath}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) addCmd := newAddCalendarLocalCmd() + addCmd.SetContext(testCtx) addCmd.SetArgs([]string{"--headless", "Alice.Example@Example.COM"}) getOutput := captureStdout(t) - err := addCmd.Execute() + err := addCmd.ExecuteContext(testCtx) out := getOutput() require.NoError(err) @@ -270,11 +281,11 @@ func TestCalendarSyncOAuthAppDecisionFallsBackToGmailBinding(t *testing.T) { require := require.New(t) st := newCalendarDecisionStore(t) - src, err := st.GetOrCreateSource(sourceTypeGmail, "User@Acme.com") + src, err := st.GetOrCreateSource(sourceTypeGmail, "User@example.com") require.NoError(err) require.NoError(st.UpdateSourceOAuthApp(src.ID, sql.NullString{String: "acme", Valid: true})) - decision, err := calendarSyncOAuthAppDecision(st, "user@acme.com", nil, "", false) + decision, err := calendarSyncOAuthAppDecision(st, "user@example.com", nil, "", false) require.NoError(err) assert.Equal("acme", decision.OAuthApp) @@ -304,6 +315,9 @@ func TestCalendarSyncOAuthAppDecisionKeepsCalendarDefaultOverGmailBinding(t *tes } func TestCalendarAddTokenReusableRejectsMismatchedInheritedClient(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + assert := assert.New(t) require := require.New(t) @@ -312,7 +326,7 @@ func TestCalendarAddTokenReusableRejectsMismatchedInheritedClient(t *testing.T) require.NoError(os.WriteFile(secretsPath, []byte(fakeClientSecrets), 0600)) tokensDir := filepath.Join(tmpDir, "tokens") require.NoError(os.MkdirAll(tokensDir, 0700)) - writeCalendarToken(t, tokensDir, "user@acme.com", "wrong-client.apps.googleusercontent.com") + writeCalendarToken(t, tokensDir, "user-a@example.com", "wrong-client.apps.googleusercontent.com") savedCfg, savedLogger := cfg, logger defer func() { cfg, logger = savedCfg, savedLogger }() @@ -325,13 +339,19 @@ func TestCalendarAddTokenReusableRejectsMismatchedInheritedClient(t *testing.T) }, }, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) - mgr, err := newCalendarOAuthManager(secretsPath, "user@acme.com") + mgr, err := newCalendarOAuthManager( + secretsPath, + "user-a@example.com", + invocationFromContext(testCtx), + ) require.NoError(err) decision := calendarAddOAuthApp{OAuthApp: "acme", NeedsClientCheck: true} - assert.False(calendarAddTokenReusable(mgr, "user@acme.com", decision), + assert.False(calendarAddTokenReusable(mgr, "user-a@example.com", decision), "a calendar token minted by another OAuth client must force reauthorization") } diff --git a/cmd/msgvault/cmd/carddav.go b/cmd/msgvault/cmd/carddav.go index 1a21f2d07..ad916c6e9 100644 --- a/cmd/msgvault/cmd/carddav.go +++ b/cmd/msgvault/cmd/carddav.go @@ -13,6 +13,7 @@ import ( "github.com/mattn/go-isatty" "github.com/spf13/cobra" "go.kenn.io/msgvault/internal/carddav" + "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/daemonclient" "go.kenn.io/msgvault/internal/textutil" apiclient "go.kenn.io/msgvault/pkg/client" @@ -20,7 +21,7 @@ import ( ) func newAddCardDAVCmd() *cobra.Command { - var schedule string + var schedule, connection string var disabled, google bool var oauthApp string cmd := &cobra.Command{Use: "add-carddav | --google ", Short: "Discover and configure a CardDAV account", Args: func(cmd *cobra.Command, args []string) error { @@ -30,11 +31,14 @@ func newAddCardDAVCmd() *cobra.Command { return cobra.ExactArgs(2)(cmd, args) }} cmd.RunE = func(cmd *cobra.Command, args []string) error { + selector, err := cardDAVCLIConnection(cmd, connection) + if err != nil { + return err + } if !google && oauthApp != "" { return usageErr(cmd, errors.New("--oauth-app requires --google")) } var password string - var err error if !google { password, err = readCardDAVPassword() if err != nil { @@ -50,7 +54,7 @@ func newAddCardDAVCmd() *cobra.Command { return err } defer func() { _ = client.Close() }() - body := generated.SaveCardDAVAccountBody{BaseURL: baseURL, Username: username, Password: &password, Enabled: !disabled} + body := generated.SaveCardDAVAccountBody{BaseURL: baseURL, Username: username, Password: &password, Enabled: !disabled, Connection: selector} if google { provider := generated.Google body.Provider = &provider @@ -70,6 +74,7 @@ func newAddCardDAVCmd() *cobra.Command { textutil.SanitizeTerminal(resp.JSON200.Username), resp.JSON200.Books) return nil } + cmd.Flags().StringVar(&connection, "connection", "", "Connection name (default when omitted)") cmd.Flags().BoolVar(&google, "google", false, "Use Google Contacts with an authorized OAuth token") cmd.Flags().StringVar(&oauthApp, "oauth-app", "", "Named Google OAuth application") cmd.Flags().StringVar(&schedule, "schedule", "", "cron schedule for background synchronization") @@ -100,14 +105,19 @@ func readCardDAVPassword() (string, error) { func newSyncCardDAVCmd() *cobra.Command { var full bool - cmd := &cobra.Command{Use: "sync-carddav", Short: "Synchronize the configured CardDAV account", Args: cobra.NoArgs} + var connection string + cmd := &cobra.Command{Use: "sync-carddav", Short: "Synchronize enabled CardDAV connections", Args: cobra.NoArgs} cmd.RunE = func(cmd *cobra.Command, _ []string) error { + selector, err := cardDAVCLIConnection(cmd, connection) + if err != nil { + return err + } client, _, err := OpenHTTPStore(cmd.Context()) if err != nil { return err } defer func() { _ = client.Close() }() - body := generated.SyncCardDAVBody{Full: &full} + body := generated.SyncCardDAVBody{Full: &full, Connection: selector} resp, err := daemonclient.APIResponse(client, func(api *apiclient.Client) (*generated.SyncCardDAVResp, error) { return api.SyncCardDAVWithResponse(cmd.Context(), &generated.SyncCardDAVRequestOptions{Body: &body}) }) @@ -115,8 +125,21 @@ func newSyncCardDAVCmd() *cobra.Command { return err } _, _ = fmt.Fprintf(cmd.OutOrStdout(), "CardDAV sync: %d books, %d created, %d updated, %d removed\n", resp.JSON200.Books, resp.JSON200.Created, resp.JSON200.Updated, resp.JSON200.Removed) + if resp.JSON200.Status != nil && *resp.JSON200.Status != "succeeded" { + for _, outcome := range resp.JSON200.Connections { + if outcome.Status != "succeeded" { + code := "sync_failed" + if outcome.ErrorCode != nil { + code = *outcome.ErrorCode + } + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "CardDAV %s: %s\n", textutil.SanitizeTerminal(outcome.Connection), textutil.SanitizeTerminal(code)) + } + } + return fmt.Errorf("CardDAV sync %s", *resp.JSON200.Status) + } return nil } + cmd.Flags().StringVar(&connection, "connection", "", "Sync one saved connection, including disabled connections") cmd.Flags().BoolVar(&full, "full", false, "force a full address-book reconciliation") return cmd } @@ -124,6 +147,7 @@ func newSyncCardDAVCmd() *cobra.Command { func newCardDAVCmd() *cobra.Command { root := &cobra.Command{Use: "carddav", Short: "Manage CardDAV connections, books, and conflicts"} books := &cobra.Command{Use: "books", Short: "List discovered CardDAV address books", Args: cobra.NoArgs, RunE: runCardDAVBooks} + books.Flags().String("connection", "", "List books for one saved connection") var writeTarget, subscribed, lookup bool setRole := &cobra.Command{Use: "set-role ", Short: "Set all roles for a CardDAV address book", Args: cobra.ExactArgs(1)} setRole.RunE = func(cmd *cobra.Command, args []string) error { @@ -154,26 +178,34 @@ func newCardDAVCmd() *cobra.Command { conflicts.AddCommand(&cobra.Command{Use: "show ", Short: "Show safe base, local, and remote summaries for a CardDAV conflict", Args: cobra.ExactArgs(1), RunE: runCardDAVConflictShow}) resolve := &cobra.Command{Use: "resolve ", Short: "Resolve one CardDAV conflict", Args: cobra.ExactArgs(2), RunE: runCardDAVResolve} conflicts.AddCommand(resolve) - root.AddCommand(books, conflicts, newAuthorizeGoogleCardDAVCmd()) + root.AddCommand(books, conflicts, newAuthorizeGoogleCardDAVCmd(), &cobra.Command{Use: "connections", Short: "List saved CardDAV connections", Args: cobra.NoArgs, RunE: runCardDAVConnections}) return root } func runCardDAVBooks(cmd *cobra.Command, _ []string) error { + name, err := cmd.Flags().GetString("connection") + if err != nil { + return fmt.Errorf("read CardDAV connection flag: %w", err) + } + selector, err := cardDAVCLIConnection(cmd, name) + if err != nil { + return err + } client, _, err := OpenHTTPStore(cmd.Context()) if err != nil { return err } defer func() { _ = client.Close() }() resp, err := daemonclient.APIResponse(client, func(api *apiclient.Client) (*generated.ListCardDAVBooksResp, error) { - return api.ListCardDAVBooksWithResponse(cmd.Context()) + return api.ListCardDAVBooksWithResponse(cmd.Context(), &generated.ListCardDAVBooksRequestOptions{Query: &generated.ListCardDAVBooksQuery{Connection: selector}}) }) if err != nil { return err } w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 2, ' ', 0) - _, _ = fmt.Fprintln(w, "ID\tNAME\tWRITE\tSUBSCRIBED\tLOOKUP\tRECONCILE") + _, _ = fmt.Fprintln(w, "ID\tCONNECTION\tNAME\tWRITE\tSUBSCRIBED\tLOOKUP\tRECONCILE") for _, b := range resp.JSON200.Books { - _, _ = fmt.Fprintf(w, "%d\t%s\t%t\t%t\t%t\t%t\n", b.ID, + _, _ = fmt.Fprintf(w, "%d\t%s\t%s\t%t\t%t\t%t\t%t\n", b.ID, cardDAVCLIOptionalString(b.Connection), textutil.SanitizeTerminal(b.Name), b.WriteTarget, b.Subscribed, b.LookupSource, b.NeedsFullReconcile) } @@ -302,3 +334,47 @@ func init() { rootCmd.AddCommand(newAddCardDAVCmd(), newSyncCardDAVCmd(), newCardDAVCmd()) personCmd.AddCommand(newPersonCardDAVCommand("publish", true), newPersonCardDAVCommand("unpublish", false)) } + +func cardDAVCLIConnection(cmd *cobra.Command, name string) (*string, error) { + if name == "" && !cmd.Flags().Changed("connection") { + return nil, nil //nolint:nilnil // A nil selector deliberately means aggregate sync or an unfiltered read. + } + if err := config.ValidateCardDAVConnectionName(name); err != nil { + return nil, usageErr(cmd, fmt.Errorf("invalid CardDAV connection: %w", err)) + } + return &name, nil +} + +func cardDAVCLIOptionalString(value *string) string { + if value == nil { + return "" + } + return textutil.SanitizeTerminal(*value) +} + +func runCardDAVConnections(cmd *cobra.Command, _ []string) error { + client, _, err := OpenHTTPStore(cmd.Context()) + if err != nil { + return err + } + defer func() { _ = client.Close() }() + response, err := daemonclient.APIResponse(client, func(api *apiclient.Client) (*generated.ListCardDAVConnectionsResp, error) { + return api.ListCardDAVConnectionsWithResponse(cmd.Context()) + }) + if err != nil { + return err + } + w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 2, ' ', 0) + _, _ = fmt.Fprintln(w, "CONNECTION\tENABLED\tAVAILABLE\tSTATE") + for _, connection := range response.JSON200.Connections { + state := "configured" + if connection.Orphaned { + state = "orphaned (restore configuration)" + } + _, _ = fmt.Fprintf(w, "%s\t%t\t%t\t%s\n", textutil.SanitizeTerminal(connection.Connection), connection.Status.Enabled, connection.Status.Available, state) + } + if err := w.Flush(); err != nil { + return fmt.Errorf("flush CardDAV connections: %w", err) + } + return nil +} diff --git a/cmd/msgvault/cmd/carddav_google.go b/cmd/msgvault/cmd/carddav_google.go index 6f7b796cc..f17a45b5f 100644 --- a/cmd/msgvault/cmd/carddav_google.go +++ b/cmd/msgvault/cmd/carddav_google.go @@ -20,6 +20,12 @@ func newAuthorizeGoogleCardDAVCmd() *cobra.Command { Long: "Authorize Google Contacts on this machine. Reuse a matching Google authorization when available, or store separate CardDAV credentials. Then select Google Contacts in CardDAV settings. For a remote daemon, copy the token to that host using the same OAuth client configuration.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger email := strings.ToLower(strings.TrimSpace(args[0])) address, err := mail.ParseAddress(email) if err != nil || address.Address != email { @@ -31,7 +37,7 @@ func newAuthorizeGoogleCardDAVCmd() *cobra.Command { } mgr, err := carddav.NewGoogleOAuthManager(secrets, cfg.TokensDir(), app, email, logger) if err != nil { - return wrapOAuthError(fmt.Errorf("create oauth manager: %w", err)) + return wrapOAuthError(fmt.Errorf("create oauth manager: %w", err), cfg) } if mgr.HasToken(email) && !mgr.HasScopeMetadata(email) { if _, err := fmt.Fprintln(cmd.ErrOrStderr(), "Warning: existing Google permissions are not recorded. This sign-in requests Contacts access; Gmail or Calendar may need separate reauthorization afterward."); err != nil { diff --git a/cmd/msgvault/cmd/carddav_google_test.go b/cmd/msgvault/cmd/carddav_google_test.go index add344bd9..b1f214aa5 100644 --- a/cmd/msgvault/cmd/carddav_google_test.go +++ b/cmd/msgvault/cmd/carddav_google_test.go @@ -14,7 +14,12 @@ import ( func TestAuthorizeGoogleCardDAVValidatesEmailAndExplainsMissingSecrets(t *testing.T) { dir := t.TempDir() - withStoreResolverConfig(t, &config.Config{HomeDir: dir, Data: config.DataConfig{DataDir: dir}, OAuth: config.OAuthConfig{ClientSecrets: filepath.Join(dir, "missing.json")}}) + cfg := &config.Config{ + HomeDir: dir, + Data: config.DataConfig{DataDir: dir}, + OAuth: config.OAuthConfig{ClientSecrets: filepath.Join(dir, "missing.json")}, + } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) for _, tc := range []struct{ email, wantError string }{ {"person name@example.com", "invalid email address"}, {"Person ", "invalid email address"}, @@ -22,6 +27,7 @@ func TestAuthorizeGoogleCardDAVValidatesEmailAndExplainsMissingSecrets(t *testin } { t.Run(tc.email, func(t *testing.T) { cmd := newAuthorizeGoogleCardDAVCmd() + cmd.SetContext(testCtx) err := cmd.RunE(cmd, []string{tc.email}) require.ErrorContains(t, err, tc.wantError) }) @@ -33,7 +39,12 @@ func TestAuthorizeGoogleCardDAVAllowsClientRotation(t *testing.T) { dir := t.TempDir() secrets := filepath.Join(dir, "client.json") required.NoError(os.WriteFile(secrets, []byte(`{"installed":{"client_id":"selected-client","client_secret":"synthetic-secret","auth_uri":"https://accounts.example/authorize","token_uri":"https://accounts.example/token","redirect_uris":["http://localhost"]}}`), 0600)) - withStoreResolverConfig(t, &config.Config{HomeDir: dir, Data: config.DataConfig{DataDir: dir}, OAuth: config.OAuthConfig{ClientSecrets: secrets}}) + cfg := &config.Config{ + HomeDir: dir, + Data: config.DataConfig{DataDir: dir}, + OAuth: config.OAuthConfig{ClientSecrets: secrets}, + } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) mgr, err := carddav.NewGoogleOAuthManager(secrets, cfg.TokensDir(), "", "person@example.com", nil) required.NoError(err) required.NoError(os.MkdirAll(filepath.Dir(mgr.TokenPath("person@example.com")), 0700)) @@ -44,7 +55,7 @@ func TestAuthorizeGoogleCardDAVAllowsClientRotation(t *testing.T) { required := require.New(t) cmd := newAuthorizeGoogleCardDAVCmd() required.NoError(cmd.Flags().Set("no-browser", manual)) - ctx, cancel := context.WithCancel(t.Context()) + ctx, cancel := context.WithCancel(testCtx) cancel() cmd.SetContext(ctx) required.ErrorIs(cmd.RunE(cmd, []string{"person@example.com"}), context.Canceled) diff --git a/cmd/msgvault/cmd/carddav_test.go b/cmd/msgvault/cmd/carddav_test.go index dd88ecce1..f9fa86479 100644 --- a/cmd/msgvault/cmd/carddav_test.go +++ b/cmd/msgvault/cmd/carddav_test.go @@ -17,31 +17,31 @@ import ( ) func TestCardDAVCommandsExposeSafeOperatorSurface(t *testing.T) { - assert := assert.New(t) + assertions := assert.New(t) require := require.New(t) add := newAddCardDAVCmd() - assert.Nil(add.Flags().Lookup("password"), "passwords must never be accepted on argv") - assert.Equal("add-carddav", add.Name()) + assertions.Nil(add.Flags().Lookup("password"), "passwords must never be accepted on argv") + assertions.Equal("add-carddav", add.Name()) root := newCardDAVCmd() books, _, err := root.Find([]string{"books"}) require.NoError(err) - assert.Equal("books", books.Name()) + assertions.Equal("books", books.Name()) setRole, _, err := root.Find([]string{"books", "set-role"}) require.NoError(err) - assert.Equal("set-role", setRole.Name()) + assertions.Equal("set-role", setRole.Name()) resolve, _, err := root.Find([]string{"conflicts", "resolve"}) require.NoError(err) - assert.Equal("resolve", resolve.Name()) + assertions.Equal("resolve", resolve.Name()) show, _, err := root.Find([]string{"conflicts", "show"}) require.NoError(err) - assert.Equal("show", show.Name()) - assert.Equal("Show safe base, local, and remote summaries for a CardDAV conflict", show.Short) + assertions.Equal("show", show.Name()) + assertions.Equal("Show safe base, local, and remote summaries for a CardDAV conflict", show.Short) } func TestCardDAVCLIProductionRoutes(t *testing.T) { - assert := assert.New(t) + assertions := assert.New(t) require := require.New(t) requests := make([]string, 0, 8) @@ -51,18 +51,18 @@ func TestCardDAVCLIProductionRoutes(t *testing.T) { switch r.Method + " " + r.URL.Path { case "PUT /api/v1/carddav/account": var body map[string]any - assert.NoError(json.NewDecoder(r.Body).Decode(&body)) - assert.Equal("https://contacts.example/dav", body["base_url"]) - assert.Equal("alice", body["username"]) - assert.Equal("synthetic-password", body["password"]) - assert.Equal("0 3 * * *", body["schedule"]) + assertions.NoError(json.NewDecoder(r.Body).Decode(&body)) + assertions.Equal("https://contacts.example/dav", body["base_url"]) + assertions.Equal("alice", body["username"]) + assertions.Equal("synthetic-password", body["password"]) + assertions.Equal("0 3 * * *", body["schedule"]) _, _ = w.Write([]byte(`{"base_url":"https://contacts.example/dav","username":"alice","enabled":true,"schedule":"0 3 * * *","books":1}`)) case "GET /api/v1/carddav/books": _, _ = w.Write([]byte(`{"books":[{"id":9,"name":"Personal","url":"https://contacts.example/books/personal/","write_target":true,"subscribed":true,"lookup_source":false,"needs_full_reconcile":false}]}`)) case "PATCH /api/v1/carddav/books/9": var body map[string]bool - assert.NoError(json.NewDecoder(r.Body).Decode(&body)) - assert.Equal(map[string]bool{"write_target": true, "subscribed": true, "lookup_source": true}, body) + assertions.NoError(json.NewDecoder(r.Body).Decode(&body)) + assertions.Equal(map[string]bool{"write_target": true, "subscribed": true, "lookup_source": true}, body) _, _ = w.Write([]byte(`{"id":9,"name":"Personal","url":"https://contacts.example/books/personal/","write_target":true,"subscribed":true,"lookup_source":true,"needs_full_reconcile":false}`)) case "GET /api/v1/carddav/conflicts": _, _ = w.Write([]byte(`{"conflicts":[]}`)) @@ -70,8 +70,8 @@ func TestCardDAVCLIProductionRoutes(t *testing.T) { _, _ = w.Write([]byte(`{"id":7,"address_book":{"id":9,"name":"Personal"},"status":"unresolved","base":{"state":"unavailable","emails":[],"phones":[]},"local":{"state":"present","display_name":"Local Alice","emails":["local@example.test"],"phones":[]},"remote":{"state":"present","display_name":"Remote Alice","emails":[],"phones":["+12025550123"]},"allowed_resolutions":["keep_local","keep_remote"],"created_at":"2026-08-28T09:10:11Z","updated_at":"2026-08-28T10:11:12Z"}`)) case "POST /api/v1/carddav/conflicts/7/resolve": var body map[string]string - assert.NoError(json.NewDecoder(r.Body).Decode(&body)) - assert.Equal("keep_remote", body["choice"]) + assertions.NoError(json.NewDecoder(r.Body).Decode(&body)) + assertions.Equal("keep_remote", body["choice"]) _, _ = w.Write([]byte(`{"id":7,"status":"resolved","resolution":"keep_remote"}`)) case "POST /api/v1/carddav/publications/11": _, _ = w.Write([]byte(`{"person_id":11,"state":"published","desired":true,"address_book":{"id":9,"name":"Personal"}}`)) @@ -79,8 +79,8 @@ func TestCardDAVCLIProductionRoutes(t *testing.T) { _, _ = w.Write([]byte(`{"person_id":11,"address_book":{"id":9,"name":"Personal"},"kind":"current","vcard":"BEGIN:VCARD\r\nEND:VCARD\r\n","approval_token":"synthetic-token","review_required":true}`)) case "POST /api/v1/carddav/publications/11/approve": var body map[string]string - assert.NoError(json.NewDecoder(r.Body).Decode(&body)) - assert.Equal("synthetic-token", body["approval_token"]) + assertions.NoError(json.NewDecoder(r.Body).Decode(&body)) + assertions.Equal("synthetic-token", body["approval_token"]) _, _ = w.Write([]byte(`{"person_id":11,"state":"published","desired":true,"address_book":{"id":9,"name":"Personal"}}`)) case "DELETE /api/v1/carddav/publications/11": _, _ = w.Write([]byte(`{"person_id":11,"state":"unpublished","desired":false,"address_book":{"id":9,"name":"Personal"}}`)) @@ -90,7 +90,7 @@ func TestCardDAVCLIProductionRoutes(t *testing.T) { })) t.Cleanup(server.Close) home := t.TempDir() - withStoreResolverConfig(t, &config.Config{HomeDir: home, Data: config.DataConfig{DataDir: home}, Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) + testCtx := withStoreResolverConfig(t, &config.Config{HomeDir: home, Data: config.DataConfig{DataDir: home}, Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) readEnd, writeEnd, err := os.Pipe() require.NoError(err) @@ -101,6 +101,7 @@ func TestCardDAVCLIProductionRoutes(t *testing.T) { os.Stdin = readEnd t.Cleanup(func() { os.Stdin = originalStdin; _ = readEnd.Close() }) add := newAddCardDAVCmd() + add.SetContext(testCtx) add.SetOut(&bytes.Buffer{}) add.SetArgs([]string{"https://contacts.example/dav", "alice", "--schedule", "0 3 * * *"}) require.NoError(add.Execute()) @@ -121,17 +122,18 @@ func TestCardDAVCLIProductionRoutes(t *testing.T) { {cmd: newPersonCardDAVCommand("unpublish", false), args: []string{"11"}}, } { out := &bytes.Buffer{} + invocation.cmd.SetContext(testCtx) invocation.cmd.SetOut(out) invocation.cmd.SetErr(&bytes.Buffer{}) invocation.cmd.SetArgs(invocation.args) require.NoError(invocation.cmd.Execute()) if slices.Contains(invocation.args, "--preview") { - assert.Contains(out.String(), `"approval_token":"synthetic-token"`) - assert.Contains(out.String(), `"vcard":"BEGIN:VCARD`) + assertions.Contains(out.String(), `"approval_token":"synthetic-token"`) + assertions.Contains(out.String(), `"vcard":"BEGIN:VCARD`) } } - assert.Equal([]string{ + assertions.Equal([]string{ "PUT /api/v1/carddav/account", "GET /api/v1/carddav/books", "PATCH /api/v1/carddav/books/9", @@ -143,19 +145,19 @@ func TestCardDAVCLIProductionRoutes(t *testing.T) { "POST /api/v1/carddav/publications/11/approve", "DELETE /api/v1/carddav/publications/11", }, requests) - assert.NotContains(strings.Join(requests, "\n"), "synthetic-password") + assertions.NotContains(strings.Join(requests, "\n"), "synthetic-password") } func TestCardDAVConflictShowPrintsSafeSummariesWithoutRawVCardFields(t *testing.T) { - assert := assert.New(t) + assertions := assert.New(t) require := require.New(t) const localRawMarker = "synthetic-local-raw-card" const remoteRawMarker = "synthetic-remote-raw-card" server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - assert.Equal(http.MethodGet, r.Method) - assert.Equal("/api/v1/carddav/conflicts/7", r.URL.Path) + assertions.Equal(http.MethodGet, r.Method) + assertions.Equal("/api/v1/carddav/conflicts/7", r.URL.Path) w.Header().Set("Content-Type", "application/json") - assert.NoError(json.NewEncoder(w).Encode(map[string]any{ + assertions.NoError(json.NewEncoder(w).Encode(map[string]any{ "id": 7, "address_book": map[string]any{"id": 9, "name": "Personal"}, "status": "resolved", "resolution": "keep_remote", "base": map[string]any{"state": "unavailable", "emails": []string{}, "phones": []string{}}, @@ -174,17 +176,18 @@ func TestCardDAVConflictShowPrintsSafeSummariesWithoutRawVCardFields(t *testing. })) t.Cleanup(server.Close) home := t.TempDir() - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ HomeDir: home, Data: config.DataConfig{DataDir: home}, Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) var stdout bytes.Buffer cmd := newCardDAVCmd() + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetArgs([]string{"conflicts", "show", "7"}) require.NoError(cmd.Execute()) - assert.JSONEq(`{ + assertions.JSONEq(`{ "id":7, "address_book":{"id":9,"name":"Personal"}, "status":"resolved", @@ -197,14 +200,14 @@ func TestCardDAVConflictShowPrintsSafeSummariesWithoutRawVCardFields(t *testing. "updated_at":"2026-08-28T10:11:12Z", "resolved_at":"2026-08-28T11:12:13Z" }`, stdout.String()) - assert.NotContains(stdout.String(), "local_vcard") - assert.NotContains(stdout.String(), "remote_vcard") - assert.NotContains(stdout.String(), localRawMarker) - assert.NotContains(stdout.String(), remoteRawMarker) + assertions.NotContains(stdout.String(), "local_vcard") + assertions.NotContains(stdout.String(), "remote_vcard") + assertions.NotContains(stdout.String(), localRawMarker) + assertions.NotContains(stdout.String(), remoteRawMarker) } func TestCardDAVBooksSanitizesTerminalControls(t *testing.T) { - assert := assert.New(t) + assertions := assert.New(t) require := require.New(t) type bookResponse struct { ID int64 `json:"id"` @@ -217,10 +220,10 @@ func TestCardDAVBooksSanitizesTerminalControls(t *testing.T) { } malicious := "\x1b[31mPersonal\x1b[0m \x1b]8;;https://attacker.test\x07link\x1b]8;;\x07" server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - assert.Equal(http.MethodGet, r.Method) - assert.Equal("/api/v1/carddav/books", r.URL.Path) + assertions.Equal(http.MethodGet, r.Method) + assertions.Equal("/api/v1/carddav/books", r.URL.Path) w.Header().Set("Content-Type", "application/json") - assert.NoError(json.NewEncoder(w).Encode(struct { + assertions.NoError(json.NewEncoder(w).Encode(struct { Books []bookResponse `json:"books"` }{ Books: []bookResponse{{ @@ -231,30 +234,33 @@ func TestCardDAVBooksSanitizesTerminalControls(t *testing.T) { })) t.Cleanup(server.Close) home := t.TempDir() - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ HomeDir: home, Data: config.DataConfig{DataDir: home}, Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) var stdout bytes.Buffer cmd := newCardDAVCmd() + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetArgs([]string{"books"}) require.NoError(cmd.Execute()) - assert.NotContains(stdout.String(), "\x1b") - assert.NotContains(stdout.String(), "https://attacker.test") - assert.Contains(stdout.String(), "Personal link") + assertions.NotContains(stdout.String(), "\x1b") + assertions.NotContains(stdout.String(), "https://attacker.test") + assertions.Contains(stdout.String(), "Personal link") } func TestSyncCardDAVUsesTheDaemonServiceRoute(t *testing.T) { + assertions := assert.New(t) + var full bool server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - assert.Equal(t, http.MethodPost, r.Method) - assert.Equal(t, "/api/v1/carddav/sync", r.URL.Path) + assertions.Equal(http.MethodPost, r.Method) + assertions.Equal("/api/v1/carddav/sync", r.URL.Path) var body struct { Full bool `json:"full"` } - assert.NoError(t, json.NewDecoder(r.Body).Decode(&body)) + assertions.NoError(json.NewDecoder(r.Body).Decode(&body)) full = body.Full w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"books":1,"created":2,"updated":3,"removed":4}`)) @@ -262,16 +268,115 @@ func TestSyncCardDAVUsesTheDaemonServiceRoute(t *testing.T) { t.Cleanup(server.Close) home := t.TempDir() - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ HomeDir: home, Data: config.DataConfig{DataDir: home}, Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) var stdout bytes.Buffer cmd := newSyncCardDAVCmd() + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetArgs([]string{"--full"}) require.NoError(t, cmd.Execute()) - assert.True(t, full) - assert.Equal(t, "CardDAV sync: 1 books, 2 created, 3 updated, 4 removed\n", stdout.String()) + assertions.True(full) + assertions.Equal("CardDAV sync: 1 books, 2 created, 3 updated, 4 removed\n", stdout.String()) +} + +func TestCardDAVCLISelectsConnectionsAndReportsAggregateFailures(t *testing.T) { + for _, state := range []string{"succeeded", "partial", "failed"} { + t.Run(state, func(t *testing.T) { + assertions := assert.New(t) + require := require.New(t) + + var requests []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests = append(requests, r.URL.RequestURI()) + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/api/v1/carddav/connections": + _, _ = w.Write([]byte(`{"connections":[{"connection":"work","account_id":2,"orphaned":false,"status":{"enabled":true,"available":true}},{"connection":"old","account_id":3,"orphaned":true,"status":{"enabled":false,"available":false}}]}`)) + case "/api/v1/carddav/books": + assertions.Equal("work", r.URL.Query().Get("connection")) + _, _ = w.Write([]byte(`{"books":[{"id":9,"account_id":2,"connection":"work","name":"Work contacts","url":"https://contacts.example/books/","write_target":false,"subscribed":true,"lookup_source":false,"needs_full_reconcile":false}]}`)) + case "/api/v1/carddav/sync": + var body map[string]any + if !assertions.NoError(json.NewDecoder(r.Body).Decode(&body)) { + http.Error(w, "invalid synthetic request", http.StatusBadRequest) + return + } + if body["connection"] != nil { + assertions.Equal("work", body["connection"]) + _, _ = w.Write([]byte(`{"books":1,"created":0,"updated":0,"removed":0}`)) + return + } + _, _ = w.Write([]byte(`{"books":1,"created":0,"updated":0,"removed":0,"status":"` + state + `","connections":[{"connection":"work","status":"` + state + `","error_code":"connection_unavailable","error_message":"Connection unavailable","books":0,"created":0,"updated":0,"removed":0}]}`)) + case "/api/v1/carddav/account": + var body map[string]any + if !assertions.NoError(json.NewDecoder(r.Body).Decode(&body)) { + http.Error(w, "invalid synthetic request", http.StatusBadRequest) + return + } + assertions.Equal("work", body["connection"]) + assertions.Equal("google", body["provider"]) + _, _ = w.Write([]byte(`{"base_url":"https://www.googleapis.com/carddav/v1/principals/person@example.com/lists/","username":"person@example.com","enabled":true,"books":1}`)) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(server.Close) + home := t.TempDir() + ctx := withStoreResolverConfig(t, &config.Config{HomeDir: home, Data: config.DataConfig{DataDir: home}, Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) + for _, invocation := range []struct { + cmd *cobra.Command + args []string + failure bool + }{ + {newCardDAVCmd(), []string{"connections"}, false}, + {newCardDAVCmd(), []string{"books", "--connection", "work"}, false}, + {newAddCardDAVCmd(), []string{"--google", "person@example.com", "--connection", "work"}, false}, + {newSyncCardDAVCmd(), []string{"--connection", "work"}, false}, + {newSyncCardDAVCmd(), nil, state != "succeeded"}, + } { + var out, stderr bytes.Buffer + invocation.cmd.SetContext(ctx) + invocation.cmd.SetOut(&out) + invocation.cmd.SetErr(&stderr) + invocation.cmd.SetArgs(invocation.args) + err := invocation.cmd.Execute() + if slices.Equal(invocation.args, []string{"connections"}) { + assertions.Contains(out.String(), "orphaned (restore configuration)") + } + if invocation.failure { + require.Error(err) + assertions.Contains(stderr.String(), "work") + assertions.Contains(stderr.String(), "connection_unavailable") + } else { + require.NoError(err) + } + if len(invocation.args) > 0 && (invocation.args[0] == "books" || invocation.args[0] == "connections") { + assertions.Contains(out.String(), "work") + } + } + assertions.Len(requests, 5) + }) + } +} + +func TestCardDAVCLIRejectsInvalidConnectionBeforeRequest(t *testing.T) { + for _, command := range []*cobra.Command{newAddCardDAVCmd(), newSyncCardDAVCmd(), newCardDAVCmd()} { + args := []string{"--connection", "../invalid"} + switch command.Name() { + case "add-carddav": + args = append(args, "https://contacts.example/dav", "person") + case "carddav": + args = append([]string{"books"}, args...) + } + command.SetOut(&bytes.Buffer{}) + command.SetErr(&bytes.Buffer{}) + command.SetArgs(args) + err := command.Execute() + require.Error(t, err) + assert.ErrorContains(t, err, "connection") + } } diff --git a/cmd/msgvault/cmd/changes_api_e2e_test.go b/cmd/msgvault/cmd/changes_api_e2e_test.go index 25973dbb3..e85fac4b3 100644 --- a/cmd/msgvault/cmd/changes_api_e2e_test.go +++ b/cmd/msgvault/cmd/changes_api_e2e_test.go @@ -69,7 +69,7 @@ func settleChangesFeed(t *testing.T, baseURL string) api.ChangesResponse { "transaction open", start) return page } - time.Sleep(200 * time.Microsecond) + time.Sleep(200 * time.Microsecond) //nolint:kennlint // polls the database clock through the feed } } diff --git a/cmd/msgvault/cmd/circleback.go b/cmd/msgvault/cmd/circleback.go index 5fbde13ee..2b5a36e6a 100644 --- a/cmd/msgvault/cmd/circleback.go +++ b/cmd/msgvault/cmd/circleback.go @@ -7,10 +7,7 @@ import ( "errors" "fmt" "io" - "os" - "os/signal" "strings" - "syscall" "time" "github.com/spf13/cobra" @@ -39,7 +36,10 @@ Then run 'msgvault add-circleback ' to authorize via browser` // resolveCirclebackSource picks the [[circleback]] entry for an optional CLI // argument: an explicit identifier must match a configured entry; with no // argument there must be exactly one entry. -func resolveCirclebackSource(args []string) (*config.CirclebackSource, error) { +func resolveCirclebackSource(args []string, cfg *config.Config) (*config.CirclebackSource, error) { + if cfg == nil { + return nil, errors.New("configuration is unavailable") + } if len(cfg.Circleback) == 0 { return nil, errors.New("no [[circleback]] sources configured\n\n" + circlebackConfigHint) } @@ -61,8 +61,12 @@ func resolveCirclebackSource(args []string) (*config.CirclebackSource, error) { return &src, nil } -func circlebackManager(src *config.CirclebackSource) *circleback.Manager { - return circleback.NewManager(src.Endpoint, cfg.TokensDir(), logger) +func circlebackManager(src *config.CirclebackSource, state *invocation) *circleback.Manager { + state = invocationState(context.Background(), state) + if state == nil || state.cfg == nil { + return circleback.NewManager(src.Endpoint, "", nil) + } + return circleback.NewManager(src.Endpoint, state.cfg.TokensDir(), state.logger) } func newAddCirclebackCmd() *cobra.Command { @@ -84,10 +88,14 @@ func newAddCirclebackCmd() *cobra.Command { // waits on human consent while holding the operation gate (add-teams // pattern). func preflightAddCirclebackAuthorize(cmd *cobra.Command, args []string) error { - if err := validateAddCirclebackOAuthRouting(); err != nil { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + if err := validateAddCirclebackOAuthRouting(state); err != nil { return err } - src, err := resolveCirclebackSource(args) + src, err := resolveCirclebackSource(args, state.cfg) if err != nil { return err } @@ -95,7 +103,7 @@ func preflightAddCirclebackAuthorize(cmd *cobra.Command, args []string) error { return err } fmt.Printf("Authorizing %s with Circleback...\n", src.Identifier) - if err := circlebackManager(src).Authorize(cmd.Context(), src.Identifier); err != nil { + if err := circlebackManager(src, state).Authorize(cmd.Context(), src.Identifier); err != nil { return fmt.Errorf("authorize Circleback: %w", err) } if err := cmd.Flags().Set(oauthPreflightedFlag, "true"); err != nil { @@ -104,8 +112,8 @@ func preflightAddCirclebackAuthorize(cmd *cobra.Command, args []string) error { return nil } -func validateAddCirclebackOAuthRouting() error { - if !IsRemoteMode() { +func validateAddCirclebackOAuthRouting(state *invocation) error { + if !IsRemoteMode(state) { return nil } return errors.New("add-circleback cannot run through a configured remote: the localhost OAuth callback would run on the daemon host; run msgvault add-circleback on the daemon host, or use --local to authorize a local account") @@ -132,7 +140,12 @@ Examples: } func runAddCirclebackLocal(cmd *cobra.Command, args []string) error { - src, err := resolveCirclebackSource(args) + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + src, err := resolveCirclebackSource(args, cfg) if err != nil { return err } @@ -147,12 +160,12 @@ func runAddCirclebackLocal(cmd *cobra.Command, args []string) error { } if !preflighted { fmt.Printf("Authorizing %s with Circleback...\n", src.Identifier) - if err := circlebackManager(src).Authorize(cmd.Context(), src.Identifier); err != nil { + if err := circlebackManager(src, state).Authorize(cmd.Context(), src.Identifier); err != nil { return fmt.Errorf("authorize Circleback: %w", err) } } - s, cleanup, err := openWritableStoreAndInitForIngest() + s, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) if err != nil { return err } @@ -163,7 +176,7 @@ func runAddCirclebackLocal(cmd *cobra.Command, args []string) error { ); err != nil { return err } - if err := runPostSourceCreateMigrations(s); err != nil { + if err := runPostSourceCreateMigrationsForInvocation(s, state); err != nil { return fmt.Errorf("post-source-create migrations: %w", err) } @@ -198,13 +211,18 @@ Examples: msgvault sync-circleback --probe`, Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg if !isDaemonCLISubprocess() { return runDaemonCLICommandHTTPFromCobra(cmd, args) } var sources []config.CirclebackSource if len(args) > 0 || len(cfg.Circleback) == 1 { - src, err := resolveCirclebackSource(args) + src, err := resolveCirclebackSource(args, cfg) if err != nil { return err } @@ -230,26 +248,15 @@ Examples: return probeCircleback(cmd, &src) } - s, cleanup, err := openWritableStoreAndInitForIngest() + s, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) if err != nil { return err } defer cleanup() dbPath := cfg.DatabaseDSN() - ctx, cancel := context.WithCancel(cmd.Context()) - defer cancel() - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) - defer signal.Stop(sigChan) - go func() { - select { - case <-sigChan: - _, _ = fmt.Fprintln(cmd.ErrOrStderr(), "\nInterrupted. Stopping Circleback sync...") - cancel() - case <-ctx.Done(): - } - }() + ctx, stop := withInterruptCancel(cmd, "\nInterrupted. Stopping Circleback sync...") + defer stop() pendingCacheWrites := &circleback.ImportSummary{} for i := range sources { @@ -257,21 +264,21 @@ Examples: accountEmail, err := src.EffectiveAccountEmail() if err != nil { return finishCirclebackImport(ctx, src.Identifier, pendingCacheWrites, err, func() error { - return rebuildCacheAfterWrite(dbPath) + return rebuildCacheAfterManualSync(dbPath, state) }) } if ctx.Err() != nil { return finishCirclebackImport(ctx, src.Identifier, pendingCacheWrites, nil, func() error { - return rebuildCacheAfterWrite(dbPath) + return rebuildCacheAfterManualSync(dbPath, state) }) } _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Syncing Circleback for %s\n\n", src.Identifier) - mgr := circlebackManager(&src) + mgr := circlebackManager(&src, state) session, err := circleback.Connect(ctx, mgr.Endpoint(), mgr.Handler(src.Identifier)) if err != nil { return finishCirclebackImport(ctx, src.Identifier, pendingCacheWrites, err, func() error { - return rebuildCacheAfterWrite(dbPath) + return rebuildCacheAfterManualSync(dbPath, state) }) } imp := circleback.NewImporter(s, session) @@ -289,7 +296,7 @@ Examples: _, _ = fmt.Fprintln(cmd.OutOrStdout(), "\nInterrupted — re-run sync-circleback to resume.") } if finishErr := finishCirclebackImport(ctx, src.Identifier, pendingCacheWrites, err, func() error { - return rebuildCacheAfterWrite(dbPath) + return rebuildCacheAfterManualSync(dbPath, state) }); finishErr != nil { return finishErr } @@ -299,10 +306,10 @@ Examples: if ctx.Err() != nil { return finishCirclebackImport(ctx, sources[len(sources)-1].Identifier, pendingCacheWrites, nil, func() error { - return rebuildCacheAfterWrite(dbPath) + return rebuildCacheAfterManualSync(dbPath, state) }) } - return rebuildCacheAfterWrite(dbPath) + return rebuildCacheAfterManualSync(dbPath, state) }, } @@ -358,7 +365,7 @@ func writeCirclebackSummary(out io.Writer, sum *circleback.ImportSummary) { // probeCircleback prints the MCP tool inventory and one raw SearchMeetings // result so field-name drift can be diagnosed without touching the archive. func probeCircleback(cmd *cobra.Command, src *config.CirclebackSource) error { - mgr := circlebackManager(src) + mgr := circlebackManager(src, invocationFromCommand(cmd)) session, err := circleback.Connect(cmd.Context(), mgr.Endpoint(), mgr.Handler(src.Identifier)) if err != nil { return err @@ -406,6 +413,10 @@ func runCirclebackProbe(ctx context.Context, out io.Writer, session circlebackPr // runConfiguredCirclebackSync is the daemon-scheduler entry point for one // [[circleback]] source. func runConfiguredCirclebackSync(ctx context.Context, st *store.Store, src config.CirclebackSource) error { + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } registered, err := st.ListSources(circleback.SourceType) if err != nil { return fmt.Errorf("list registered Circleback sources: %w", err) @@ -425,7 +436,7 @@ func runConfiguredCirclebackSync(ctx context.Context, st *store.Store, src confi if err != nil { return err } - mgr := circleback.NewManager(src.Endpoint, cfg.TokensDir(), logger) + mgr := circleback.NewManager(src.Endpoint, state.cfg.TokensDir(), state.logger) session, err := circleback.Connect(ctx, mgr.Endpoint(), mgr.Handler(src.Identifier)) if err != nil { return err @@ -466,5 +477,5 @@ func init() { syncCirclebackCmd.Flags().BoolVar(&syncCirclebackFull, "full", false, "ignore the stored creation watermark and re-fetch every meeting (repairs existing rows in place)") syncCirclebackCmd.Flags().BoolVar(&syncCirclebackProbe, "probe", false, "print the MCP tool inventory and a sample result instead of syncing") rootCmd.AddCommand(newAddCirclebackCmd()) - rootCmd.AddCommand(syncCirclebackCmd) + rootCmd.AddCommand(addManualSyncCacheFlags(syncCirclebackCmd)) } diff --git a/cmd/msgvault/cmd/circleback_test.go b/cmd/msgvault/cmd/circleback_test.go index ca3cb74cb..db7be7616 100644 --- a/cmd/msgvault/cmd/circleback_test.go +++ b/cmd/msgvault/cmd/circleback_test.go @@ -256,7 +256,11 @@ func TestConfiguredCirclebackMissingRegisteredSourceStopsBeforeConnect(t *testin require := require.New(t) assert := assert.New(t) st := testutil.NewTestStore(t) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(testInvocationContext( + context.Background(), + config.NewDefaultConfig(), + invocationOptions{}, + )) cancel() err := runConfiguredCirclebackSync(ctx, st, config.CirclebackSource{ @@ -295,15 +299,19 @@ func TestAddCirclebackIdentityConfirmsPrimaryWhenAliasExists(t *testing.T) { } func TestAddCirclebackConfiguredRemoteRejectsHostLocalOAuthBeforeProxy(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) server, requests := newDaemonCLIRunnerTestServer(t, nil, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx cfg.Circleback = []config.CirclebackSource{{ Identifier: "work", AccountEmail: "user-a@example.com", }} cmd := newAddCirclebackCmd() + cmd.SetContext(testCtx) cmd.SetArgs([]string{"work"}) err := cmd.Execute() @@ -316,13 +324,19 @@ func TestAddCirclebackConfiguredRemoteRejectsHostLocalOAuthBeforeProxy(t *testin } func TestAddCirclebackLocalOverrideAllowsHostLocalOAuth(t *testing.T) { + cfg := testConfigValue() + useLocal := false + savedCfg, savedUseLocal := cfg, useLocal t.Cleanup(func() { cfg = savedCfg useLocal = savedUseLocal }) cfg = &config.Config{Remote: config.RemoteConfig{URL: "https://remote.example.com"}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true - require.NoError(t, validateAddCirclebackOAuthRouting()) + require.NoError(t, validateAddCirclebackOAuthRouting(invocationFromContext(testCtx))) } diff --git a/cmd/msgvault/cmd/cliprogress_test.go b/cmd/msgvault/cmd/cliprogress_test.go index 93076924b..842048075 100644 --- a/cmd/msgvault/cmd/cliprogress_test.go +++ b/cmd/msgvault/cmd/cliprogress_test.go @@ -4,6 +4,7 @@ import ( "bytes" "strings" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -27,14 +28,14 @@ func TestCLIProgress_OnProgressBeforeOnStart(t *testing.T) { } func TestCLIProgress_OnStartResetsForReuse(t *testing.T) { - p := &CLIProgress{} - p.OnStart(100) - first := p.startTime - - time.Sleep(5 * time.Millisecond) - p.OnStart(200) - - require.True(t, p.startTime.After(first), "OnStart should reset startTime on subsequent calls") + synctest.Test(t, func(t *testing.T) { + p := &CLIProgress{} + p.OnStart(100) + first := p.startTime + synctest.Sleep(5 * time.Millisecond) + p.OnStart(200) + require.True(t, p.startTime.After(first), "OnStart should reset startTime on subsequent calls") + }) } func TestCLIProgress_PlainModeEmitsNewlineTerminatedUpdates(t *testing.T) { diff --git a/cmd/msgvault/cmd/collection_test.go b/cmd/msgvault/cmd/collection_test.go index f8853b7e6..62e844b6c 100644 --- a/cmd/msgvault/cmd/collection_test.go +++ b/cmd/msgvault/cmd/collection_test.go @@ -2,7 +2,6 @@ package cmd import ( "bytes" - "context" "encoding/json" "net/http" "net/http/httptest" @@ -21,6 +20,9 @@ import ( ) func TestCollectionListUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) dataDir := t.TempDir() @@ -39,11 +41,15 @@ func TestCollectionListUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { Data: config.DataConfig{DataDir: dataDir}, Remote: config.RemoteConfig{URL: "http://configured-daemonclient.invalid"}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true var stdout bytes.Buffer var stderr bytes.Buffer cmd := &cobra.Command{Use: "list", RunE: runCollectionList} + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetErr(&stderr) @@ -59,6 +65,9 @@ func TestCollectionListUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { } func TestCollectionShowUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) dataDir := t.TempDir() @@ -77,7 +86,10 @@ func TestCollectionShowUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { Data: config.DataConfig{DataDir: dataDir}, Remote: config.RemoteConfig{URL: "http://configured-daemonclient.invalid"}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true var stdout bytes.Buffer var stderr bytes.Buffer @@ -86,6 +98,7 @@ func TestCollectionShowUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { Args: collectionShowCmd.Args, RunE: runCollectionShow, } + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetErr(&stderr) cmd.SetArgs([]string{"Team"}) @@ -105,6 +118,9 @@ func TestCollectionShowUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { } func TestCollectionCreateUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) dataDir := t.TempDir() @@ -125,7 +141,10 @@ func TestCollectionCreateUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { Data: config.DataConfig{DataDir: dataDir}, Remote: config.RemoteConfig{URL: "http://configured-daemonclient.invalid"}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true collectionCreateAccounts = "alice@example.com,bob@example.com" var stdout bytes.Buffer @@ -135,6 +154,7 @@ func TestCollectionCreateUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { Args: collectionCreateCmd.Args, RunE: runCollectionCreate, } + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetErr(&stderr) cmd.SetArgs([]string{"Team"}) @@ -148,6 +168,9 @@ func TestCollectionCreateUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { } func TestCollectionAddUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) dataDir := t.TempDir() @@ -168,7 +191,10 @@ func TestCollectionAddUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { Data: config.DataConfig{DataDir: dataDir}, Remote: config.RemoteConfig{URL: "http://configured-daemonclient.invalid"}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true collectionAddAccounts = "alice@example.com,bob@example.com" var stdout bytes.Buffer @@ -178,6 +204,7 @@ func TestCollectionAddUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { Args: collectionAddCmd.Args, RunE: runCollectionAdd, } + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetErr(&stderr) cmd.SetArgs([]string{"Team"}) @@ -191,6 +218,9 @@ func TestCollectionAddUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { } func TestCollectionRemoveUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) dataDir := t.TempDir() @@ -211,7 +241,10 @@ func TestCollectionRemoveUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { Data: config.DataConfig{DataDir: dataDir}, Remote: config.RemoteConfig{URL: "http://configured-daemonclient.invalid"}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true collectionRemoveAccounts = "alice@example.com,bob@example.com" var stdout bytes.Buffer @@ -221,6 +254,7 @@ func TestCollectionRemoveUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { Args: collectionRemoveCmd.Args, RunE: runCollectionRemove, } + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetErr(&stderr) cmd.SetArgs([]string{"Team"}) @@ -234,6 +268,9 @@ func TestCollectionRemoveUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { } func TestCollectionDeleteUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) dataDir := t.TempDir() @@ -252,7 +289,10 @@ func TestCollectionDeleteUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { Data: config.DataConfig{DataDir: dataDir}, Remote: config.RemoteConfig{URL: "http://configured-daemonclient.invalid"}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true var stdout bytes.Buffer var stderr bytes.Buffer @@ -261,6 +301,7 @@ func TestCollectionDeleteUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { Args: collectionDeleteCmd.Args, RunE: runCollectionDelete, } + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetErr(&stderr) cmd.SetArgs([]string{"Team"}) @@ -387,6 +428,9 @@ func collectionHTTPDaemon(t *testing.T) (*httptest.Server, *atomic.Int32) { } func TestCollectionShowPrintsReadableSourceNames(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) savedCfg := cfg @@ -402,7 +446,10 @@ func TestCollectionShowPrintsReadableSourceNames(t *testing.T) { Data: config.DataConfig{DataDir: tmpDir}, Remote: config.RemoteConfig{URL: "http://configured-daemonclient.invalid"}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true dbPath := filepath.Join(tmpDir, "msgvault.db") st, err := store.Open(dbPath) @@ -421,7 +468,8 @@ func TestCollectionShowPrintsReadableSourceNames(t *testing.T) { done := captureStdout(t) cmd := &cobra.Command{} - cmd.SetContext(context.Background()) + cmd.SetContext(testCtx) + cmd.SetContext(testCtx) require.NoError(runCollectionShow(cmd, []string{"team"}), "runCollectionShow") out := done() diff --git a/cmd/msgvault/cmd/compose_draft.go b/cmd/msgvault/cmd/compose_draft.go new file mode 100644 index 000000000..b91b0307c --- /dev/null +++ b/cmd/msgvault/cmd/compose_draft.go @@ -0,0 +1,30 @@ +package cmd + +import "github.com/spf13/cobra" + +func init() { + rootCmd.AddCommand(newDraftComposeCommand()) +} + +func newDraftComposeCommand() *cobra.Command { + command := &cobra.Command{ + Use: "draft-compose", + Short: "Create an email, native Beeper, or local chat draft", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return runDaemonCLICommandHTTPFromCobra(cmd, nil) + }, + } + command.Flags().String("account", "", "source account or display name") + command.Flags().Int64("source-id", 0, "exact source ID") + command.Flags().String(draftFromFlag, "", "confirmed source identity for the draft") + command.Flags().StringArray("to", nil, "recipient address, repeatable; for Beeper, the chat ID") + command.Flags().StringArray("cc", nil, "Cc recipient address, repeatable") + command.Flags().StringArray("bcc", nil, "Bcc recipient address, repeatable") + command.Flags().String("subject", "", "draft subject") + command.Flags().String("body", "", "draft body") + command.Flags().Int64("conversation", 0, "chat conversation for a local draft") + command.Flags().Int64("reply-to", 0, "archived chat message the local draft replies to") + command.Flags().Bool("json", false, "emit one JSON result") + return command +} diff --git a/cmd/msgvault/cmd/constants.go b/cmd/msgvault/cmd/constants.go index 119a9fda7..d6506d18e 100644 --- a/cmd/msgvault/cmd/constants.go +++ b/cmd/msgvault/cmd/constants.go @@ -1,5 +1,7 @@ package cmd +import "go.kenn.io/msgvault/internal/msmail" + // Source-type identifiers stored in sources.source_type and matched against // when dispatching sync/import logic per account kind. const ( @@ -7,13 +9,16 @@ const ( sourceTypeIMAP = "imap" sourceTypeMbox = "mbox" sourceTypeTeams = "teams" + sourceTypeMSMail = msmail.SourceType sourceTypeCalendar = "gcal" sourceTypeBeeper = "beeper" sourceTypeSlack = "slack" sourceTypeSlackdump = "slackdump" sourceTypeGranola = "granola" sourceTypeCircleback = "circleback" + sourceTypePlaud = "plaud" sourceTypeNotionMeetings = "notion_meetings" + sourceTypeMuesli = "muesli" ) // Analytics dataset / SQLite table names: the Parquet subdirectory under diff --git a/cmd/msgvault/cmd/create_subset.go b/cmd/msgvault/cmd/create_subset.go index 1ece31163..f5c44aa44 100644 --- a/cmd/msgvault/cmd/create_subset.go +++ b/cmd/msgvault/cmd/create_subset.go @@ -67,6 +67,11 @@ func init() { } func runCreateSubset(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg if !isDaemonCLISubprocess() { return runDaemonCLICommandHTTPFromCobra(cmd, args) } @@ -94,7 +99,7 @@ func runCreateSubset(cmd *cobra.Command, args []string) error { ) } - release, err := acquireDirectSQLiteWriteLock(cfg) + release, err := acquireDirectSQLiteWriteLock(cfg, state) if err != nil { return err } diff --git a/cmd/msgvault/cmd/create_subset_test.go b/cmd/msgvault/cmd/create_subset_test.go index e7155ecb1..9e142eaa5 100644 --- a/cmd/msgvault/cmd/create_subset_test.go +++ b/cmd/msgvault/cmd/create_subset_test.go @@ -26,7 +26,8 @@ func TestCreateSubsetVCardResourcesRequireFlag(t *testing.T) { dataDir := t.TempDir() testCfg := lifecycleTestConfig(dataDir) - withStoreResolverConfig(t, testCfg) + testCtx := withStoreResolverConfig(t, testCfg) + _ = testCtx t.Setenv(daemonCLISubprocessEnv, strconv.Itoa(os.Getppid())) st, err := store.Open(testCfg.DatabaseDSN()) @@ -91,7 +92,9 @@ func TestCreateSubsetVCardResourcesRequireFlag(t *testing.T) { subsetOutput = filepath.Join(t.TempDir(), "profiles") subsetIncludeProfiles, subsetIncludeVCardResources = true, false profilesStderr := captureStderrDuring(t, func() { - require.NoError(runCreateSubset(&cobra.Command{Use: "create-subset"}, nil)) + cmd := &cobra.Command{Use: "create-subset"} + cmd.SetContext(testCtx) + require.NoError(runCreateSubset(cmd, nil)) }) assert.NotContains(profilesStderr, "--include-vcard-resources", "an unset opt-in must not warn about vCard bodies") @@ -108,7 +111,9 @@ func TestCreateSubsetVCardResourcesRequireFlag(t *testing.T) { subsetOutput = filepath.Join(t.TempDir(), "orphan-resources") subsetIncludeProfiles, subsetIncludeVCardResources = false, true orphanStderr := captureStderrDuring(t, func() { - err := runCreateSubset(&cobra.Command{Use: "create-subset"}, nil) + cmd := &cobra.Command{Use: "create-subset"} + cmd.SetContext(testCtx) + err := runCreateSubset(cmd, nil) require.ErrorContains(err, "--include-vcard-resources requires --include-profiles") }) assert.NotContains(orphanStderr, "WARNING: --include-vcard-resources", @@ -119,7 +124,9 @@ func TestCreateSubsetVCardResourcesRequireFlag(t *testing.T) { subsetOutput = filepath.Join(t.TempDir(), "resources") subsetIncludeProfiles, subsetIncludeVCardResources = true, true resourcesStderr := captureStderrDuring(t, func() { - require.NoError(runCreateSubset(&cobra.Command{Use: "create-subset"}, nil)) + cmd := &cobra.Command{Use: "create-subset"} + cmd.SetContext(testCtx) + require.NoError(runCreateSubset(cmd, nil)) }) assert.Contains(resourcesStderr, "WARNING: --include-vcard-resources", "the opt-in must state what it exposes before copying it") diff --git a/cmd/msgvault/cmd/credentials.go b/cmd/msgvault/cmd/credentials.go new file mode 100644 index 000000000..87ac57f14 --- /dev/null +++ b/cmd/msgvault/cmd/credentials.go @@ -0,0 +1,228 @@ +package cmd + +import ( + "encoding/json/v2" + "errors" + "fmt" + "io" + "os" + "strings" + + "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/personenrichment" + "go.kenn.io/msgvault/internal/providercredentials" +) + +func init() { rootCmd.AddCommand(newCredentialsCommand()) } + +func newCredentialsCommand() *cobra.Command { + command := &cobra.Command{Use: "credentials", Short: "Manage provider credentials on the daemon host"} + var fromFile, endpoint string + var fromStdin, asJSON bool + set := &cobra.Command{ + Use: "set ", Short: "Store a provider key from an owner-only file or standard input", Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := credentialsCommandConfig(cmd) + if err != nil { + return err + } + id := args[0] + if err := validateHeadlessCredentialID(id); err != nil { + return err + } + if (fromFile != "") == fromStdin { + return usageErr(cmd, errors.New("provide exactly one of --from-file or --stdin")) + } + if id == providercredentials.PersonEnrichmentSuppressionID && cmd.Flags().Changed("endpoint") { + return usageErr(cmd, errors.New("suppression keys have no endpoint")) + } + if id != providercredentials.PersonEnrichmentSuppressionID { + if cmd.Flags().Changed("endpoint") { + if endpoint == "" { + return usageErr(cmd, errors.New("--endpoint must not be empty")) + } + } else { + endpoint, err = configuredCredentialEndpoint(cfg, id) + if err != nil { + return err + } + } + } + var key string + if fromStdin { + key, err = providercredentials.ReadSecret(cmd.InOrStdin()) + } else { + key, err = providercredentials.ReadSecretFile(fromFile) + } + if err != nil { + return err + } + snapshot, err := providercredentials.Read(cfg.TokensDir()) + if err != nil { + return err + } + if _, err := putHeadlessCredential(cfg, snapshot, id, endpoint, key); err != nil { + return err + } + _, err = fmt.Fprintf(cmd.OutOrStdout(), "Saved %s\n", id) + if err != nil { + return fmt.Errorf("write saved credential status: %w", err) + } + return nil + }, + } + set.Flags().StringVar(&fromFile, "from-file", "", "Read the key from an owner-only file") + set.Flags().BoolVar(&fromStdin, "stdin", false, "Read the key from standard input") + set.Flags().StringVar(&endpoint, "endpoint", "", "Bind the key to this provider URL (defaults to configured endpoint)") + set.MarkFlagsMutuallyExclusive("from-file", "stdin") + list := &cobra.Command{ + Use: "list", Short: "List stored IDs and bound origins without key values", Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + cfg, err := credentialsCommandConfig(cmd) + if err != nil { + return err + } + snapshot, err := providercredentials.Read(cfg.TokensDir()) + if err != nil { + return err + } + entries := snapshot.Metadata() + if asJSON { + writer := cmd.OutOrStdout() + if err := json.MarshalWrite(writer, entries); err != nil { + return fmt.Errorf("write credential metadata JSON: %w", err) + } + if _, err := io.WriteString(writer, "\n"); err != nil { + return fmt.Errorf("write credential metadata JSON: %w", err) + } + return nil + } + for _, entry := range entries { + if _, err := fmt.Fprintf(cmd.OutOrStdout(), "%s\t%s\n", entry.ID, entry.Origin); err != nil { + return fmt.Errorf("write credential listing: %w", err) + } + } + return nil + }, + } + list.Flags().BoolVar(&asJSON, "json", false, "Print IDs and origins as JSON") + importEnv := &cobra.Command{ + Use: "import-env", Short: "Import present configured provider keys without replacing stored keys", Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + cfg, err := credentialsCommandConfig(cmd) + if err != nil { + return err + } + return importCredentialEnvironment(cmd, cfg) + }, + } + command.AddCommand(set, list, importEnv) + return command +} + +func credentialsCommandConfig(cmd *cobra.Command) (*config.Config, error) { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return nil, errors.New("configuration is unavailable") + } + if isRemoteModeFor(state) { + return nil, errors.New("credentials are stored on the daemon host; run there or pass --local") + } + return state.cfg, nil +} + +func validateHeadlessCredentialID(id string) error { + if id == providercredentials.PeopleSweepID { + return errors.New("people.sweep uses named profile credentials; use msgvault person provider add --api-key-stdin or --credential-env") + } + if id == providercredentials.PersonEnrichmentSuppressionID { + return nil + } + return providercredentials.ValidateID(id) +} + +func configuredCredentialEndpoint(cfg *config.Config, id string) (string, error) { + switch id { + case providercredentials.VectorEmbeddingsID: + return cfg.Vector.Embeddings.Endpoint, nil + case providercredentials.VectorMultimodalID: + return cfg.Vector.Multimodal.Endpoint, nil + } + for _, provider := range cfg.People.Enrichment.Providers { + if providercredentials.PersonEnrichmentID(provider.Name) == id { + provider.ApplyDefaults() + return provider.CredentialEndpoint() + } + } + return "", errors.New("provider is not configured; supply --endpoint") +} + +func putHeadlessCredential(cfg *config.Config, snapshot providercredentials.Snapshot, id, endpoint, key string) (providercredentials.Snapshot, error) { + if id == providercredentials.PersonEnrichmentSuppressionID { + if _, err := personenrichment.NewSuppressionHasher([]byte(key)); err != nil { + return providercredentials.Snapshot{}, fmt.Errorf("validate suppression key: %w", err) + } + return providercredentials.PutSuppression(cfg.TokensDir(), snapshot.ETag, key) + } + return providercredentials.Put(cfg.TokensDir(), snapshot.ETag, id, endpoint, key) +} + +type credentialEnvironment struct{ id, environment string } + +func configuredCredentialEnvironments(cfg *config.Config) []credentialEnvironment { + entries := []credentialEnvironment{ + {providercredentials.VectorEmbeddingsID, cfg.Vector.Embeddings.APIKeyEnv}, + {providercredentials.VectorMultimodalID, cfg.Vector.Multimodal.APIKeyEnv}, + } + if cfg.People.Enrichment.SuppressionKeyEnv != providercredentials.StoredSuppressionEnvironment { + entries = append(entries, credentialEnvironment{providercredentials.PersonEnrichmentSuppressionID, cfg.People.Enrichment.SuppressionKeyEnv}) + } + for _, provider := range cfg.People.Enrichment.Providers { + provider.ApplyDefaults() + entries = append(entries, credentialEnvironment{providercredentials.PersonEnrichmentID(provider.Name), provider.APIKeyEnv}) + } + return entries +} + +func importCredentialEnvironment(cmd *cobra.Command, cfg *config.Config) error { + snapshot, err := providercredentials.Read(cfg.TokensDir()) + if err != nil { + return err + } + imported, kept := 0, 0 + for _, entry := range configuredCredentialEnvironments(cfg) { + if snapshot.Stored(entry.id) { + kept++ + continue + } + if entry.environment == "" { + continue + } + key, ok := os.LookupEnv(entry.environment) + key = strings.TrimSpace(key) + if !ok || key == "" { + continue + } + if len(key) > 64<<10 { + return errors.New("provider environment credential exceeds 64 KiB limit") + } + var endpoint string + if entry.id != providercredentials.PersonEnrichmentSuppressionID { + endpoint, err = configuredCredentialEndpoint(cfg, entry.id) + if err != nil { + return err + } + } + snapshot, err = putHeadlessCredential(cfg, snapshot, entry.id, endpoint, key) + if err != nil { + return err + } + imported++ + } + _, err = fmt.Fprintf(cmd.OutOrStdout(), "Imported %d credentials; kept %d existing credentials\n", imported, kept) + if err != nil { + return fmt.Errorf("write credential import summary: %w", err) + } + return nil +} diff --git a/cmd/msgvault/cmd/credentials_test.go b/cmd/msgvault/cmd/credentials_test.go new file mode 100644 index 000000000..576c6fa7c --- /dev/null +++ b/cmd/msgvault/cmd/credentials_test.go @@ -0,0 +1,272 @@ +package cmd + +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/fileutil" + "go.kenn.io/msgvault/internal/personenrichment" + "go.kenn.io/msgvault/internal/providercredentials" +) + +func runCredentialsTestCommand(t *testing.T, cfg *config.Config, input string, args ...string) (string, error) { + t.Helper() + command := newCredentialsCommand() + command.SetContext(withStoreResolverConfig(t, cfg)) + command.SetIn(strings.NewReader(input)) + var output bytes.Buffer + command.SetOut(&output) + command.SetErr(&output) + command.SetArgs(args) + if err := command.Execute(); err != nil { + return output.String(), fmt.Errorf("execute credentials test command: %w", err) + } + return output.String(), nil +} + +type credentialsTestErrorWriter struct{ err error } + +func (w credentialsTestErrorWriter) Write([]byte) (int, error) { return 0, w.err } + +func runCredentialsTestCommandWithOutput(t *testing.T, cfg *config.Config, input string, output io.Writer, args ...string) error { + t.Helper() + command := newCredentialsCommand() + command.SetContext(withStoreResolverConfig(t, cfg)) + command.SetIn(strings.NewReader(input)) + command.SetOut(output) + command.SetErr(io.Discard) + command.SetArgs(args) + if err := command.Execute(); err != nil { + return fmt.Errorf("execute credentials test command with output: %w", err) + } + return nil +} + +func TestCredentialsCommandsWrapOutputErrors(t *testing.T) { + tests := []struct { + name string + input string + args []string + prepare func(*testing.T, *config.Config) + wantContext string + }{ + { + name: "set status", + input: "provider-key", + args: []string{"set", providercredentials.VectorEmbeddingsID, "--stdin"}, + wantContext: "write saved credential status", + }, + { + name: "list text", + args: []string{"list"}, + prepare: func(t *testing.T, cfg *config.Config) { + t.Helper() + snapshot, err := providercredentials.Read(cfg.TokensDir()) + require.NoError(t, err) + _, err = providercredentials.Put(cfg.TokensDir(), snapshot.ETag, providercredentials.VectorEmbeddingsID, cfg.Vector.Embeddings.Endpoint, "provider-key") + require.NoError(t, err) + }, + wantContext: "write credential listing", + }, + { + name: "list JSON", + args: []string{"list", "--json"}, + prepare: func(t *testing.T, cfg *config.Config) { + t.Helper() + snapshot, err := providercredentials.Read(cfg.TokensDir()) + require.NoError(t, err) + _, err = providercredentials.Put(cfg.TokensDir(), snapshot.ETag, providercredentials.VectorEmbeddingsID, cfg.Vector.Embeddings.Endpoint, "provider-key") + require.NoError(t, err) + }, + wantContext: "write credential metadata JSON", + }, + { + name: "import summary", + args: []string{"import-env"}, + prepare: func(t *testing.T, cfg *config.Config) { + t.Helper() + cfg.Vector.Embeddings.APIKeyEnv = "MSGVAULT_TEST_CREDENTIAL_OUTPUT_KEY" + t.Setenv(cfg.Vector.Embeddings.APIKeyEnv, "provider-key") + }, + wantContext: "write credential import summary", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cfg := credentialsTestConfig(t) + if tc.prepare != nil { + tc.prepare(t, cfg) + } + writeErr := errors.New("output is closed") + err := runCredentialsTestCommandWithOutput(t, cfg, tc.input, credentialsTestErrorWriter{err: writeErr}, tc.args...) + require.ErrorIs(t, err, writeErr) + assert.Contains(t, err.Error(), tc.wantContext) + }) + } +} + +func TestCredentialsNamedProviderAndMalformedStore(t *testing.T) { + t.Parallel() + assert := assert.New(t) + require := require.New(t) + cfg := credentialsTestConfig(t) + cfg.People.Enrichment.Providers = []personenrichment.ProviderConfig{{Name: "research", Kind: personenrichment.ProviderExa, Endpoint: "https://research.example.test/search"}} + _, err := runCredentialsTestCommand(t, cfg, "named-provider-key", "set", providercredentials.PersonEnrichmentID("research"), "--stdin") + require.NoError(err) + key, configured, err := personEnrichmentProviderCredentialLookup(cfg)(personenrichment.ProviderProfile{Name: "research", Kind: personenrichment.ProviderExa, Endpoint: "https://research.example.test/search"}) + require.NoError(err) + assert.True(configured) + assert.Equal("named-provider-key", key) + output, err := runCredentialsTestCommand(t, cfg, "", "list", "--json") + require.NoError(err) + assert.JSONEq(`[{"id":"people.enrichment/research","origin":"https://research.example.test"}]`, output) + assert.NotContains(output, "named-provider-key") + require.NoError(fileutil.SecureWriteFile(filepath.Join(cfg.TokensDir(), providercredentials.Filename), []byte("invalid store"), 0o600)) + _, err = runCredentialsTestCommand(t, cfg, "replacement", "set", providercredentials.VectorEmbeddingsID, "--stdin") + require.Error(err) + contents, err := os.ReadFile(filepath.Join(cfg.TokensDir(), providercredentials.Filename)) + require.NoError(err) + assert.Equal("invalid store", string(contents)) +} + +func credentialsTestConfig(t *testing.T) *config.Config { + t.Helper() + cfg := config.NewDefaultConfig() + cfg.HomeDir = t.TempDir() + cfg.Data.DataDir = cfg.HomeDir + cfg.Vector.Embeddings.Endpoint = "https://embed.example.test/v1" + cfg.Vector.Embeddings.APIKeyEnv = "MSGVAULT_TEST_UNSET_EMBEDDINGS_KEY" + cfg.Vector.Multimodal.APIKeyEnv = "MSGVAULT_TEST_UNSET_MULTIMODAL_KEY" + return cfg +} + +func TestCredentialsSetListAndRuntimePrecedence(t *testing.T) { + t.Parallel() + assert := assert.New(t) + require := require.New(t) + cfg := credentialsTestConfig(t) + file := filepath.Join(t.TempDir(), "key") + require.NoError(fileutil.SecureWriteFile(file, []byte("stored-provider-key\n"), 0o600)) + output, err := runCredentialsTestCommand(t, cfg, "", "set", providercredentials.VectorEmbeddingsID, "--from-file", file) + require.NoError(err) + assert.NotContains(output, "stored-provider-key") + output, err = runCredentialsTestCommand(t, cfg, "", "list") + require.NoError(err) + assert.Contains(output, providercredentials.VectorEmbeddingsID) + assert.Contains(output, "https://embed.example.test") + assert.NotContains(output, "stored-provider-key") + snapshot, err := providercredentials.Read(cfg.TokensDir()) + require.NoError(err) + key, state, err := snapshot.Resolve(providercredentials.VectorEmbeddingsID, cfg.Vector.Embeddings.Endpoint, "LOWER_PRIORITY_ENV", func(string) (string, bool) { return "lower-priority-key", true }) + require.NoError(err) + assert.Equal("stored-provider-key", key) + assert.Equal(providercredentials.SourceStored, state.Source) + _, _, err = snapshot.Resolve(providercredentials.VectorEmbeddingsID, "https://other.example.test/v1", "LOWER_PRIORITY_ENV", func(string) (string, bool) { return "lower-priority-key", true }) + require.ErrorIs(err, providercredentials.ErrOriginMismatch) + _, err = os.Stat(cfg.ConfigFilePath()) + assert.ErrorIs(err, os.ErrNotExist) +} + +func TestCredentialsStdinSuppressionUsesStoredRuntimeValue(t *testing.T) { //nolint:paralleltest // process environment + assert := assert.New(t) + require := require.New(t) + cfg := credentialsTestConfig(t) + cfg.People.Enrichment.SuppressionKeyEnv = "MSGVAULT_TEST_SUPPRESSION_KEY" + t.Setenv(cfg.People.Enrichment.SuppressionKeyEnv, "old-environment-value") + output, err := runCredentialsTestCommand(t, cfg, "stored-suppression-key-32bytes-123\n", "set", providercredentials.PersonEnrichmentSuppressionID, "--stdin") + require.NoError(err) + assert.NotContains(output, "stored-suppression-key") + key, ok := personEnrichmentEnvironmentLookup(cfg)(cfg.People.Enrichment.SuppressionKeyEnv) + require.True(ok) + assert.Equal("stored-suppression-key-32bytes-123", key) +} + +func TestCredentialsValidateSuppressionBeforeSaving(t *testing.T) { //nolint:paralleltest // process environment + for _, args := range [][]string{ + {"set", providercredentials.PersonEnrichmentSuppressionID, "--stdin"}, + {"import-env"}, + } { + t.Run(args[0], func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + cfg := credentialsTestConfig(t) + cfg.People.Enrichment.SuppressionKeyEnv = "MSGVAULT_TEST_SUPPRESSION_KEY" + shortKey := strings.Repeat("s", 31) + t.Setenv(cfg.People.Enrichment.SuppressionKeyEnv, shortKey) + _, err := runCredentialsTestCommand(t, cfg, shortKey, args...) + require.ErrorContains(err, "suppression key must contain at least 32 bytes") + snapshot, err := providercredentials.Read(cfg.TokensDir()) + require.NoError(err) + assert.False(snapshot.Stored(providercredentials.PersonEnrichmentSuppressionID)) + + validKey := strings.Repeat("s", 32) + t.Setenv(cfg.People.Enrichment.SuppressionKeyEnv, validKey) + _, err = runCredentialsTestCommand(t, cfg, validKey, args...) + require.NoError(err) + snapshot, err = providercredentials.Read(cfg.TokensDir()) + require.NoError(err) + key, configured, err := snapshot.ResolveSuppression() + require.NoError(err) + assert.True(configured) + assert.Equal(validKey, key) + }) + } +} + +func TestCredentialsImportEnvironmentPreservesStoredEntries(t *testing.T) { //nolint:paralleltest // process environment + assert := assert.New(t) + require := require.New(t) + cfg := credentialsTestConfig(t) + cfg.Vector.Embeddings.APIKeyEnv = "MSGVAULT_TEST_EMBED_KEY" + cfg.People.Enrichment.SuppressionKeyEnv = "MSGVAULT_TEST_IMPORT_SUPPRESSION" + t.Setenv(cfg.Vector.Embeddings.APIKeyEnv, "initial-environment-key") + t.Setenv(cfg.People.Enrichment.SuppressionKeyEnv, "stable-suppression-key-32bytes-123") + output, err := runCredentialsTestCommand(t, cfg, "", "import-env") + require.NoError(err) + assert.NotContains(output, "initial-environment-key") + snapshot, err := providercredentials.Read(cfg.TokensDir()) + require.NoError(err) + firstETag := snapshot.ETag + t.Setenv(cfg.Vector.Embeddings.APIKeyEnv, "rotated-environment-key") + _, err = runCredentialsTestCommand(t, cfg, "", "import-env") + require.NoError(err) + snapshot, err = providercredentials.Read(cfg.TokensDir()) + require.NoError(err) + assert.Equal(firstETag, snapshot.ETag) + key, _, err := snapshot.Resolve(providercredentials.VectorEmbeddingsID, cfg.Vector.Embeddings.Endpoint, cfg.Vector.Embeddings.APIKeyEnv, os.LookupEnv) + require.NoError(err) + assert.Equal("initial-environment-key", key) +} + +func TestCredentialsRejectInvalidInputsAndUnusedSweepID(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + name, input string + args []string + }{ + {"missing input", "", []string{"set", providercredentials.VectorEmbeddingsID}}, + {"empty input", " \n", []string{"set", providercredentials.VectorEmbeddingsID, "--stdin"}}, + {"oversized input", strings.Repeat("x", 65537), []string{"set", providercredentials.VectorEmbeddingsID, "--stdin"}}, + {"unused sweep store", "key", []string{"set", providercredentials.PeopleSweepID, "--stdin", "--endpoint", "https://provider.example.test"}}, + {"suppression endpoint", "key", []string{"set", providercredentials.PersonEnrichmentSuppressionID, "--stdin", "--endpoint", "https://provider.example.test"}}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + cfg := credentialsTestConfig(t) + _, err := runCredentialsTestCommand(t, cfg, tc.input, tc.args...) + require.Error(t, err) + snapshot, err := providercredentials.Read(cfg.TokensDir()) + require.NoError(t, err) + assert.False(t, snapshot.Stored(providercredentials.VectorEmbeddingsID)) + }) + } +} diff --git a/cmd/msgvault/cmd/daemon_cli_http.go b/cmd/msgvault/cmd/daemon_cli_http.go index 9a834c5b0..024e9f4b3 100644 --- a/cmd/msgvault/cmd/daemon_cli_http.go +++ b/cmd/msgvault/cmd/daemon_cli_http.go @@ -80,7 +80,7 @@ func runDaemonCLICommandHTTPWithEnv( if err != nil { return err } - if api.IsCLIRunDraftReply(args) { + if api.IsCLIRunDraftCreate(args) || api.IsCLIRunDraftLifecycle(args) || api.IsCLIRunDraftSendAs(args) { cwd = "" } @@ -97,7 +97,7 @@ func runDaemonCLICommandHTTPWithEnv( if _, err := fmt.Fprint(cmd.ErrOrStderr(), data); err != nil { return fmt.Errorf("write CLI stderr: %w", err) } - draftFailureReported = api.IsCLIRunDraftReply(args) && data != "" + draftFailureReported = (api.IsCLIRunDraftCreate(args) || api.IsCLIRunDraftLifecycle(args) || api.IsCLIRunDraftSendAs(args)) && data != "" } return nil }) diff --git a/cmd/msgvault/cmd/daemon_cli_http_test.go b/cmd/msgvault/cmd/daemon_cli_http_test.go index 9c4b210ce..f474069e2 100644 --- a/cmd/msgvault/cmd/daemon_cli_http_test.go +++ b/cmd/msgvault/cmd/daemon_cli_http_test.go @@ -302,23 +302,18 @@ func daemonCLIRunTestHandler( } } -func configureRemoteDaemonForTest(t *testing.T, url string) { +func configureRemoteDaemonForTest(t *testing.T, url string, configs ...*config.Config) context.Context { t.Helper() - - savedCfg := cfg - savedUseLocal := useLocal - t.Cleanup(func() { - cfg = savedCfg - useLocal = savedUseLocal - }) - cfg = &config.Config{ - HomeDir: t.TempDir(), - Remote: config.RemoteConfig{ - URL: url, - AllowInsecure: true, - }, + cfg := &config.Config{} + if len(configs) > 0 && configs[0] != nil { + cfg = configs[0] + } + if cfg.HomeDir == "" { + cfg.HomeDir = t.TempDir() } - useLocal = false + cfg.Remote.URL = url + cfg.Remote.AllowInsecure = true + return testInvocationContext(t.Context(), cfg, invocationOptions{}) } func TestDaemonCLIArgsFromCobraForwardsCommandFlagsAndPositionals(t *testing.T) { @@ -369,10 +364,12 @@ func TestRunDaemonCLICommandHTTPOmitsCallerCwdForConfiguredRemote(t *testing.T) assert.Equal([]string{"import-mbox", "alice@example.com", "export.mbox"}, req.Args, "args") assert.Empty(req.Cwd, "configured remote must not receive caller-local cwd") }, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx cmd := &cobra.Command{Use: "import-mbox"} - cmd.SetContext(context.Background()) + cmd.SetContext(testCtx) + cmd.SetContext(testCtx) require.NoError(runDaemonCLICommandHTTPFromCobra(cmd, []string{"alice@example.com", "export.mbox"}), "run daemon command") assert.Equal(1, int(requests.Load()), "runner endpoint calls") } diff --git a/cmd/msgvault/cmd/daemon_cli_subprocess.go b/cmd/msgvault/cmd/daemon_cli_subprocess.go index c2521428c..1f405dd05 100644 --- a/cmd/msgvault/cmd/daemon_cli_subprocess.go +++ b/cmd/msgvault/cmd/daemon_cli_subprocess.go @@ -119,7 +119,7 @@ func newDaemonCLISubprocessCommand(ctx context.Context, commandArgs []string, en if err != nil { return nil, fmt.Errorf("locate msgvault executable: %w", err) } - args := globalConfigFlagArgs() + args := globalConfigFlagArgs(optionsFromContext(ctx)) args = append(args, "--no-log-file") args = append(args, commandArgs...) @@ -127,7 +127,7 @@ func newDaemonCLISubprocessCommand(ctx context.Context, commandArgs []string, en if cwd != "" { cmd.Dir = cwd } - cmd.Env = daemonCLIChildEnv(os.Environ(), os.Getpid(), env) + cmd.Env = daemonRuntimeChildEnv(ctx, daemonCLIChildEnv(os.Environ(), os.Getpid(), env)) cmd.Cancel = func() error { if cmd.Process == nil { return os.ErrProcessDone diff --git a/cmd/msgvault/cmd/daemon_runtime.go b/cmd/msgvault/cmd/daemon_runtime.go index 2f64e1631..6f9707d2e 100644 --- a/cmd/msgvault/cmd/daemon_runtime.go +++ b/cmd/msgvault/cmd/daemon_runtime.go @@ -35,7 +35,7 @@ const ( runtimeCreateTime = "create_time" runtimeShutdownToken = "shutdown_token" runtimeStartupPhase = "startup_phase" - minimumDaemonAPISchemaVersion = "2.14.0" + minimumDaemonAPISchemaVersion = "3.0.0" runtimeStartupCacheBuildOutcome = "startup_cache_build_outcome" daemonProbeTick = 250 * time.Millisecond ) diff --git a/cmd/msgvault/cmd/deduplicate.go b/cmd/msgvault/cmd/deduplicate.go index 2f6709997..afc45e4b5 100644 --- a/cmd/msgvault/cmd/deduplicate.go +++ b/cmd/msgvault/cmd/deduplicate.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "io" + "log/slog" "os" "path/filepath" "slices" @@ -94,6 +95,12 @@ var ( ) func runDeduplicate(cmd *cobra.Command, _ []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger if !isDaemonCLISubprocess() { if deduplicateCanUseDaemonRunner() { return runDaemonCLICommandHTTPFromCobra(cmd, nil) @@ -101,7 +108,7 @@ func runDeduplicate(cmd *cobra.Command, _ []string) error { return runDeduplicateInteractiveHTTP(cmd) } - st, cleanup, err := openWritableStoreAndInit() + st, cleanup, err := openWritableStoreAndInitForInvocation(state) if err != nil { return err } @@ -193,7 +200,7 @@ func runDeduplicate(cmd *cobra.Command, _ []string) error { if len(scope.SourceIDs) == 0 { // Per-source path constructs its own scoped engines per // source, so no top-level engine is needed here. - return runDeduplicatePerSource(cmd, st, dbPath, config) + return runDeduplicatePerSource(cmd, st, dbPath, config, logger) } // Single-account/single-collection path uses one engine shared @@ -465,6 +472,12 @@ func planCLIDeduplicate( st *store.Store, req api.CLIDeduplicatePlanRequest, ) (api.CLIDeduplicatePlanResponse, error) { + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil || state.logger == nil { + return api.CLIDeduplicatePlanResponse{}, errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger preference := deduplicateSourcePreference(req.Prefer, nil) deletionsDir := filepath.Join(cfg.Data.DataDir, "deletions") scope, err := resolveDeduplicateScope(st, deduplicateScopeRequest{ @@ -507,6 +520,11 @@ func planCLIDeduplicatePerSource( st *store.Store, base dedup.Config, ) (api.CLIDeduplicatePlanResponse, error) { + state := invocationFromContext(ctx) + if state == nil || state.logger == nil { + return api.CLIDeduplicatePlanResponse{}, errors.New("configuration is unavailable") + } + logger := state.logger sources, err := st.ListSources("") if err != nil { return api.CLIDeduplicatePlanResponse{}, fmt.Errorf("list sources: %w", err) @@ -698,7 +716,9 @@ func runDeduplicatePerSource( st *store.Store, dbPath string, cfgBase dedup.Config, + logger *slog.Logger, ) error { + logger = repairLogger(logger) sources, err := st.ListSources("") if err != nil { return fmt.Errorf("list sources: %w", err) @@ -1107,7 +1127,7 @@ func init() { "Skip database backup before merging (backup covers pre-dedup state for all sources, not per-batch)") deduplicateCmd.Flags().StringVar(&dedupPrefer, "prefer", "", "Comma-separated source type preference order "+ - "(default: gmail,imap,mbox,emlx,hey)") + "(default: gmail,imap,msmail,mbox,emlx,hey)") deduplicateCmd.Flags().BoolVar(&dedupContentHash, "content-hash", false, "Also detect duplicates by normalized raw MIME content") deduplicateCmd.Flags().StringArrayVar(&dedupUndo, "undo", nil, diff --git a/cmd/msgvault/cmd/deduplicate_test.go b/cmd/msgvault/cmd/deduplicate_test.go index 756860c7d..5b609999c 100644 --- a/cmd/msgvault/cmd/deduplicate_test.go +++ b/cmd/msgvault/cmd/deduplicate_test.go @@ -2,6 +2,7 @@ package cmd import ( "bytes" + "context" "database/sql" "encoding/json" "log/slog" @@ -24,7 +25,7 @@ func TestPlanCLIDeduplicateRequiresConfirmationForDerivableBackfill(t *testing.T require := require.New(t) assert := assert.New(t) f := storetest.New(t) - withDeduplicateTestConfig(t) + testCtx := withDeduplicateTestConfig(t) readyID := f.CreateMessage("ready-metadata") require.NoError(f.Store.UpsertMessageRaw(readyID, []byte( @@ -33,7 +34,7 @@ func TestPlanCLIDeduplicateRequiresConfirmationForDerivableBackfill(t *testing.T require.NoError(f.Store.UpsertMessageRaw(failedID, []byte( "From: sender@example.test\r\nSubject: no identifier\r\n\r\nBody"))) - plan, err := planCLIDeduplicate(t.Context(), f.Store, api.CLIDeduplicatePlanRequest{ + plan, err := planCLIDeduplicate(testCtx, f.Store, api.CLIDeduplicatePlanRequest{ Account: f.Source.Identifier, }) @@ -51,13 +52,13 @@ func TestPlanCLIDeduplicateMalformedOnlyBackfillDoesNotRequireConfirmation(t *te require := require.New(t) assert := assert.New(t) f := storetest.New(t) - withDeduplicateTestConfig(t) + testCtx := withDeduplicateTestConfig(t) messageID := f.CreateMessage("malformed-only") require.NoError(f.Store.UpsertMessageRaw(messageID, []byte( "From: sender@example.test\r\nSubject: no identifier\r\n\r\nBody"))) - plan, err := planCLIDeduplicate(t.Context(), f.Store, api.CLIDeduplicatePlanRequest{ + plan, err := planCLIDeduplicate(testCtx, f.Store, api.CLIDeduplicatePlanRequest{ Account: f.Source.Identifier, }) @@ -201,7 +202,7 @@ func TestDeduplicateSingleAndMultiSourceBackfillOnlyOmitUndo(t *testing.T) { cmd.SetContext(t.Context()) done := captureStdout(t) - err = runDeduplicatePerSource(cmd, f.Store, "", dedup.Config{}) + err = runDeduplicatePerSource(cmd, f.Store, "", dedup.Config{}, testDiscardLogger()) out := done() require.NoError(err) @@ -266,7 +267,7 @@ func TestDeduplicateLocalAndPerSourceMergeOutputIncludesBatch(t *testing.T) { cmd.SetContext(t.Context()) done := captureStdout(t) - err = runDeduplicatePerSource(cmd, f.Store, "", dedup.Config{}) + err = runDeduplicatePerSource(cmd, f.Store, "", dedup.Config{}, testDiscardLogger()) out := done() require.NoError(err) @@ -319,16 +320,21 @@ func TestDeduplicateLocalAndDaemonPromptDescribeDerivationFence(t *testing.T) { serverCfg := config.NewDefaultConfig() serverCfg.Data.DataDir = t.TempDir() apiServer := api.NewServerWithOptions(api.ServerOptions{ - Config: serverCfg, - Store: &storeAPIAdapter{store: f.Store}, + Config: serverCfg, + Store: &storeAPIAdapter{ + store: f.Store, + config: serverCfg, + logger: slog.New(slog.DiscardHandler), + }, Logger: slog.New(slog.DiscardHandler), DaemonVersion: Version, }) httpServer := httptest.NewServer(apiServer.Router()) t.Cleanup(httpServer.Close) - configureRemoteDaemonForTest(t, httpServer.URL) + testCtx := configureRemoteDaemonForTest(t, httpServer.URL) cmd := newDeduplicateRoutingTestCommand() + cmd.SetContext(testCtx) var stdout bytes.Buffer cmd.SetOut(&stdout) cmd.SetIn(strings.NewReader("n\n")) @@ -355,7 +361,7 @@ func TestDeduplicateLocalAndDaemonPromptDescribeDerivationFence(t *testing.T) { cmd.SetIn(strings.NewReader("n\n")) done := captureStdout(t) - err := runDeduplicatePerSource(cmd, f.Store, "", dedup.Config{}) + err := runDeduplicatePerSource(cmd, f.Store, "", dedup.Config{}, testDiscardLogger()) out := done() require.NoError(err) @@ -384,7 +390,7 @@ func TestDeduplicateLocalAndDaemonPromptDescribeDerivationFence(t *testing.T) { cmd.SetIn(strings.NewReader("n\nn\n")) done := captureStdout(t) - err = runDeduplicatePerSource(cmd, f.Store, "", dedup.Config{}) + err = runDeduplicatePerSource(cmd, f.Store, "", dedup.Config{}, testDiscardLogger()) out := done() require.NoError(err) @@ -469,12 +475,12 @@ func TestDeduplicatePlanChangedOutputReportsCommitAndNoBatch(t *testing.T) { assert.Zero(hidden, "plan fence hides no messages") } -func withDeduplicateTestConfig(t *testing.T) { +func withDeduplicateTestConfig(t *testing.T) context.Context { t.Helper() - savedCfg := cfg - t.Cleanup(func() { cfg = savedCfg }) - cfg = config.NewDefaultConfig() + cfg := config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) cfg.Data.DataDir = t.TempDir() + return testCtx } func createPendingRFC822Message( @@ -543,9 +549,10 @@ func TestDeduplicateNonInteractiveFormsUseDaemonRunner(t *testing.T) { server, requests := newDaemonCLIRunnerTestServer(t, func(req daemonCLIRunTestRequest) { assert.Equal(tt.want, req.Args, "args") }, `{"type":"stdout","data":`+string(stdoutJSON)+`}`, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) cmd := newDeduplicateRoutingTestCommand() + cmd.SetContext(testCtx) var stdout bytes.Buffer cmd.SetOut(&stdout) cmd.SetArgs(tt.args) @@ -586,9 +593,11 @@ func TestDeduplicateInteractiveAccountPlansPromptsAndExecutesThroughDaemon(t *te "--yes", }, req.Args, "runner args") }, `{"type":"stdout","data":"Merging duplicates...\n"}`, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx cmd := newDeduplicateRoutingTestCommand() + cmd.SetContext(testCtx) var stdout bytes.Buffer cmd.SetOut(&stdout) cmd.SetIn(strings.NewReader("y\n")) @@ -617,9 +626,11 @@ func TestDeduplicateInteractiveAccountCancelDoesNotExecute(t *testing.T) { }, }, }, nil) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx cmd := newDeduplicateRoutingTestCommand() + cmd.SetContext(testCtx) var stdout bytes.Buffer cmd.SetOut(&stdout) cmd.SetIn(strings.NewReader("n\n")) @@ -660,9 +671,11 @@ func TestDeduplicateInteractivePerSourcePromptsShareInput(t *testing.T) { assert.Contains(req.Args, "--dedup-source-plan=101:fp-alice", "alice approval") assert.Contains(req.Args, "--dedup-source-plan=202:fp-bob", "bob approval") }, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx cmd := newDeduplicateRoutingTestCommand() + cmd.SetContext(testCtx) var stdout bytes.Buffer cmd.SetOut(&stdout) cmd.SetIn(strings.NewReader("y\ny\n")) diff --git a/cmd/msgvault/cmd/delete_deduped_test.go b/cmd/msgvault/cmd/delete_deduped_test.go index d57aedd2f..036068ce9 100644 --- a/cmd/msgvault/cmd/delete_deduped_test.go +++ b/cmd/msgvault/cmd/delete_deduped_test.go @@ -68,6 +68,8 @@ func TestDeleteDeduped_MutualExclusion(t *testing.T) { } func TestDeleteDedupedUsesConfiguredRemoteHTTPAndPreservesOutput(t *testing.T) { + logger := testLoggerValue() + assert := assert.New(t) var planRequests atomic.Int32 @@ -134,7 +136,7 @@ func TestDeleteDedupedUsesConfiguredRemoteHTTPAndPreservesOutput(t *testing.T) { t.Cleanup(server.Close) dataDir := t.TempDir() - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ HomeDir: dataDir, Data: config.DataConfig{DataDir: dataDir}, Remote: config.RemoteConfig{ @@ -164,6 +166,7 @@ func TestDeleteDedupedUsesConfiguredRemoteHTTPAndPreservesOutput(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer cmd := &cobra.Command{Use: "delete-deduped", RunE: runDeleteDeduped} + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetErr(&stderr) diff --git a/cmd/msgvault/cmd/deletion_readonly_scope_test.go b/cmd/msgvault/cmd/deletion_readonly_scope_test.go index ca01efd78..c0035cd16 100644 --- a/cmd/msgvault/cmd/deletion_readonly_scope_test.go +++ b/cmd/msgvault/cmd/deletion_readonly_scope_test.go @@ -1,10 +1,12 @@ package cmd import ( + "path/filepath" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/oauth" "go.kenn.io/msgvault/internal/store" @@ -72,12 +74,23 @@ func TestDeleteStagedScopeEscalationForSource_ReadonlyWorld(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, restore := seedTokenEnv(t, tt.tokenJSON) + tokenPath, restore := seedTokenEnv(t, tt.tokenJSON) defer restore() + configHome := filepath.Dir(filepath.Dir(tokenPath)) + cfg := config.NewDefaultConfig() + cfg.HomeDir = configHome + cfg.Data.DataDir = configHome + cfg.OAuth.ClientSecrets = filepath.Join(configHome, "client_secret.json") + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) src := &store.Source{SourceType: sourceTypeGmail} escalation, err := deleteStagedScopeEscalationForSource( - scopeEscalationAccount, src, tt.permanent, cfg.OAuth.ClientSecrets) + scopeEscalationAccount, + src, + tt.permanent, + cfg.OAuth.ClientSecrets, + invocationFromContext(testCtx), + ) require.NoError(t, err) assert.Equal(t, tt.wantPrompt, escalation.Needed, tt.description) diff --git a/cmd/msgvault/cmd/deletions.go b/cmd/msgvault/cmd/deletions.go index 225fe90f6..d7260533f 100644 --- a/cmd/msgvault/cmd/deletions.go +++ b/cmd/msgvault/cmd/deletions.go @@ -22,8 +22,12 @@ import ( "github.com/spf13/cobra" "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/daemonclient" "go.kenn.io/msgvault/internal/deletion" + "go.kenn.io/msgvault/internal/gmail" + "go.kenn.io/msgvault/internal/msgraph" + "go.kenn.io/msgvault/internal/msmail" "go.kenn.io/msgvault/internal/oauth" "go.kenn.io/msgvault/internal/sourceops" "go.kenn.io/msgvault/internal/store" @@ -44,6 +48,11 @@ full, untruncated batch IDs suitable for show-deletion and delete-staged.`, } func runListDeletions(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg if !isDaemonCLISubprocess() { return runDaemonCLICommandHTTPFromCobra(cmd, args) } @@ -148,6 +157,11 @@ var showDeletionCmd = &cobra.Command{ } func runShowDeletion(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg batchID := strings.TrimSpace(args[0]) if batchID == "" { return errors.New("batch ID is required") @@ -186,6 +200,11 @@ Examples: } func runCancelDeletion(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg if cancelAll && len(args) > 0 { return usageErr(cmd, errors.New("cannot use --all with a batch ID argument")) } @@ -280,6 +299,7 @@ var ( // remote rung should be too unless the user explicitly says // otherwise. deletePermanent bool + deleteHeadless bool deleteYes bool deleteDryRun bool deleteList bool @@ -309,12 +329,17 @@ const ( // unconditionally; only the destructive source-server call is gated. const remoteDeleteEnvVar = "MSGVAULT_ENABLE_REMOTE_DELETE" -func remoteDeleteEnabled(daemonSubprocess bool) bool { +func remoteDeleteEnabled(daemonSubprocess bool, state *invocation) bool { + var currentCfg *config.Config + if state != nil { + currentCfg = state.cfg + } return os.Getenv(remoteDeleteEnvVar) == "1" || - (!daemonSubprocess && cfg != nil && cfg.Deletion.RemoteEnabled) + (!daemonSubprocess && currentCfg != nil && currentCfg.Deletion.RemoteEnabled) } type deleteStagedPlanOptions struct { + Invocation *invocation BatchID string PlannedBatchIDs []string Permanent bool @@ -344,12 +369,17 @@ type deleteStagedPlan struct { ScopeEscalationBodyLines []string ScopeEscalationCancelHint string ScopeEscalationAccount string + ScopeEscalationSourceType string ScopeEscalationOAuthApp string BlockedError string RemoteDeleteEnvVar string } func buildDeleteStagedPlan(opts deleteStagedPlanOptions) (deleteStagedPlan, error) { + if opts.Invocation == nil || opts.Invocation.cfg == nil { + return deleteStagedPlan{}, errors.New("configuration is unavailable") + } + cfg := opts.Invocation.cfg if opts.SourceIDSet && opts.SourceID <= 0 { return deleteStagedPlan{}, newDeleteStagedUsageError(errors.New("source ID must be positive")) } @@ -456,7 +486,7 @@ func buildDeleteStagedPlan(opts deleteStagedPlanOptions) (deleteStagedPlan, erro // scopes requested by the caller — is derived from opts.Permanent, while // execution honors the method a resumed batch was started with. Refuse // rather than let those disagree: resuming a permanent-delete batch - // without --permanent would print "trash (30-day recovery)" and take a + // without --permanent would print "trash (recoverable)" and take a // trash confirmation for what is actually an unrecoverable deletion. if err := assertDeleteStagedMethodMatchesFlag(manifests, opts.Permanent); err != nil { return deleteStagedPlan{}, err @@ -467,7 +497,7 @@ func buildDeleteStagedPlan(opts deleteStagedPlanOptions) (deleteStagedPlan, erro totalMessages += len(m.GmailIDs) } - method := "trash (30-day recovery)" + method := "trash (recoverable)" if opts.Permanent { method = "PERMANENT DELETE (fast, no recovery)" } @@ -702,8 +732,8 @@ func resolveDeleteStagedTargetWithSourceID( if err != nil { return deleteStagedTarget{}, err } - if source.SourceType != sourceTypeGmail && source.SourceType != sourceTypeIMAP { - return deleteStagedTarget{}, fmt.Errorf("source %d is not a gmail or imap source", source.ID) + if !canDeleteAtSource(source.SourceType) { + return deleteStagedTarget{}, fmt.Errorf("source %d is not a gmail, imap or msmail source", source.ID) } if requestedSourceIDSet { selected, selectErr := sourceops.ResolveExactOne(st, sourceops.Selector{ @@ -758,8 +788,8 @@ func resolveDeleteStagedTargetWithSourceID( if err != nil { return deleteStagedTarget{}, err } - if source.SourceType != sourceTypeGmail && source.SourceType != sourceTypeIMAP { - return deleteStagedTarget{}, fmt.Errorf("source %d is not a gmail or imap source", source.ID) + if !canDeleteAtSource(source.SourceType) { + return deleteStagedTarget{}, fmt.Errorf("source %d is not a gmail, imap or msmail source", source.ID) } for _, manifest := range manifests { if manifest.Filters.Account != "" && manifest.Filters.Account != source.Identifier { @@ -770,6 +800,11 @@ func resolveDeleteStagedTargetWithSourceID( return deleteStagedTarget{Account: source.Identifier, Source: source}, nil } +// canDeleteAtSource reports whether delete-staged has a client for a source type. +func canDeleteAtSource(sourceType string) bool { + return sourceType == sourceTypeGmail || sourceType == sourceTypeIMAP || sourceType == sourceTypeMSMail +} + type deleteStagedScopeEscalation struct { Needed bool Account string @@ -785,6 +820,7 @@ func deleteStagedScopeEscalationForSource( src *store.Source, permanent bool, clientSecretsPath string, + state *invocation, ) (deleteStagedScopeEscalation, error) { if src == nil || src.SourceType != sourceTypeGmail { return deleteStagedScopeEscalation{}, nil @@ -793,9 +829,13 @@ func deleteStagedScopeEscalationForSource( if permanent { requiredScopes = oauth.ScopesDeletion } - oauthMgr, err := oauth.NewManagerWithScopes(clientSecretsPath, cfg.TokensDir(), logger, requiredScopes) + state = invocationState(context.Background(), state) + if state == nil || state.cfg == nil || state.logger == nil { + return deleteStagedScopeEscalation{}, errors.New("configuration is unavailable") + } + oauthMgr, err := oauth.NewManagerWithScopes(clientSecretsPath, state.cfg.TokensDir(), state.logger, requiredScopes) if err != nil { - return deleteStagedScopeEscalation{}, wrapOAuthError(fmt.Errorf("create oauth manager: %w", err)) + return deleteStagedScopeEscalation{}, wrapOAuthError(fmt.Errorf("create oauth manager: %w", err), state.cfg) } if !oauthMgr.HasScopeMetadata(account) { if permanent && oauthMgr.HasToken(account) { @@ -809,6 +849,34 @@ func deleteStagedScopeEscalationForSource( return newDeleteStagedScopeEscalation(account, permanent, clientSecretsPath), nil } +// msmailScopeEscalation reports whether a Graph mail token lacks +// Mail.ReadWrite. Trash and permanent delete need the same scope. +func msmailScopeEscalation(account string, state *invocation) (deleteStagedScopeEscalation, error) { + if state == nil || state.cfg == nil || state.logger == nil { + return deleteStagedScopeEscalation{}, errors.New("configuration is unavailable") + } + ok, err := newGraphMailWriteManager(state).HasScopes(account) + if err != nil { + return deleteStagedScopeEscalation{}, fmt.Errorf("read Microsoft Graph mail token for %s: %w", account, err) + } + if ok { + return deleteStagedScopeEscalation{}, nil + } + return deleteStagedScopeEscalation{ + Needed: true, + Account: account, + Headline: deleteStagedScopeEscalationHeadline, + BodyLines: []string{ + "Deletion requires the Microsoft Graph Mail.ReadWrite permission.", + "", + "Your current token can only read mail. Sign in to grant read and", + "write access to this mailbox. Your existing token keeps working", + "until the new grant succeeds.", + }, + CancelHint: "Cancelled.", + }, nil +} + // grantCoversDeletion reports whether an account's granted scopes already // permit the requested deletion, so no re-consent prompt is needed. // @@ -853,8 +921,9 @@ var deleteStagedCmd = &cobra.Command{ Short: "Execute staged deletions", Long: `Execute pending deletion batches. -By default, messages are moved to Gmail trash (recoverable for 30 days). -Use --permanent for batch-API permanent deletion (fast, no recovery). +By default, messages are moved to the trash: Gmail Trash (recoverable for +30 days), or Deleted Items for Microsoft Graph mail. +Use --permanent for permanent deletion (no recovery). The default is trash because every other rung of the deletion progression in msgvault is locally reversible; the remote rung is too unless the user explicitly opts out of recoverability. @@ -877,8 +946,14 @@ Examples: msgvault delete-staged # With durable config consent msgvault delete-staged batch-123 # With durable config consent msgvault delete-staged --permanent # With durable config consent - MSGVAULT_ENABLE_REMOTE_DELETE=1 msgvault delete-staged --yes # One command`, + MSGVAULT_ENABLE_REMOTE_DELETE=1 msgvault delete-staged --yes # One command`, RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger daemonSubprocess := isDaemonCLISubprocess() if !daemonSubprocess { return runDeleteStagedHTTP(cmd, args) @@ -905,6 +980,7 @@ Examples: batchID = args[0] } plan, err := buildDeleteStagedPlan(deleteStagedPlanOptions{ + Invocation: state, BatchID: batchID, PlannedBatchIDs: deletePlannedBatchIDs, Permanent: deletePermanent, @@ -914,7 +990,7 @@ Examples: Account: deleteAccount, SourceID: deleteSourceID, SourceIDSet: cmd.Flags().Changed("source-id"), - RemoteDeleteEnabled: remoteDeleteEnabled(daemonSubprocess), + RemoteDeleteEnabled: remoteDeleteEnabled(daemonSubprocess, invocationFromCommand(cmd)), }) if err != nil { return err @@ -940,7 +1016,7 @@ Examples: manager := plan.Manager manifests := plan.Manifests - release, err := acquireDirectSQLiteWriteLock(cfg) + release, err := acquireDirectSQLiteWriteLock(cfg, state) if err != nil { return err } @@ -957,7 +1033,7 @@ Examples: if err := s.InitSchema(); err != nil { return fmt.Errorf("init schema: %w", err) } - if err := runStartupMigrations(s); err != nil { + if err := runStartupMigrationsContext(cmd.Context(), s, state); err != nil { return fmt.Errorf("startup migrations: %w", err) } // Resolve the target before any durable claim. The digest-checked claim @@ -996,7 +1072,7 @@ Examples: var clientSecretsPath string if src.SourceType == sourceTypeGmail { if !cfg.OAuth.HasAnyConfig() { - return errOAuthNotConfigured() + return errOAuthNotConfigured(cfg) } appName := sourceOAuthApp(src) isServiceAccount := cfg.OAuth.ServiceAccountKeyFor(appName) != "" @@ -1007,7 +1083,7 @@ Examples: return err } - escalation, err := deleteStagedScopeEscalationForSource(account, src, deletePermanent, clientSecretsPath) + escalation, err := deleteStagedScopeEscalationForSource(account, src, deletePermanent, clientSecretsPath, state) if err != nil { return err } @@ -1028,6 +1104,25 @@ Examples: } } + if src.SourceType == sourceTypeMSMail { + escalation, err := msmailScopeEscalation(account, state) + if err != nil { + return err + } + if escalation.Needed { + if !scopeEscalationConfirmed { + ok, err := promptScopeEscalationConfirmation(os.Stdin, os.Stdout, + escalation.Headline, escalation.BodyLines, escalation.CancelHint) + if err != nil || !ok { + return err + } + } + if err := authorizeGraphMailWrite(ctx, account, state); err != nil { + return err + } + } + } + // Build API client — reuses the same factory as sync. getOAuthMgr := func(appName string) (*oauth.Manager, error) { secretsPath := clientSecretsPath @@ -1050,11 +1145,21 @@ Examples: if deletePermanent { saScopes = oauth.ScopesDeletion } - client, err := buildAPIClient(ctx, src, getOAuthMgr, saScopes) - if err != nil { - return err + var client gmail.MessageDeleter + if src.SourceType == sourceTypeMSMail { + tokenFn, err := newGraphMailWriteManager(state).TokenSource(ctx, account) + if err != nil { + return err + } + client = msmail.NewClient(msmail.GraphBaseURL, tokenFn, msmailQPS) + } else { + apiClient, err := buildAPIClient(ctx, src, getOAuthMgr, saScopes) + if err != nil { + return err + } + defer func() { _ = apiClient.Close() }() + client = apiClient } - defer func() { _ = client.Close() }() // Create executor executor := deletion.NewExecutor(manager, s, client). @@ -1117,6 +1222,14 @@ Examples: continue } + if src.SourceType == sourceTypeMSMail && isInsufficientScopeError(execErr) { + return fmt.Errorf( + "graph mail delete for %s was denied: add the delegated Mail.ReadWrite "+ + "permission to the app registration, grant consent, then run delete-staged again: %w", + account, execErr, + ) + } + // Check if this is a scope error - offer to re-authorize (Gmail only) if src.SourceType == sourceTypeGmail && isInsufficientScopeError(execErr) { if cfg.OAuth.ServiceAccountKeyFor(sourceOAuthApp(src)) != "" { @@ -1145,7 +1258,7 @@ Examples: fmt.Println("\nDeletion complete!") return nil }, func() error { - return rebuildCacheAfterWrite(dbPath) + return rebuildCacheAfterWrite(dbPath, state) }) }, } @@ -1164,7 +1277,7 @@ func runDeleteStagedHTTP(cmd *cobra.Command, args []string) error { if len(args) > 0 { batchID = args[0] } - remoteDeleteAllowed := remoteDeleteEnabled(false) + remoteDeleteAllowed := remoteDeleteEnabled(false, invocationFromCommand(cmd)) st, _, err := OpenHTTPStore(cmd.Context()) if err != nil { return err @@ -1261,7 +1374,7 @@ func deleteStagedSourceIDPtr(cmd *cobra.Command) *int64 { return &value } -// preflightDeleteStagedScopeEscalation performs the confirmed Gmail scope +// preflightDeleteStagedScopeEscalation performs the confirmed Gmail or Graph scope // upgrade in this process for local daemons, so the browser consent never // runs in the daemon subprocess while it holds the operation gate. After a // successful preflight the subprocess re-checks the token, finds the scopes @@ -1269,16 +1382,34 @@ func deleteStagedSourceIDPtr(cmd *cobra.Command) *int64 { // subprocess-side flow because tokens live on that host, as do older daemons // whose plan response does not name the escalation account. func preflightDeleteStagedScopeEscalation(ctx context.Context, plan *daemonclient.CLIDeleteStagedPlan) error { - if IsRemoteMode() || plan.ScopeEscalationAccount == "" { + state := invocationFromContext(ctx) + if IsRemoteMode(state) || plan.ScopeEscalationAccount == "" { return nil } - clientSecretsPath, err := cfg.OAuth.ClientSecretsFor(plan.ScopeEscalationOAuthApp) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + if plan.ScopeEscalationSourceType == sourceTypeMSMail { + return authorizeGraphMailWrite(ctx, plan.ScopeEscalationAccount, state) + } + clientSecretsPath, err := state.cfg.OAuth.ClientSecretsFor(plan.ScopeEscalationOAuthApp) if err != nil { return err } return authorizeDeletionScopeEscalation(ctx, plan.ScopeEscalationAccount, deletePermanent, clientSecretsPath) } +func authorizeGraphMailWrite(ctx context.Context, account string, state *invocation) error { + mgr := newGraphMailWriteManager(state) + if deleteHeadless { + mgr.UseDeviceCode() + } + if err := mgr.Authorize(ctx, account); err != nil { + return fmt.Errorf("authorize Microsoft Graph mail: %w", err) + } + return nil +} + func confirmDeleteStaged(in io.Reader, out io.Writer, mode string) (bool, error) { reader := stagedDeletePromptReader(in) switch mode { @@ -1294,7 +1425,7 @@ func confirmDeleteStaged(in io.Reader, out io.Writer, mode string) (bool, error) } return true, nil case deleteStagedConfirmModeTrash: - _, _ = fmt.Fprint(out, "Proceed with deletion? Messages move to Gmail/Trash (recoverable ~30 days). [y/N]: ") + _, _ = fmt.Fprint(out, "Proceed with deletion? Messages move to the trash, where you can restore them. [y/N]: ") answer, ok, err := readStagedDeletePromptLine(reader) if err != nil { return false, fmt.Errorf("read confirmation: %w", err) @@ -1334,10 +1465,15 @@ func readStagedDeletePromptLine(reader *bufio.Reader) (string, bool, error) { } func planCLIDeleteStaged( - _ context.Context, + ctx context.Context, st *store.Store, req api.CLIDeleteStagedPlanRequest, ) (api.CLIDeleteStagedPlanResponse, error) { + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil || state.logger == nil { + return api.CLIDeleteStagedPlanResponse{}, errors.New("configuration is unavailable") + } + cfg := state.cfg var resolvedSource *store.Source if req.SourceID != nil { resolved, err := sourceops.ResolveExactOne(st, sourceops.Selector{ @@ -1363,6 +1499,7 @@ func planCLIDeleteStaged( resolvedSourceIdentifier = resolvedSource.Identifier } plan, err := buildDeleteStagedPlan(deleteStagedPlanOptions{ + Invocation: state, BatchID: req.BatchID, Permanent: req.Permanent, Yes: req.Yes, @@ -1384,9 +1521,23 @@ func planCLIDeleteStaged( if err != nil { return api.CLIDeleteStagedPlanResponse{}, err } + if target.Source.SourceType == sourceTypeMSMail { + escalation, err := msmailScopeEscalation(target.Account, state) + if err != nil { + return api.CLIDeleteStagedPlanResponse{}, err + } + if escalation.Needed { + plan.NeedsScopeEscalation = true + plan.ScopeEscalationHeadline = escalation.Headline + plan.ScopeEscalationBodyLines = escalation.BodyLines + plan.ScopeEscalationCancelHint = escalation.CancelHint + plan.ScopeEscalationAccount = escalation.Account + plan.ScopeEscalationSourceType = target.Source.SourceType + } + } if target.Source.SourceType == sourceTypeGmail { if !cfg.OAuth.HasAnyConfig() { - return api.CLIDeleteStagedPlanResponse{}, errOAuthNotConfigured() + return api.CLIDeleteStagedPlanResponse{}, errOAuthNotConfigured(cfg) } appName := sourceOAuthApp(target.Source) if cfg.OAuth.ServiceAccountKeyFor(appName) == "" { @@ -1394,7 +1545,7 @@ func planCLIDeleteStaged( if err != nil { return api.CLIDeleteStagedPlanResponse{}, err } - escalation, err := deleteStagedScopeEscalationForSource(target.Account, target.Source, req.Permanent, clientSecretsPath) + escalation, err := deleteStagedScopeEscalationForSource(target.Account, target.Source, req.Permanent, clientSecretsPath, state) if err != nil { return api.CLIDeleteStagedPlanResponse{}, err } @@ -1404,6 +1555,7 @@ func planCLIDeleteStaged( plan.ScopeEscalationBodyLines = escalation.BodyLines plan.ScopeEscalationCancelHint = escalation.CancelHint plan.ScopeEscalationAccount = escalation.Account + plan.ScopeEscalationSourceType = target.Source.SourceType plan.ScopeEscalationOAuthApp = appName } } @@ -1426,6 +1578,7 @@ func planCLIDeleteStaged( ScopeEscalationBodyLines: plan.ScopeEscalationBodyLines, ScopeEscalationCancelHint: plan.ScopeEscalationCancelHint, ScopeEscalationAccount: plan.ScopeEscalationAccount, + ScopeEscalationSourceType: plan.ScopeEscalationSourceType, ScopeEscalationOAuthApp: plan.ScopeEscalationOAuthApp, BlockedError: plan.BlockedError, RemoteDeleteEnvVar: plan.RemoteDeleteEnvVar, @@ -1631,6 +1784,10 @@ func authorizeScopeEscalation( requiredScopes []string, clientSecretsPath string, ) error { + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil || state.logger == nil { + return errors.New("configuration is unavailable") + } // Re-authorize with the upgraded scope set. We deliberately do NOT delete // the existing token first: Authorize overwrites it atomically only after a // successful, validated grant, so the old token survives a cancelled or @@ -1638,7 +1795,7 @@ func authorizeScopeEscalation( fmt.Println("\nStarting OAuth flow...") fmt.Println() - newMgr, err := oauth.NewManagerWithScopes(clientSecretsPath, cfg.TokensDir(), logger, requiredScopes) + newMgr, err := oauth.NewManagerWithScopes(clientSecretsPath, state.cfg.TokensDir(), state.logger, requiredScopes) if err != nil { return fmt.Errorf("create oauth manager: %w", err) } @@ -1654,7 +1811,7 @@ func authorizeScopeEscalation( // promptDeletionScopeEscalation is the deletion-specific wrapper that maps the // batchDelete bool to the right scopes/copy and delegates to the generic helper. func promptDeletionScopeEscalation(ctx context.Context, account string, batchDelete bool, clientSecretsPath string) error { - requiredScopes, err := deletionEscalationScopesForAccount(account, batchDelete, clientSecretsPath) + requiredScopes, err := deletionEscalationScopesForAccountWithState(ctx, account, batchDelete, clientSecretsPath) if err != nil { return err } @@ -1664,7 +1821,7 @@ func promptDeletionScopeEscalation(ctx context.Context, account string, batchDel } func authorizeDeletionScopeEscalation(ctx context.Context, account string, batchDelete bool, clientSecretsPath string) error { - requiredScopes, err := deletionEscalationScopesForAccount(account, batchDelete, clientSecretsPath) + requiredScopes, err := deletionEscalationScopesForAccountWithState(ctx, account, batchDelete, clientSecretsPath) if err != nil { return err } @@ -1694,8 +1851,20 @@ func deletionScopeEscalationPrompt(batchDelete bool) ([]string, string) { }, "Cancelled. Drop --permanent to use trash deletion without elevated permissions." } -func deletionEscalationScopesForAccount(account string, batchDelete bool, clientSecretsPath string) ([]string, error) { - mgr, err := oauth.NewManagerWithScopes(clientSecretsPath, cfg.TokensDir(), logger, oauth.ScopesGmailCalendar) +func deletionEscalationScopesForAccount(account string, batchDelete bool, clientSecretsPath string, state *invocation) ([]string, error) { + ctx := context.Background() + if state != nil { + ctx = withInvocation(ctx, state) + } + return deletionEscalationScopesForAccountWithState(ctx, account, batchDelete, clientSecretsPath) +} + +func deletionEscalationScopesForAccountWithState(ctx context.Context, account string, batchDelete bool, clientSecretsPath string) ([]string, error) { + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil || state.logger == nil { + return nil, errors.New("configuration is unavailable") + } + mgr, err := oauth.NewManagerWithScopes(clientSecretsPath, state.cfg.TokensDir(), state.logger, oauth.ScopesGmailCalendar) if err != nil { return nil, fmt.Errorf("create oauth manager: %w", err) } @@ -1729,15 +1898,17 @@ func isInsufficientScopeError(err error) bool { msg := err.Error() return strings.Contains(msg, "ACCESS_TOKEN_SCOPE_INSUFFICIENT") || strings.Contains(msg, "insufficient authentication scopes") || - strings.Contains(msg, "Insufficient Permission") + strings.Contains(msg, "Insufficient Permission") || + errors.Is(err, msgraph.ErrForbidden) // Graph mail without Mail.ReadWrite } func init() { + deleteStagedCmd.Flags().BoolVar(&deleteHeadless, "headless", false, "Use device-code sign-in for Microsoft Graph permission upgrades") deleteStagedCmd.Flags().BoolVar(&deletePermanent, "permanent", false, "DESTRUCTIVE: permanently delete via batch API instead of moving to trash (fast, no recovery)") deleteStagedCmd.Flags().BoolVarP(&deleteYes, "yes", "y", false, "Skip confirmation") deleteStagedCmd.Flags().BoolVar(&deleteDryRun, "dry-run", false, "Show what would be deleted") deleteStagedCmd.Flags().BoolVarP(&deleteList, "list", "l", false, "List staged batches without executing") - deleteStagedCmd.Flags().StringVar(&deleteAccount, "account", "", "Account to use (Gmail or IMAP)") + deleteStagedCmd.Flags().StringVar(&deleteAccount, "account", "", "Account to use (Gmail, IMAP or Microsoft Graph mail)") deleteStagedCmd.Flags().Int64Var(&deleteSourceID, "source-id", 0, "Exact source ID to use") deleteStagedCmd.Flags().Bool(deleteStagedConfirmedFlag, false, "Internal confirmation marker") deleteStagedCmd.Flags().Bool(deleteStagedSkipPreludeFlag, false, "Internal planning marker") diff --git a/cmd/msgvault/cmd/deletions_api_e2e_test.go b/cmd/msgvault/cmd/deletions_api_e2e_test.go index 2d7a3638e..ce0190ec9 100644 --- a/cmd/msgvault/cmd/deletions_api_e2e_test.go +++ b/cmd/msgvault/cmd/deletions_api_e2e_test.go @@ -2,7 +2,6 @@ package cmd import ( "bytes" - "context" "database/sql" "encoding/json" "log/slog" @@ -19,6 +18,8 @@ import ( ) func TestDeletionStagingEndToEnd(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) @@ -26,6 +27,8 @@ func TestDeletionStagingEndToEnd(t *testing.T) { savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = &config.Config{Data: config.DataConfig{DataDir: tmpDir}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx s, err := store.Open(tmpDir + "/msgvault.db") require.NoError(err, "open store") @@ -48,8 +51,8 @@ func TestDeletionStagingEndToEnd(t *testing.T) { t.Cleanup(func() { _ = engine.Close() }) srv := api.NewServerWithOptions(api.ServerOptions{ - Config: &config.Config{Data: config.DataConfig{DataDir: tmpDir}}, - Store: &storeAPIAdapter{store: s}, + Config: cfg, + Store: &storeAPIAdapter{store: s, config: cfg}, Engine: engine, Logger: slog.New(slog.DiscardHandler), }) @@ -74,8 +77,8 @@ func TestDeletionStagingEndToEnd(t *testing.T) { // The persisted manifest must carry the account delete-staged // executes against. - adapter := &storeAPIAdapter{store: s} - persisted, _, err := adapter.GetDeletionManifest(context.Background(), staged.ID) + adapter := &storeAPIAdapter{store: s, config: cfg} + persisted, _, err := adapter.GetDeletionManifest(testCtx, staged.ID) require.NoError(err, "load persisted manifest") assert.Equal("alice@example.com", persisted.Filters.Account, "manifest account") diff --git a/cmd/msgvault/cmd/deletions_routing_test.go b/cmd/msgvault/cmd/deletions_routing_test.go index a924461d6..d84aecd82 100644 --- a/cmd/msgvault/cmd/deletions_routing_test.go +++ b/cmd/msgvault/cmd/deletions_routing_test.go @@ -65,10 +65,11 @@ func TestDeletionManifestCommandsUseDaemonRunner(t *testing.T) { server, requests := newDaemonCLIRunnerTestServer(t, func(req daemonCLIRunTestRequest) { requestAssert.Equal(tt.want, req.Args, "args") }, `{"type":"stdout","data":`+string(stdoutJSON)+`}`, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) var stdout bytes.Buffer cmd := tt.cmd() + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetArgs(tt.args) @@ -112,9 +113,11 @@ func TestDeleteStagedTrashPromptsBeforeDaemonRunner(t *testing.T) { }, req.Args, "args") assert.Equal(map[string]string{remoteDeleteEnvVar: "1"}, req.Env, "env") }, `{"type":"stdout","data":"Deletion complete!\n"}`, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx cmd := newDeleteStagedRoutingTestCommand() + cmd.SetContext(testCtx) var stdout bytes.Buffer cmd.SetIn(bytes.NewBufferString("y\n")) cmd.SetOut(&stdout) @@ -129,6 +132,8 @@ func TestDeleteStagedTrashPromptsBeforeDaemonRunner(t *testing.T) { } func TestDeleteStagedConfigConsentReachesRemotePlanAndExecution(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) resetDeleteStagedRoutingGlobals(t) @@ -146,10 +151,12 @@ func TestDeleteStagedConfigConsentReachesRemotePlanAndExecution(t *testing.T) { }, func(req daemonCLIRunTestRequest) { assert.Equal(map[string]string{remoteDeleteEnvVar: "1"}, req.Env, "config consent becomes the synthetic marker") }, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL, cfg) + _ = testCtx cfg.Deletion.RemoteEnabled = true cmd := newDeleteStagedRoutingTestCommand() + cmd.SetContext(testCtx) cmd.SetArgs([]string{"--yes", "batch-123"}) require.NoError(cmd.Execute(), "delete-staged") @@ -158,6 +165,8 @@ func TestDeleteStagedConfigConsentReachesRemotePlanAndExecution(t *testing.T) { } func TestDeleteStagedDisabledConfigBlocksBeforeRemoteExecution(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) resetDeleteStagedRoutingGlobals(t) @@ -175,10 +184,12 @@ func TestDeleteStagedDisabledConfigBlocksBeforeRemoteExecution(t *testing.T) { "plan_fingerprint": "fp-disabled", "remote_delete_env_var": remoteDeleteEnvVar, }, nil) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL, cfg) + _ = testCtx cfg.Deletion.RemoteEnabled = false cmd := newDeleteStagedRoutingTestCommand() + cmd.SetContext(testCtx) cmd.SetArgs([]string{"--yes", "batch-123"}) err := cmd.Execute() @@ -217,9 +228,11 @@ func TestDeleteStagedDisplayNamePlanPinsSourceIDForDaemonRunner(t *testing.T) { "--yes", }, req.Args, "args") }, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx cmd := newDeleteStagedRoutingTestCommand() + cmd.SetContext(testCtx) cmd.SetArgs([]string{"--account", "Work", "--yes"}) require.NoError(cmd.Execute(), "delete-staged") @@ -257,9 +270,11 @@ func TestDeleteStagedPermanentPromptsBeforeDaemonRunner(t *testing.T) { }, req.Args, "args") assert.Equal(map[string]string{remoteDeleteEnvVar: "1"}, req.Env, "env") }, `{"type":"stdout","data":"Deletion complete!\n"}`, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx cmd := newDeleteStagedRoutingTestCommand() + cmd.SetContext(testCtx) var stdout bytes.Buffer cmd.SetIn(bytes.NewBufferString("delete\n")) cmd.SetOut(&stdout) @@ -302,9 +317,11 @@ func TestDeleteStagedWithoutBatchPinsPlannedBatchesForDaemonRunner(t *testing.T) }, req.Args, "args") assert.Equal(map[string]string{remoteDeleteEnvVar: "1"}, req.Env, "env") }, `{"type":"stdout","data":"Deletion complete!\n"}`, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx cmd := newDeleteStagedRoutingTestCommand() + cmd.SetContext(testCtx) var stdout bytes.Buffer cmd.SetOut(&stdout) cmd.SetArgs([]string{"--yes"}) @@ -349,9 +366,11 @@ func TestDeleteStagedScopeEscalationPromptsBeforeDaemonRunner(t *testing.T) { }, req.Args, "args") assert.Equal(map[string]string{remoteDeleteEnvVar: "1"}, req.Env, "env") }, `{"type":"stdout","data":"Deletion complete!\n"}`, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx cmd := newDeleteStagedRoutingTestCommand() + cmd.SetContext(testCtx) var stdout bytes.Buffer cmd.SetIn(bytes.NewBufferString("y\n")) cmd.SetOut(&stdout) @@ -364,6 +383,33 @@ func TestDeleteStagedScopeEscalationPromptsBeforeDaemonRunner(t *testing.T) { assert.Contains(stdout.String(), "Deletion complete!", "daemon output") } +func TestDeleteStagedRemoteGraphHeadlessUpgrade(t *testing.T) { + resetDeleteStagedRoutingGlobals(t) + t.Setenv(remoteDeleteEnvVar, "1") + server, runRequests, _ := newDaemonCLIDeleteStagedTestServer(t, nil, map[string]any{ + "needs_execution": true, + "planned_batch_ids": []string{"batch-graph"}, + "plan_fingerprint": "fp-graph", + "needs_scope_escalation": true, + "scope_escalation_headline": "PERMISSION UPGRADE REQUIRED", + "scope_escalation_account": "user@example.com", + "scope_escalation_source_type": "msmail", + }, func(req daemonCLIRunTestRequest) { + assert.Contains(t, req.Args, "--headless") + assert.Contains(t, req.Args, "--scope-escalation-confirmed") + }, `{"type":"complete"}`) + cmd := newDeleteStagedRoutingTestCommand() + cmd.SetContext(configureRemoteDaemonForTest(t, server.URL)) + cmd.SetIn(bytes.NewBufferString("y\n")) + var stdout bytes.Buffer + cmd.SetOut(&stdout) + cmd.SetArgs([]string{"--headless", "batch-graph"}) + // The frontend has no Microsoft configuration: remote authorization must + // stay in the worker that owns the token, with the headless flag intact. + require.NoError(t, cmd.Execute()) + assert.Equal(t, int32(1), runRequests.Load()) +} + func TestDeleteStagedConfirmationAndScopePromptsShareInput(t *testing.T) { require := require.New(t) assert := assert.New(t) @@ -393,9 +439,11 @@ func TestDeleteStagedConfirmationAndScopePromptsShareInput(t *testing.T) { "--skip-prelude", }, req.Args, "args") }, `{"type":"stdout","data":"Deletion complete!\n"}`, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx cmd := newDeleteStagedRoutingTestCommand() + cmd.SetContext(testCtx) var stdout bytes.Buffer cmd.SetIn(bytes.NewBufferString("delete\ny\n")) cmd.SetOut(&stdout) @@ -418,9 +466,11 @@ func TestCancelDeletionUsageErrorBeforeDaemonRunner(t *testing.T) { }) server, requests := newDaemonCLIRunnerTestServer(t, nil, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx cmd := newCancelDeletionRoutingTestCommand() + cmd.SetContext(testCtx) cmd.SetArgs([]string{"--all", "batch-123"}) err := cmd.Execute() @@ -433,6 +483,7 @@ func TestCancelDeletionUsageErrorBeforeDaemonRunner(t *testing.T) { func resetDeleteStagedRoutingGlobals(t *testing.T) { t.Helper() savedPermanent := deletePermanent + savedHeadless := deleteHeadless savedYes := deleteYes savedDryRun := deleteDryRun savedList := deleteList @@ -440,6 +491,7 @@ func resetDeleteStagedRoutingGlobals(t *testing.T) { savedSourceID := deleteSourceID savedPlannedBatchIDs := deletePlannedBatchIDs deletePermanent = false + deleteHeadless = false deleteYes = false deleteDryRun = false deleteList = false @@ -448,6 +500,7 @@ func resetDeleteStagedRoutingGlobals(t *testing.T) { deletePlannedBatchIDs = nil t.Cleanup(func() { deletePermanent = savedPermanent + deleteHeadless = savedHeadless deleteYes = savedYes deleteDryRun = savedDryRun deleteList = savedList @@ -464,6 +517,7 @@ func newDeleteStagedRoutingTestCommand() *cobra.Command { RunE: deleteStagedCmd.RunE, } cmd.Flags().BoolVar(&deletePermanent, "permanent", false, "Permanent") + cmd.Flags().BoolVar(&deleteHeadless, "headless", false, "Device-code sign-in") cmd.Flags().BoolVarP(&deleteYes, "yes", "y", false, "Skip confirmation") cmd.Flags().BoolVar(&deleteDryRun, "dry-run", false, "Dry run") cmd.Flags().BoolVarP(&deleteList, "list", "l", false, "List") diff --git a/cmd/msgvault/cmd/deletions_test.go b/cmd/msgvault/cmd/deletions_test.go index cd4089c76..1c3300303 100644 --- a/cmd/msgvault/cmd/deletions_test.go +++ b/cmd/msgvault/cmd/deletions_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "errors" + "net/http" "os" "path/filepath" "strconv" @@ -16,12 +17,17 @@ import ( "github.com/stretchr/testify/require" "go.kenn.io/msgvault/internal/api" "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/daemonclient" "go.kenn.io/msgvault/internal/deletion" + "go.kenn.io/msgvault/internal/microsoft" "go.kenn.io/msgvault/internal/store" "go.kenn.io/msgvault/internal/testutil" + "golang.org/x/oauth2" ) func TestRemoteDeleteEnabledUsesConfigOrEnvironment(t *testing.T) { + cfg := testConfigValue() + t.Setenv(daemonCLISubprocessEnv, "") tests := []struct { name string @@ -44,9 +50,13 @@ func TestRemoteDeleteEnabledUsesConfigOrEnvironment(t *testing.T) { savedCfg := cfg cfg = config.NewDefaultConfig() cfg.Deletion.RemoteEnabled = tt.configEnabled + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) t.Cleanup(func() { cfg = savedCfg }) - assert.Equal(t, tt.want, remoteDeleteEnabled(false)) + assert.Equal(t, tt.want, remoteDeleteEnabled( + false, + invocationFromContext(testCtx), + )) }) } @@ -56,9 +66,10 @@ func TestRemoteDeleteEnabledUsesConfigOrEnvironment(t *testing.T) { savedCfg := cfg cfg = config.NewDefaultConfig() cfg.Deletion.RemoteEnabled = true + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) t.Cleanup(func() { cfg = savedCfg }) - assert.False(t, remoteDeleteEnabled(true)) + assert.False(t, remoteDeleteEnabled(true, invocationFromContext(testCtx))) }) } @@ -135,7 +146,7 @@ func TestDeleteStagedFailsFastWhenArchiveOwned(t *testing.T) { New(t) dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) t.Setenv(remoteDeleteEnvVar, "1") savedPermanent := deletePermanent @@ -171,7 +182,8 @@ func TestDeleteStagedFailsFastWhenArchiveOwned(t *testing.T) { t.Cleanup(func() { require.NoError(owner.Close(), "close owner lock") }) cmd := &cobra.Command{Use: "delete-staged"} - cmd.SetContext(context.Background()) + cmd.SetContext(testCtx) + cmd.SetContext(testCtx) err = deleteStagedCmd.RunE(cmd, nil) require.Error(err, "delete-staged should fail while the archive is owned") assert.Contains(err.Error(), "write operation is in progress") @@ -183,7 +195,8 @@ func TestBuildDeleteStagedPlanPinsPlannedBatches(t *testing.T) { assert := assert.New(t) dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + _ = testCtx mgr, err := deletion.NewManager(filepath.Join(dataDir, "deletions")) require.NoError(err, "NewManager") @@ -193,6 +206,7 @@ func TestBuildDeleteStagedPlanPinsPlannedBatches(t *testing.T) { require.NoError(err, "CreateManifest second") plan, err := buildDeleteStagedPlan(deleteStagedPlanOptions{ + Invocation: invocationFromContext(testCtx), RemoteDeleteEnabled: true, Yes: true, }) @@ -204,6 +218,7 @@ func TestBuildDeleteStagedPlanPinsPlannedBatches(t *testing.T) { require.NoError(err, "CreateManifest new") pinned, err := buildDeleteStagedPlan(deleteStagedPlanOptions{ + Invocation: invocationFromContext(testCtx), PlannedBatchIDs: plan.PlannedBatchIDs, RemoteDeleteEnabled: true, Yes: true, @@ -215,6 +230,7 @@ func TestBuildDeleteStagedPlanPinsPlannedBatches(t *testing.T) { first.GmailIDs = append(first.GmailIDs, "gmail-4") require.NoError(mgr.SaveManifest(first), "SaveManifest changed first") changed, err := buildDeleteStagedPlan(deleteStagedPlanOptions{ + Invocation: invocationFromContext(testCtx), PlannedBatchIDs: plan.PlannedBatchIDs, RemoteDeleteEnabled: true, Yes: true, @@ -228,7 +244,8 @@ func TestBuildDeleteStagedPlanListGuidanceNamesBothConsentPaths(t *testing.T) { assert := assert.New(t) dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + _ = testCtx mgr, err := deletion.NewManager(filepath.Join(dataDir, "deletions")) require.NoError(err) manifest := deletion.NewManifestForSource("pending", []string{"gm-1"}, deletion.SourceReference{ @@ -236,7 +253,10 @@ func TestBuildDeleteStagedPlanListGuidanceNamesBothConsentPaths(t *testing.T) { }) require.NoError(mgr.SaveManifest(manifest)) - listPlan, err := buildDeleteStagedPlan(deleteStagedPlanOptions{List: true}) + listPlan, err := buildDeleteStagedPlan(deleteStagedPlanOptions{ + Invocation: invocationFromContext(testCtx), + List: true, + }) require.NoError(err) durable := "[deletion] remote_enabled = true" oneCommand := "One-command alternative: MSGVAULT_ENABLE_REMOTE_DELETE=1" @@ -251,7 +271,8 @@ func TestBuildDeleteStagedPlanBlockedErrorNamesBothConsentPaths(t *testing.T) { assert := assert.New(t) dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + _ = testCtx mgr, err := deletion.NewManager(filepath.Join(dataDir, "deletions")) require.NoError(err) manifest := deletion.NewManifestForSource("pending", []string{"gm-1"}, deletion.SourceReference{ @@ -259,7 +280,10 @@ func TestBuildDeleteStagedPlanBlockedErrorNamesBothConsentPaths(t *testing.T) { }) require.NoError(mgr.SaveManifest(manifest)) - blockedPlan, err := buildDeleteStagedPlan(deleteStagedPlanOptions{Yes: true}) + blockedPlan, err := buildDeleteStagedPlan(deleteStagedPlanOptions{ + Invocation: invocationFromContext(testCtx), + Yes: true, + }) require.NoError(err) durable := "[deletion] remote_enabled = true" require.Contains(blockedPlan.BlockedError, durable) @@ -274,7 +298,8 @@ func TestBuildDeleteStagedPlanFiltersVersionTwoBySourceID(t *testing.T) { assert := assert.New(t) dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + _ = testCtx mgr, err := deletion.NewManager(filepath.Join(dataDir, "deletions")) require.NoError(err) first := deletion.NewManifestForSource("first", []string{"gm-1"}, deletion.SourceReference{ @@ -287,7 +312,8 @@ func TestBuildDeleteStagedPlanFiltersVersionTwoBySourceID(t *testing.T) { require.NoError(mgr.SaveManifest(second)) plan, err := buildDeleteStagedPlan(deleteStagedPlanOptions{ - SourceID: 22, SourceIDSet: true, List: true, + Invocation: invocationFromContext(testCtx), + SourceID: 22, SourceIDSet: true, List: true, ResolvedSourceType: "gmail", ResolvedSourceIdentifier: "second@example.invalid", }) require.NoError(err) @@ -298,7 +324,8 @@ func TestBuildDeleteStagedPlanFiltersVersionTwoBySourceID(t *testing.T) { func TestBuildDeleteStagedPlanDoesNotSelectVersionTwoByLegacyFilterAccount(t *testing.T) { dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + _ = testCtx mgr, err := deletion.NewManager(filepath.Join(dataDir, "deletions")) require.NoError(t, err) manifest := deletion.NewManifestForSource("durable source", []string{"gm-1"}, deletion.SourceReference{ @@ -308,7 +335,8 @@ func TestBuildDeleteStagedPlanDoesNotSelectVersionTwoByLegacyFilterAccount(t *te require.NoError(t, mgr.SaveManifest(manifest)) _, err = buildDeleteStagedPlan(deleteStagedPlanOptions{ - BatchID: manifest.ID, Account: "other@example.invalid", List: true, + Invocation: invocationFromContext(testCtx), + BatchID: manifest.ID, Account: "other@example.invalid", List: true, }) require.ErrorContains(t, err, "does not match the requested source") } @@ -318,14 +346,16 @@ func TestBuildDeleteStagedPlanAllowsExplicitSelectorForUnboundLegacyManifest(t * assert := assert.New(t) dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + _ = testCtx mgr, err := deletion.NewManager(filepath.Join(dataDir, "deletions")) require.NoError(err) manifest, err := mgr.CreateManifest("legacy", []string{"gm-1"}, deletion.Filters{}) require.NoError(err) plan, err := buildDeleteStagedPlan(deleteStagedPlanOptions{ - Account: "source@example.invalid", List: true, + Invocation: invocationFromContext(testCtx), + Account: "source@example.invalid", List: true, }) require.NoError(err) assert.Equal([]string{manifest.ID}, plan.PlannedBatchIDs) @@ -336,7 +366,8 @@ func TestBuildDeleteStagedPlanAllowsSourceIDForLegacyManifestWithMatchingAccount assert := assert.New(t) dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + _ = testCtx mgr, err := deletion.NewManager(filepath.Join(dataDir, "deletions")) require.NoError(err) manifest, err := mgr.CreateManifest("legacy", []string{"gm-1"}, deletion.Filters{ @@ -349,7 +380,8 @@ func TestBuildDeleteStagedPlanAllowsSourceIDForLegacyManifestWithMatchingAccount require.NoError(err) plan, err := buildDeleteStagedPlan(deleteStagedPlanOptions{ - SourceID: 11, SourceIDSet: true, List: true, + Invocation: invocationFromContext(testCtx), + SourceID: 11, SourceIDSet: true, List: true, ResolvedSourceType: "gmail", ResolvedSourceIdentifier: "source@example.invalid", }) require.NoError(err) @@ -371,7 +403,7 @@ func TestBuildDeleteStagedPlanInspectsUnboundLegacyManifestMixedWithVersionTwo(t require := require.New(t) assert := assert.New(t) dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) mgr, err := deletion.NewManager(filepath.Join(dataDir, "deletions")) require.NoError(err) legacy, err := mgr.CreateManifest("legacy", []string{"gm-1"}, deletion.Filters{}) @@ -381,6 +413,7 @@ func TestBuildDeleteStagedPlanInspectsUnboundLegacyManifestMixedWithVersionTwo(t }) require.NoError(mgr.SaveManifest(bound)) + tt.opts.Invocation = invocationFromContext(testCtx) plan, err := buildDeleteStagedPlan(tt.opts) require.NoError(err) assert.ElementsMatch([]string{legacy.ID, bound.ID}, plan.PlannedBatchIDs) @@ -393,7 +426,8 @@ func TestBuildDeleteStagedPlanInspectsUnboundLegacyManifestMixedWithVersionTwo(t func TestBuildDeleteStagedPlanRejectsUnboundLegacyManifestMixedWithVersionTwoDuringExecution(t *testing.T) { require := require.New(t) dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + _ = testCtx mgr, err := deletion.NewManager(filepath.Join(dataDir, "deletions")) require.NoError(err) _, err = mgr.CreateManifest("legacy", []string{"gm-1"}, deletion.Filters{}) @@ -402,7 +436,11 @@ func TestBuildDeleteStagedPlanRejectsUnboundLegacyManifestMixedWithVersionTwoDur ID: 11, Type: "gmail", Identifier: "source@example.invalid", }))) - _, err = buildDeleteStagedPlan(deleteStagedPlanOptions{RemoteDeleteEnabled: true, Yes: true}) + _, err = buildDeleteStagedPlan(deleteStagedPlanOptions{ + Invocation: invocationFromContext(testCtx), + RemoteDeleteEnabled: true, + Yes: true, + }) require.ErrorContains(err, "legacy deletion manifest") } @@ -455,7 +493,8 @@ func TestDeleteStagedRejectsUnsupportedSourceBeforeClaim(t *testing.T) { assert := assert.New(t) require := require.New(t) dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + cfg := lifecycleTestConfig(dataDir) + testCtx := withStoreResolverConfig(t, cfg) t.Setenv(remoteDeleteEnvVar, "1") t.Setenv(daemonCLISubprocessEnv, strconv.Itoa(os.Getppid())) resetDeleteStagedRoutingGlobals(t) @@ -475,11 +514,12 @@ func TestDeleteStagedRejectsUnsupportedSourceBeforeClaim(t *testing.T) { require.NoError(mgr.SaveManifest(manifest)) cmd := newDeleteStagedRoutingTestCommand() + cmd.SetContext(testCtx) cmd.SetOut(new(bytes.Buffer)) cmd.SetErr(new(bytes.Buffer)) cmd.SetArgs([]string{"--yes", manifest.ID}) err = cmd.Execute() - require.ErrorContains(err, "not a gmail or imap source") + require.ErrorContains(err, "not a gmail, imap or msmail source") assert.FileExists(filepath.Join(mgr.PendingDir(), manifest.ID+".json")) assert.NoFileExists(filepath.Join(mgr.InProgressDir(), manifest.ID+".json")) } @@ -488,7 +528,8 @@ func TestDeleteStagedOAuthSetupFailureLeavesManifestPending(t *testing.T) { assert := assert.New(t) require := require.New(t) dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + cfg := lifecycleTestConfig(dataDir) + testCtx := withStoreResolverConfig(t, cfg) t.Setenv(remoteDeleteEnvVar, "1") t.Setenv(daemonCLISubprocessEnv, strconv.Itoa(os.Getppid())) resetDeleteStagedRoutingGlobals(t) @@ -508,6 +549,7 @@ func TestDeleteStagedOAuthSetupFailureLeavesManifestPending(t *testing.T) { require.NoError(mgr.SaveManifest(manifest)) cmd := newDeleteStagedRoutingTestCommand() + cmd.SetContext(testCtx) cmd.SetOut(new(bytes.Buffer)) cmd.SetErr(new(bytes.Buffer)) cmd.SetArgs([]string{"--yes", manifest.ID}) @@ -562,7 +604,7 @@ func TestBuildDeleteStagedPlanInspectsMultipleVersionTwoSources(t *testing.T) { require := require.New(t) assert := assert.New(t) dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) mgr, err := deletion.NewManager(filepath.Join(dataDir, "deletions")) require.NoError(err) first := deletion.NewManifestForSource("first", []string{"gm-1"}, deletion.SourceReference{ @@ -574,6 +616,7 @@ func TestBuildDeleteStagedPlanInspectsMultipleVersionTwoSources(t *testing.T) { require.NoError(mgr.SaveManifest(first)) require.NoError(mgr.SaveManifest(second)) + tt.opts.Invocation = invocationFromContext(testCtx) plan, err := buildDeleteStagedPlan(tt.opts) require.NoError(err) assert.ElementsMatch([]string{first.ID, second.ID}, plan.PlannedBatchIDs) @@ -586,7 +629,8 @@ func TestBuildDeleteStagedPlanInspectsMultipleVersionTwoSources(t *testing.T) { func TestBuildDeleteStagedPlanRejectsMultipleVersionTwoSourcesDuringExecution(t *testing.T) { require := require.New(t) dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + _ = testCtx mgr, err := deletion.NewManager(filepath.Join(dataDir, "deletions")) require.NoError(err) require.NoError(mgr.SaveManifest(deletion.NewManifestForSource("first", []string{"gm-1"}, deletion.SourceReference{ @@ -596,7 +640,11 @@ func TestBuildDeleteStagedPlanRejectsMultipleVersionTwoSourcesDuringExecution(t ID: 22, Type: "gmail", Identifier: "second@example.invalid", }))) - _, err = buildDeleteStagedPlan(deleteStagedPlanOptions{RemoteDeleteEnabled: true, Yes: true}) + _, err = buildDeleteStagedPlan(deleteStagedPlanOptions{ + Invocation: invocationFromContext(testCtx), + RemoteDeleteEnabled: true, + Yes: true, + }) require.ErrorContains(err, "multiple sources") } @@ -620,7 +668,8 @@ func TestBuildDeleteStagedPlanRejectsMethodFlagMismatch(t *testing.T) { assert := assert.New(t) dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + _ = testCtx mgr, err := deletion.NewManager(filepath.Join(dataDir, "deletions")) require.NoError(err, "NewManager") @@ -632,13 +681,18 @@ func TestBuildDeleteStagedPlanRejectsMethodFlagMismatch(t *testing.T) { require.NoError(err, "ClaimManifest") require.Equal(deletion.MethodDelete, claimed.Execution.Method) - _, err = buildDeleteStagedPlan(deleteStagedPlanOptions{RemoteDeleteEnabled: true, Yes: true}) + _, err = buildDeleteStagedPlan(deleteStagedPlanOptions{ + Invocation: invocationFromContext(testCtx), + RemoteDeleteEnabled: true, + Yes: true, + }) require.Error(err, "trash-flag resume of a permanent batch must be refused") assert.Contains(err.Error(), "must be resumed with it") assert.Contains(err.Error(), "--permanent") // The same batch plans cleanly once the flag matches the stored method. plan, err := buildDeleteStagedPlan(deleteStagedPlanOptions{ + Invocation: invocationFromContext(testCtx), RemoteDeleteEnabled: true, Yes: true, Permanent: true, }) require.NoError(err, "matching flag plans cleanly") @@ -649,8 +703,11 @@ func TestPlanCLIDeleteStagedReportsDeletionScopeEscalation(t *testing.T) { require := require.New(t) assert := assert.New(t) - _, restore := seedTokenEnv(t, gmailOnlyTokenJSON) + tokenPath, restore := seedTokenEnv(t, gmailOnlyTokenJSON) defer restore() + cfg := testConfigValue() + cfg.OAuth.ClientSecrets = filepath.Join(filepath.Dir(filepath.Dir(tokenPath)), "client_secret.json") + testCtx := testInvocationContext(context.Background(), cfg, invocationOptions{}) st := testutil.NewTestStore(t) _, err := st.GetOrCreateSource(sourceTypeGmail, scopeEscalationAccount) @@ -661,7 +718,7 @@ func TestPlanCLIDeleteStagedReportsDeletionScopeEscalation(t *testing.T) { manifest, err := mgr.CreateManifest("permanent batch", []string{"gmail-1"}, deletion.Filters{Account: scopeEscalationAccount}) require.NoError(err, "CreateManifest") - got, err := planCLIDeleteStaged(context.Background(), st, api.CLIDeleteStagedPlanRequest{ + got, err := planCLIDeleteStaged(testCtx, st, api.CLIDeleteStagedPlanRequest{ Permanent: true, Yes: true, RemoteDeleteEnabled: true, @@ -680,12 +737,93 @@ func TestPlanCLIDeleteStagedReportsDeletionScopeEscalation(t *testing.T) { assert.Empty(got.ScopeEscalationOAuthApp, "default app binding") } +// An msmail batch passes the source check. A read-only Graph token needs the +// scope upgrade, and a token with Mail.ReadWrite does not. The plan names the +// account so the frontend can authorize before starting the daemon subprocess. +func TestPlanCLIDeleteStagedMSMailScopeEscalation(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + dataDir := t.TempDir() + cfg := lifecycleTestConfig(dataDir) + testCtx := withStoreResolverConfig(t, cfg) + st := testutil.NewTestStore(t) + source, err := st.GetOrCreateSource(sourceTypeMSMail, "user@company.example") + require.NoError(err) + mgr, err := deletion.NewManager(filepath.Join(dataDir, "deletions")) + require.NoError(err) + require.NoError(mgr.SaveManifest(deletion.NewManifestForSource("graph mail", []string{"AAMk-1"}, deletion.SourceReference{ + ID: source.ID, Type: source.SourceType, Identifier: source.Identifier, + }))) + tokenPath := microsoft.NewGraphMailManager("", "", "", cfg.TokensDir(), nil).TokenPath(source.Identifier) + require.NoError(os.MkdirAll(filepath.Dir(tokenPath), 0o700)) + saveScopes := func(scopes []string) { + data, err := json.Marshal(map[string]any{"access_token": "a", "token_type": "Bearer", "scopes": scopes}) + require.NoError(err) + require.NoError(os.WriteFile(tokenPath, data, 0o600)) + } + req := api.CLIDeleteStagedPlanRequest{Yes: true, RemoteDeleteEnabled: true} + + saveScopes(microsoft.GraphMailScopes()) + got, err := planCLIDeleteStaged(testCtx, st, req) + require.NoError(err) + assert.True(got.NeedsScopeEscalation) + assert.Contains(got.ScopeEscalationBodyLines, "Deletion requires the Microsoft Graph Mail.ReadWrite permission.") + assert.Equal(source.Identifier, got.ScopeEscalationAccount) + assert.Equal(sourceTypeMSMail, got.ScopeEscalationSourceType) + + saveScopes(microsoft.GraphMailWriteScopes()) + got, err = planCLIDeleteStaged(testCtx, st, req) + require.NoError(err) + assert.True(got.NeedsExecution) + assert.False(got.NeedsScopeEscalation) +} + +func TestPreflightDeleteStagedGraphHeadless(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + resetDeleteStagedRoutingGlobals(t) + deleteHeadless = true + cfg := lifecycleTestConfig(t.TempDir()) + cfg.Microsoft.ClientID = "test-client" + testCtx := withStoreResolverConfig(t, cfg) + account := "user@example.com" + tokenPath := newGraphMailManager(invocationFromContext(testCtx)).TokenPath(account) + require.NoError(os.MkdirAll(filepath.Dir(tokenPath), 0o700)) + original := []byte(`{"access_token":"existing-read-token"}`) + require.NoError(os.WriteFile(tokenPath, original, 0o600)) + + // Intercept only the OAuth HTTP boundary. Reaching the device endpoint + // proves the frontend selected Graph and headless authorization; failure + // must propagate to the caller without replacing the existing token. + requestErr := errors.New("device endpoint unavailable") + requests := 0 + httpClient := &http.Client{Transport: testTransport(func(req *http.Request) (*http.Response, error) { + requests++ + assert.Equal(http.MethodPost, req.Method) + assert.Equal("https://login.microsoftonline.com/common/oauth2/v2.0/devicecode", req.URL.String()) + require.NoError(req.ParseForm()) + assert.Equal("test-client", req.Form.Get("client_id")) + assert.Contains(strings.Fields(req.Form.Get("scope")), "https://graph.microsoft.com/Mail.ReadWrite") + return nil, requestErr + })} + testCtx = context.WithValue(testCtx, oauth2.HTTPClient, httpClient) + err := preflightDeleteStagedScopeEscalation(testCtx, &daemonclient.CLIDeleteStagedPlan{ + ScopeEscalationAccount: account, ScopeEscalationSourceType: sourceTypeMSMail, + }) + require.ErrorIs(err, requestErr) + assert.Equal(1, requests) + after, err := os.ReadFile(tokenPath) + require.NoError(err) + assert.Equal(original, after) +} + func TestPlanCLIDeleteStagedResolvesDisplayNameBeforeFiltering(t *testing.T) { require := require.New(t) assert := assert.New(t) dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) st := testutil.NewTestStore(t) gmailSource, err := st.GetOrCreateSource(sourceTypeGmail, "source@example.invalid") require.NoError(err) @@ -704,7 +842,7 @@ func TestPlanCLIDeleteStagedResolvesDisplayNameBeforeFiltering(t *testing.T) { }) require.NoError(mgr.SaveManifest(gmailManifest)) - got, err := planCLIDeleteStaged(context.Background(), st, api.CLIDeleteStagedPlanRequest{ + got, err := planCLIDeleteStaged(testCtx, st, api.CLIDeleteStagedPlanRequest{ Account: "Work", Yes: true, RemoteDeleteEnabled: true, }) require.NoError(err) @@ -717,8 +855,11 @@ func TestPlanCLIDeleteStagedEscalatesLegacyGmailTokenForPermanentDelete(t *testi require := require.New(t) assert := assert.New(t) - _, restore := seedTokenEnv(t, legacyTokenJSON) + tokenPath, restore := seedTokenEnv(t, legacyTokenJSON) defer restore() + cfg := testConfigValue() + cfg.OAuth.ClientSecrets = filepath.Join(filepath.Dir(filepath.Dir(tokenPath)), "client_secret.json") + testCtx := testInvocationContext(context.Background(), cfg, invocationOptions{}) st := testutil.NewTestStore(t) _, err := st.GetOrCreateSource(sourceTypeGmail, scopeEscalationAccount) @@ -729,7 +870,7 @@ func TestPlanCLIDeleteStagedEscalatesLegacyGmailTokenForPermanentDelete(t *testi _, err = mgr.CreateManifest("legacy token batch", []string{"gmail-1"}, deletion.Filters{Account: scopeEscalationAccount}) require.NoError(err, "CreateManifest") - got, err := planCLIDeleteStaged(context.Background(), st, api.CLIDeleteStagedPlanRequest{ + got, err := planCLIDeleteStaged(testCtx, st, api.CLIDeleteStagedPlanRequest{ Permanent: true, Yes: true, RemoteDeleteEnabled: true, diff --git a/cmd/msgvault/cmd/direct_write.go b/cmd/msgvault/cmd/direct_write.go index a61ebc425..49e5bef78 100644 --- a/cmd/msgvault/cmd/direct_write.go +++ b/cmd/msgvault/cmd/direct_write.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "errors" "fmt" @@ -17,7 +18,7 @@ import ( // // The lock is taken non-blocking, so there is no context parameter: a writer // either claims the free SQLite archive immediately or is told who holds it. -func acquireDirectSQLiteWriteLock(cfg *config.Config) (func(), error) { +func acquireDirectSQLiteWriteLock(cfg *config.Config, state *invocation) (func(), error) { if cfg == nil { return nil, errors.New("nil config") } @@ -36,7 +37,8 @@ func acquireDirectSQLiteWriteLock(cfg *config.Config) (func(), error) { } return func() { if cerr := lock.Close(); cerr != nil { - logger.Warn("release write-owner lock", "error", cerr) + _, currentLogger := invocationConfigLogger(state) + currentLogger.Warn("release write-owner lock", "error", cerr) } }, nil } @@ -102,13 +104,12 @@ func daemonAutostartPreflight(cfg *config.Config) error { return nil } -// openStoreAndInitWith opens the local archive and initializes schema while the -// caller owns the direct-writer lock. store.Open + InitSchema create the -// database file on first use, which is the right behavior for a -// freshly-installed CLI; init-db remains the explicit setup command for users -// who want to pre-create the DB. -func openStoreAndInitWith(migrate func(*store.Store) error) (*store.Store, error) { - dbPath := cfg.DatabaseDSN() +func openStoreAndInitWithInvocation(state *invocation, migrate func(*store.Store) error) (*store.Store, error) { + currentCfg, _ := invocationConfigLogger(state) + if currentCfg == nil { + return nil, errors.New("configuration is unavailable") + } + dbPath := currentCfg.DatabaseDSN() st, err := store.Open(dbPath) if err != nil { return nil, fmt.Errorf("open database: %w", err) @@ -124,21 +125,27 @@ func openStoreAndInitWith(migrate func(*store.Store) error) (*store.Store, error return st, nil } -func openWritableStoreAndInit() (*store.Store, func(), error) { - return openWritableStoreAndInitWith(runStartupMigrations) +func openWritableStoreAndInitForInvocation(state *invocation) (*store.Store, func(), error) { + return openWritableStoreAndInitWithInvocation(state, func(s *store.Store) error { + return runStartupMigrationsContext(context.Background(), s, state) + }) } -func openWritableStoreAndInitForIngest() (*store.Store, func(), error) { - return openWritableStoreAndInitWith(runStartupMigrationsForIngest) +func openWritableStoreAndInitForIngestInvocation(state *invocation) (*store.Store, func(), error) { + return openWritableStoreAndInitWithInvocation(state, runStartupMigrationsForIngest) } -func openWritableStoreAndInitWith(migrate func(*store.Store) error) (*store.Store, func(), error) { - release, err := acquireDirectSQLiteWriteLock(cfg) +func openWritableStoreAndInitWithInvocation(state *invocation, migrate func(*store.Store) error) (*store.Store, func(), error) { + currentCfg, _ := invocationConfigLogger(state) + if currentCfg == nil { + return nil, nil, errors.New("configuration is unavailable") + } + release, err := acquireDirectSQLiteWriteLock(currentCfg, state) if err != nil { return nil, nil, err } - st, err := openStoreAndInitWith(migrate) + st, err := openStoreAndInitWithInvocation(state, migrate) if err != nil { release() return nil, nil, err diff --git a/cmd/msgvault/cmd/direct_write_test.go b/cmd/msgvault/cmd/direct_write_test.go index 9f038a16b..e079d3f51 100644 --- a/cmd/msgvault/cmd/direct_write_test.go +++ b/cmd/msgvault/cmd/direct_write_test.go @@ -1,7 +1,6 @@ package cmd import ( - "context" "errors" "os" "strconv" @@ -24,7 +23,7 @@ func TestAcquireDirectSQLiteWriteLockSkipsPostgreSQL(t *testing.T) { require.NoError(t, err, "pre-acquire sqlite lock") t.Cleanup(func() { _ = owner.Close() }) - release, err := acquireDirectSQLiteWriteLock(cfg) + release, err := acquireDirectSQLiteWriteLock(cfg, &invocation{cfg: cfg, logger: testDiscardLogger()}) require.NoError(t, err, "postgres direct writer should not use sqlite flock") require.NotNil(t, release, "release") release() @@ -37,7 +36,7 @@ func TestAcquireDirectSQLiteWriteLock_HoldsThenReleases(t *testing.T) { dataDir := t.TempDir() cfg := lifecycleTestConfig(dataDir) - release, err := acquireDirectSQLiteWriteLock(cfg) + release, err := acquireDirectSQLiteWriteLock(cfg, &invocation{cfg: cfg, logger: testDiscardLogger()}) require.NoError( err, "acquire on a free archive") @@ -65,9 +64,13 @@ func TestOpenWritableStoreAndInitOwnsArchiveUntilCleanup(t *testing.T) { dataDir := t.TempDir() testCfg := lifecycleTestConfig(dataDir) - withStoreResolverConfig(t, testCfg) + testCtx := withStoreResolverConfig(t, testCfg) + _ = testCtx - st, cleanup, err := openWritableStoreAndInit() + st, cleanup, err := openWritableStoreAndInitWithInvocation( + invocationFromContext(testCtx), + runStartupMigrationsForIngest, + ) require.NoError( err, "open writable store") @@ -108,7 +111,7 @@ func TestAcquireDirectSQLiteWriteLock_ActionableErrorWhenOwned(t *testing.T) { t.Cleanup(func() { _ = owner.Close() }) - release, err := acquireDirectSQLiteWriteLock(cfg) + release, err := acquireDirectSQLiteWriteLock(cfg, &invocation{cfg: cfg, logger: testDiscardLogger()}) assert.Nil(release, "no release when blocked") require.Error(err, "acquire on an owned archive") assert.Contains(err.Error(), "owned", "names the ownership condition") @@ -209,7 +212,8 @@ func TestDeduplicateFailsFastWhenArchiveOwned(t *testing.T) { t) dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + ctx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + _ = ctx owner, err := tryAcquireWriteOwnerLock(dataDir) require.NoError( @@ -217,7 +221,9 @@ func TestDeduplicateFailsFastWhenArchiveOwned(t *testing.T) { t.Cleanup(func() { _ = owner.Close() }) - err = runDeduplicate(&cobra.Command{}, nil) + cmd := &cobra.Command{} + cmd.SetContext(ctx) + err = runDeduplicate(cmd, nil) require.Error(err, "deduplicate must fail while the archive is owned") assert.Contains(err.Error(), "write operation", "actionable ownership error") assert.Contains(err.Error(), "cannot start", "explains daemon autostart is blocked") @@ -229,7 +235,7 @@ func TestEmbeddingsRetireFailsFastWhenArchiveOwned(t *testing.T) { t) dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) owner, err := tryAcquireWriteOwnerLock(dataDir) require.NoError( @@ -238,7 +244,8 @@ func TestEmbeddingsRetireFailsFastWhenArchiveOwned(t *testing.T) { t.Cleanup(func() { _ = owner.Close() }) cmd := &cobra.Command{Use: "retire"} - cmd.SetContext(context.Background()) + cmd.SetContext(testCtx) + cmd.SetContext(testCtx) err = runEmbeddingsRetire(cmd, []string{"1"}) require.Error(err, "embeddings retire must fail while the archive is owned") assert.Contains(err.Error(), "owned", "actionable ownership error") @@ -250,14 +257,15 @@ func TestEmbeddingsPruneFailsFastWhenArchiveOwned(t *testing.T) { require := require.New(t) dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) owner, err := tryAcquireWriteOwnerLock(dataDir) require.NoError(err, "acquire owner lock") t.Cleanup(func() { _ = owner.Close() }) cmd := &cobra.Command{Use: "prune"} - cmd.SetContext(context.Background()) + cmd.SetContext(testCtx) + cmd.SetContext(testCtx) err = runEmbeddingsPrune(cmd, nil) require.Error(err, "embeddings prune must fail while the archive is owned") assert.Contains(err.Error(), "owned", "actionable ownership error") @@ -271,7 +279,7 @@ func TestEmbeddingsListFailsFastWhenArchiveOwned(t *testing.T) { dataDir := t.TempDir() testCfg := lifecycleTestConfig(dataDir) - withStoreResolverConfig(t, testCfg) + testCtx := withStoreResolverConfig(t, testCfg) st, err := store.Open(testCfg.DatabaseDSN()) require.NoError( @@ -290,7 +298,8 @@ func TestEmbeddingsListFailsFastWhenArchiveOwned(t *testing.T) { t.Cleanup(func() { _ = owner.Close() }) cmd := &cobra.Command{Use: "list"} - cmd.SetContext(context.Background()) + cmd.SetContext(testCtx) + cmd.SetContext(testCtx) err = runEmbeddingsList(cmd, nil) require.Error(err, "embeddings list must fail while the archive is owned") assert.Contains(err.Error(), "owned", "actionable ownership error") @@ -303,7 +312,8 @@ func TestInitDBFailsFastWhenArchiveOwned(t *testing.T) { t) dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + _ = testCtx owner, err := tryAcquireWriteOwnerLock(dataDir) require.NoError( @@ -311,7 +321,9 @@ func TestInitDBFailsFastWhenArchiveOwned(t *testing.T) { t.Cleanup(func() { _ = owner.Close() }) - err = initDBCmd.RunE(&cobra.Command{Use: "init-db"}, nil) + cmd := &cobra.Command{Use: "init-db"} + cmd.SetContext(testCtx) + err = initDBCmd.RunE(cmd, nil) require.Error(err, "init-db must fail while the archive is owned") assert.Contains(err.Error(), "write operation", "explains the active writer") assert.Contains(err.Error(), "wait", "points at the remedy") @@ -323,7 +335,8 @@ func TestVerifyDaemonAutostartFailsFastWhenArchiveOwned(t *testing.T) { t) dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + _ = testCtx owner, err := tryAcquireWriteOwnerLock(dataDir) require.NoError( @@ -331,7 +344,9 @@ func TestVerifyDaemonAutostartFailsFastWhenArchiveOwned(t *testing.T) { t.Cleanup(func() { _ = owner.Close() }) - err = verifyCmd.RunE(&cobra.Command{Use: "verify"}, []string{"alice@example.com"}) + cmd := &cobra.Command{Use: "verify"} + cmd.SetContext(testCtx) + err = verifyCmd.RunE(cmd, []string{"alice@example.com"}) require.Error(err, "verify must not autostart a daemon while the archive is owned") assert.Contains(err.Error(), "write operation is in progress", "actionable ownership error") assert.Contains(err.Error(), "cannot start", "daemon start is refused") @@ -344,7 +359,8 @@ func TestBuildCacheFailsFastWhenArchiveOwned(t *testing.T) { dataDir := t.TempDir() testCfg := lifecycleTestConfig(dataDir) - withStoreResolverConfig(t, testCfg) + testCtx := withStoreResolverConfig(t, testCfg) + _ = testCtx st, err := store.Open(testCfg.DatabaseDSN()) require.NoError( @@ -362,7 +378,9 @@ func TestBuildCacheFailsFastWhenArchiveOwned(t *testing.T) { t.Cleanup(func() { _ = owner.Close() }) - err = buildCacheCmd.RunE(&cobra.Command{Use: "build-cache"}, nil) + cmd := &cobra.Command{Use: "build-cache"} + cmd.SetContext(testCtx) + err = buildCacheCmd.RunE(cmd, nil) require.Error(err, "build-cache must fail while a local writer owns the archive") assert.Contains(err.Error(), "write operation is in progress", "actionable ownership error") assert.Contains(err.Error(), "cannot start", "daemon start is refused") @@ -377,7 +395,7 @@ func TestBuildCacheDaemonChildBypassesArchiveOwnershipLock(t *testing.T) { t.Cleanup(func() { _ = owner.Close() }) t.Setenv(buildCacheDaemonSubprocessEnv, strconv.Itoa(os.Getppid())) - release, err := acquireBuildCacheWriteLock(testCfg) + release, err := acquireBuildCacheWriteLock(testCfg, &invocation{cfg: testCfg, logger: testDiscardLogger()}) require.NoError(t, err, "daemon-owned build-cache child should not reacquire the daemon lock") release() } @@ -391,7 +409,7 @@ func TestDaemonCLIChildBypassesArchiveOwnershipLock(t *testing.T) { t.Cleanup(func() { _ = owner.Close() }) t.Setenv(daemonCLISubprocessEnv, strconv.Itoa(os.Getppid())) - release, err := acquireDirectSQLiteWriteLock(testCfg) + release, err := acquireDirectSQLiteWriteLock(testCfg, &invocation{cfg: testCfg, logger: testDiscardLogger()}) require.NoError(t, err, "daemon-owned CLI child should not reacquire the daemon lock") release() } @@ -403,7 +421,8 @@ func TestCreateSubsetFailsFastWhenArchiveOwned(t *testing.T) { dataDir := t.TempDir() testCfg := lifecycleTestConfig(dataDir) - withStoreResolverConfig(t, testCfg) + testCtx := withStoreResolverConfig(t, testCfg) + _ = testCtx st, err := store.Open(testCfg.DatabaseDSN()) require.NoError( @@ -430,7 +449,9 @@ func TestCreateSubsetFailsFastWhenArchiveOwned(t *testing.T) { t.Cleanup(func() { _ = owner.Close() }) - err = runCreateSubset(&cobra.Command{Use: "create-subset"}, nil) + cmd := &cobra.Command{Use: "create-subset"} + cmd.SetContext(testCtx) + err = runCreateSubset(cmd, nil) require.Error(err, "create-subset must not autostart a daemon while the archive is owned") assert.Contains(err.Error(), "write operation", "actionable ownership error") assert.Contains(err.Error(), "cannot start", "daemon start is refused") @@ -442,7 +463,7 @@ func TestCreateSubsetFailsFastWhenArchiveOwned(t *testing.T) { // archive. func TestOpenHTTPStoreFailsWhenDirectWriterOwnsArchive(t *testing.T) { dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + ctx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) stubStartServeBackgroundProcess(t, func(*config.Config, backgroundServeStartOptions) (*backgroundServeProcess, error) { require.FailNow(t, "must not spawn a daemon while a writer owns the archive") return nil, errors.New("unreachable") @@ -452,7 +473,7 @@ func TestOpenHTTPStoreFailsWhenDirectWriterOwnsArchive(t *testing.T) { require.NoError(t, err, "acquire owner lock") t.Cleanup(func() { _ = owner.Close() }) - _, _, err = OpenHTTPStore(context.Background()) + _, _, err = OpenHTTPStore(ctx) require.Error(t, err, "OpenHTTPStore must not autostart over a direct writer") assert.Contains(t, err.Error(), "write operation", "explains the contention") } diff --git a/cmd/msgvault/cmd/discord_common.go b/cmd/msgvault/cmd/discord_common.go index 84a313c1a..a78789860 100644 --- a/cmd/msgvault/cmd/discord_common.go +++ b/cmd/msgvault/cmd/discord_common.go @@ -20,6 +20,7 @@ import ( const sourceTypeDiscord = "discord" type discordCommandDeps struct { + bind func(context.Context) discordCommandDeps openStore func() (*store.Store, func(), error) tokenManager func() *discord.TokenManager apiBaseURL func() string @@ -32,18 +33,58 @@ type discordCommandDeps struct { } func defaultDiscordCommandDeps() discordCommandDeps { + return defaultDiscordCommandDepsForContext(context.Background()) +} + +func defaultDiscordCommandDepsForContext(ctx context.Context) discordCommandDeps { + state := invocationFromContext(ctx) + var currentCfg *config.Config + if state != nil { + currentCfg = state.cfg + } return discordCommandDeps{ - openStore: openWritableStoreAndInitForIngest, - tokenManager: func() *discord.TokenManager { return discord.NewTokenManager(cfg.TokensDir()) }, - apiBaseURL: func() string { return discord.DefaultBaseURL }, + bind: defaultDiscordCommandDepsForContext, + openStore: func() (*store.Store, func(), error) { + if state == nil || currentCfg == nil { + return nil, nil, errors.New("configuration is unavailable") + } + return openWritableStoreAndInitForIngestInvocation(state) + }, + tokenManager: func() *discord.TokenManager { + if currentCfg == nil { + return discord.NewTokenManager("") + } + return discord.NewTokenManager(currentCfg.TokensDir()) + }, + apiBaseURL: func() string { return discord.DefaultBaseURL }, providerConfig: func() config.DiscordConfig { - return cfg.Discord + if currentCfg == nil { + return config.DiscordConfig{} + } + return currentCfg.Discord + }, + attachmentsDir: func() string { + if currentCfg == nil { + return "" + } + return currentCfg.AttachmentsDir() + }, + databaseDSN: func() string { + if currentCfg == nil { + return "" + } + return currentCfg.DatabaseDSN() + }, + rebuildCache: func(dbPath string) error { + if currentCfg == nil { + return errors.New("configuration is unavailable") + } + return rebuildCacheAfterManualSync(dbPath, state) + }, + postSourceMigrations: func(st *store.Store) error { + return runPostSourceCreateMigrationsForInvocation(st, state) }, - attachmentsDir: func() string { return cfg.AttachmentsDir() }, - databaseDSN: func() string { return cfg.DatabaseDSN() }, - rebuildCache: rebuildCacheAfterWrite, - postSourceMigrations: runPostSourceCreateMigrations, - registerGuild: registerDiscordGuild, + registerGuild: registerDiscordGuild, } } diff --git a/cmd/msgvault/cmd/discord_routing_test.go b/cmd/msgvault/cmd/discord_routing_test.go index de5cce92f..30622c0ef 100644 --- a/cmd/msgvault/cmd/discord_routing_test.go +++ b/cmd/msgvault/cmd/discord_routing_test.go @@ -63,10 +63,11 @@ func TestDiscordCommandsRouteThroughDaemonCLIRunner(t *testing.T) { assert.Equal(t, tt.wantArgs, req.Args) assert.Equal(t, tt.wantEnv, req.Env) }, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) t.Setenv(daemonCLISubprocessEnv, "") cmd := tt.command(discordCommandDeps{}) + cmd.SetContext(testCtx) cmd.SetIn(strings.NewReader(tt.stdin)) cmd.SetArgs(tt.args) require.NoError(t, cmd.Execute()) diff --git a/cmd/msgvault/cmd/documents.go b/cmd/msgvault/cmd/documents.go index e97d38ee6..238b4b3b6 100644 --- a/cmd/msgvault/cmd/documents.go +++ b/cmd/msgvault/cmd/documents.go @@ -97,7 +97,9 @@ func (p *commandOperationPass) checkpoint(ctx context.Context, counters operatio checkpointCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), commandOperationRecorderTimeout) defer cancel() if err := p.recorder.Checkpoint(checkpointCtx, p.id, counters); err != nil { - logger.Error("operation recorder checkpoint failed", "kind", p.kind, "error", err) + if state := invocationFromContext(ctx); state != nil && state.logger != nil { + state.logger.Error("operation recorder checkpoint failed", "kind", p.kind, "error", err) + } } } @@ -110,13 +112,17 @@ func (p *commandOperationPass) finish( publicError := commandOperationPublicError(ctx, runErr) state, err := operations.DeriveInvocationState(p.kind, counters, publicError) if err != nil { - logger.Error("operation recorder finish state failed", "kind", p.kind, "error", err) + if state := invocationFromContext(ctx); state != nil && state.logger != nil { + state.logger.Error("operation recorder finish state failed", "kind", p.kind, "error", err) + } return } finishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), commandOperationRecorderTimeout) defer cancel() if err := p.recorder.Finish(finishCtx, p.id, counters, state, publicError); err != nil { - logger.Error("operation recorder finish failed", "kind", p.kind, "error", err) + if inv := invocationFromContext(ctx); inv != nil && inv.logger != nil { + inv.logger.Error("operation recorder finish failed", "kind", p.kind, "error", err) + } } } @@ -183,6 +189,7 @@ type documentBuildResult struct { type documentBuildFailure struct { CanonicalBlobHash string ReasonCode string + Detail string } type documentsCommandDeps struct { @@ -190,8 +197,8 @@ type documentsCommandDeps struct { newMistralProcessor func(*documentindex.DocumentsConfig) (documentindex.MistralProcessor, error) validateProbeFixtures func(context.Context, mistral.Policy, mistral.ProbeFixtureConfig) error runCapabilityProbe func(context.Context, *mistral.Client, mistral.ProbeConfig) (mistral.CapabilityManifest, error) - openStore func() (*store.Store, func(), error) - openAttachments func(*store.Store) (documentindex.DocumentAttachmentOpener, func() error, error) + openStore func(context.Context) (*store.Store, func(), error) + openAttachments func(context.Context, *store.Store) (documentindex.DocumentAttachmentOpener, func() error, error) openReadClient func(context.Context) (documentReadClient, func(), error) runDocumentVector func(context.Context, *store.Store, int64, int) (vectordocument.ReconcileResult, error) } @@ -213,9 +220,13 @@ func defaultDocumentsCommandDeps() documentsCommandDeps { newMistralProcessor: newConfiguredMistralProcessor, validateProbeFixtures: mistral.ValidateProbeFixtures, runCapabilityProbe: mistral.RunCapabilityProbe, - openStore: openWritableStoreAndInit, - runDocumentVector: runConfiguredDocumentVectorGeneration, - openAttachments: openDocumentAttachments, + openStore: func(ctx context.Context) (*store.Store, func(), error) { + return openWritableStoreAndInitForInvocation(invocationFromContext(ctx)) + }, + runDocumentVector: runConfiguredDocumentVectorGeneration, + openAttachments: func(ctx context.Context, st *store.Store) (documentindex.DocumentAttachmentOpener, func() error, error) { + return openDocumentAttachments(st, invocationFromContext(ctx)) + }, openReadClient: func(ctx context.Context) (documentReadClient, func(), error) { client, _, err := OpenHTTPStore(ctx) if err != nil { @@ -373,7 +384,7 @@ func newBuildDocumentsCmd(deps documentsCommandDeps) *cobra.Command { mode = documentBuildStartRebuild } if !isDaemonCLISubprocess() { - return runDaemonCLICommandHTTPFromCobraWithLocalFiles(command, args, documentProviderForwardEnv()) + return runDaemonCLICommandHTTPFromCobraWithLocalFiles(command, args, documentProviderForwardEnv(invocationFromCommand(command))) } return runBuildDocuments(command, capabilityPath, limit, mode, confirmed, deps) }, @@ -396,7 +407,7 @@ func newResumeDocumentsCmd(deps documentsCommandDeps) *cobra.Command { Args: cobra.NoArgs, RunE: func(command *cobra.Command, args []string) error { if !isDaemonCLISubprocess() { - return runDaemonCLICommandHTTPFromCobraWithLocalFiles(command, args, documentProviderForwardEnv()) + return runDaemonCLICommandHTTPFromCobraWithLocalFiles(command, args, documentProviderForwardEnv(invocationFromCommand(command))) } return runBuildDocuments(command, capabilityPath, limit, documentBuildResume, confirmed, deps) }, @@ -485,10 +496,12 @@ func newPurgeDocumentDerivedCmd(deps documentsCommandDeps) *cobra.Command { // documentProviderForwardEnv carries the caller's configured provider key to // the daemon-owned subprocess that performs an explicitly requested build. -func documentProviderForwardEnv() map[string]string { - if cfg == nil { +func documentProviderForwardEnv(state *invocation) map[string]string { + state = invocationState(context.Background(), state) + if state == nil || state.cfg == nil { return nil } + cfg := state.cfg name := cfg.Attachments.Documents.APIKeyEnv if name == "" { return nil @@ -506,9 +519,11 @@ func runProbeMistral( validateOnly bool, deps documentsCommandDeps, ) error { - if cfg == nil { + state := invocationFromCommand(command) + if state == nil || state.cfg == nil { return errors.New("document probe requires loaded configuration") } + cfg := state.cfg documentsConfig := &cfg.Attachments.Documents if !validateOnly { if !documentsConfig.Enabled { @@ -571,7 +586,7 @@ func runConsentMistral( confirmed bool, deps documentsCommandDeps, ) error { - documentsConfig, manifest, inputPolicy, profile, err := configuredDocumentProfile(capabilityPath) + documentsConfig, manifest, inputPolicy, profile, err := configuredDocumentProfile(capabilityPath, invocationFromCommand(command)) if err != nil { return err } @@ -587,7 +602,7 @@ func runConsentMistral( if manifest.MaxUnits < documentsConfig.MaxPagesPerDocument || len(inputPolicy.AllowedMediaTypes) == 0 { return errors.New("document capability manifest does not authorize the configured policy") } - st, cleanup, err := deps.openStore() + st, cleanup, err := deps.openStore(command.Context()) if err != nil { return err } @@ -601,7 +616,19 @@ func runConsentMistral( }); err != nil { return err } - if err := bootstrapDocumentOccurrencesIfConsented(command.Context(), st); err != nil { + if profile.IncludeInline { + // Repeat the full scan on every confirmed inline consent. An older + // completed bootstrap, or an interrupted consent scan, may omit rows. + reconciler, err := documentindex.NewReconciler(st, documentindex.ReconcilerConfig{ + AttachmentPageSize: 1000, ChangePageSize: 1000, + }) + if err != nil { + return err + } + if _, err := reconciler.FullReconcile(command.Context()); err != nil { + return err + } + } else if err := bootstrapDocumentOccurrencesIfConsented(command.Context(), st); err != nil { return err } _, _ = fmt.Fprintf(command.OutOrStdout(), @@ -630,6 +657,11 @@ func printDocumentConsentDisclosure( inputPolicy documentindex.ResolvedInputPolicy, ) { _, _ = fmt.Fprintln(w, "Hosted document extraction disclosure:") + roles := "standalone document attachments" + if profile.IncludeInline { + roles = "standalone and inline document attachments" + } + _, _ = fmt.Fprintf(w, "- Scope includes %s in the configured document formats and message sources.\n", roles) _, _ = fmt.Fprintf(w, "- Authenticated upload routes target %s (%s):\n", profile.Endpoint, profile.Region) for _, mediaType := range inputPolicy.AllowedMediaTypes { route := inputPolicy.Routes[mediaType] @@ -671,10 +703,15 @@ func runBuildDocuments( confirmed bool, deps documentsCommandDeps, ) (runErr error) { + state := invocationFromCommand(command) + if state == nil || state.cfg == nil { + return errors.New("document build requires loaded configuration") + } + cfg := state.cfg if limit <= 0 || limit > 10_000 { return errors.New("document build limit must be between 1 and 10000") } - documentsConfig, manifest, inputPolicy, profile, err := configuredDocumentProfile(capabilityPath) + documentsConfig, manifest, inputPolicy, profile, err := configuredDocumentProfile(capabilityPath, invocationFromCommand(command)) if err != nil { return err } @@ -692,7 +729,7 @@ func runBuildDocuments( ) return errors.New("document build requires --yes after reviewing the provider upload preflight") } - st, cleanup, err := deps.openStore() + st, cleanup, err := deps.openStore(command.Context()) if err != nil { return err } @@ -729,7 +766,7 @@ func runBuildDocuments( printDocumentBuildPreflight( command.OutOrStdout(), documentsConfig, profile, inputPolicy, status, limit, mode, ) - attachments, closeAttachments, err := deps.openAttachments(st) + attachments, closeAttachments, err := deps.openAttachments(command.Context(), st) if err != nil { return err } @@ -931,6 +968,7 @@ func executeDocumentBuild( result.Failures = append(result.Failures, documentBuildFailure{ CanonicalBlobHash: extraction.CanonicalBlobHash, ReasonCode: extraction.FailureReasonCode, + Detail: extraction.FailureDetail, }) pass.checkpoint(ctx, documentExtractionCounters(result)) continue @@ -963,6 +1001,9 @@ func executeDocumentBuild( var details strings.Builder for _, failure := range result.Failures { fmt.Fprintf(&details, "\n%s: %s", failure.CanonicalBlobHash, failure.ReasonCode) + if failure.Detail != "" { + fmt.Fprintf(&details, ": %s", failure.Detail) + } } return result, fmt.Errorf( "document build completed with %d extraction failure(s):%s\nretry one with `msgvault documents retry --capabilities --hash `", @@ -1007,7 +1048,9 @@ func runDocumentStatus( jsonOutput bool, deps documentsCommandDeps, ) error { - documentsConfig, _, inputPolicy, profile, err := configuredDocumentProfile(capabilityPath) + documentsConfig, _, inputPolicy, profile, err := configuredDocumentProfile( + capabilityPath, invocationFromCommand(command), + ) if err != nil { return err } @@ -1070,6 +1113,16 @@ func runDocumentStatus( _, _ = fmt.Fprintf(command.OutOrStdout(), "Active full rebuild: %d of %d owner(s) remaining\n", rebuildStatus.RemainingOwners, rebuildStatus.SnapshotOwners) } + for _, failure := range status.Failures { + _, _ = fmt.Fprintf(command.OutOrStdout(), "%s: %s", failure.CanonicalBlobHash, failure.ReasonCode) + if failure.Detail != "" { + _, _ = fmt.Fprintf(command.OutOrStdout(), ": %s", failure.Detail) + } + _, _ = fmt.Fprintln(command.OutOrStdout()) + } + if len(status.Failures) > 0 && !status.FailuresExhausted { + _, _ = fmt.Fprintln(command.OutOrStdout(), "Additional document failures omitted; retry reported documents and check status again.") + } return nil } @@ -1079,11 +1132,11 @@ func runRetryDocument( canonicalBlobHash string, deps documentsCommandDeps, ) error { - profile, err := configuredDocumentProfileOnly(capabilityPath) + profile, err := configuredDocumentProfileOnly(capabilityPath, invocationFromCommand(command)) if err != nil { return err } - st, cleanup, err := deps.openStore() + st, cleanup, err := deps.openStore(command.Context()) if err != nil { return err } @@ -1100,8 +1153,8 @@ func runRetryDocument( return nil } -func configuredDocumentProfileOnly(capabilityPath string) (store.DocumentExtractionProfile, error) { - documentsConfig, manifest, inputPolicy, profile, err := configuredDocumentProfile(capabilityPath) +func configuredDocumentProfileOnly(capabilityPath string, state *invocation) (store.DocumentExtractionProfile, error) { + documentsConfig, manifest, inputPolicy, profile, err := configuredDocumentProfile(capabilityPath, state) _ = documentsConfig _ = manifest _ = inputPolicy @@ -1117,7 +1170,7 @@ func runRetireDocumentProfile( if !confirmed { return errors.New("document profile retirement requires --yes") } - st, cleanup, err := deps.openStore() + st, cleanup, err := deps.openStore(command.Context()) if err != nil { return err } @@ -1142,7 +1195,7 @@ func runPurgeDocumentDerived( if !confirmed { return errors.New("document derivative purge requires --yes") } - st, cleanup, err := deps.openStore() + st, cleanup, err := deps.openStore(command.Context()) if err != nil { return err } @@ -1198,7 +1251,7 @@ func openDocumentReadClient( if deps.openReadClient != nil { return deps.openReadClient(ctx) } - st, cleanup, err := deps.openStore() + st, cleanup, err := deps.openStore(ctx) if err != nil { return nil, func() {}, err } @@ -1331,11 +1384,14 @@ func bootstrapDocumentOccurrencesIfConsented(ctx context.Context, st *store.Stor func configuredDocumentProfile( capabilityPath string, + state *invocation, ) (*documentindex.DocumentsConfig, mistral.CapabilityManifest, documentindex.ResolvedInputPolicy, store.DocumentExtractionProfile, error) { - if cfg == nil { + state = invocationState(context.Background(), state) + if state == nil || state.cfg == nil { return nil, mistral.CapabilityManifest{}, documentindex.ResolvedInputPolicy{}, store.DocumentExtractionProfile{}, errors.New("document operation requires loaded configuration") } + cfg := state.cfg documentsConfig := &cfg.Attachments.Documents if documentsConfig.RetentionPosture == documentindex.RetentionUnknown || documentsConfig.TrainingPosture == documentindex.TrainingUnknown { @@ -1394,14 +1450,20 @@ func documentProfileForConfig( Model: values.Model, RetentionPosture: values.Retention, TrainingPosture: values.Training, AllowedMediaTypes: allowedMediaTypes, PolicyJSON: policyJSON, + IncludeInline: documentsConfig.Scope.IncludeInline, } return inputPolicy, profile, nil } func openDocumentAttachments( st *store.Store, + state *invocation, ) (documentindex.DocumentAttachmentOpener, func() error, error) { - attachments, err := attachmentstore.New(store.NewPackCatalog(st), cfg.AttachmentsDir()) + state = invocationState(context.Background(), state) + if state == nil || state.cfg == nil { + return nil, nil, errors.New("configuration is unavailable") + } + attachments, err := attachmentstore.New(store.NewPackCatalog(st), state.cfg.AttachmentsDir()) if err != nil { return nil, nil, err } diff --git a/cmd/msgvault/cmd/documents_routing_test.go b/cmd/msgvault/cmd/documents_routing_test.go index bf9a829b8..28695feac 100644 --- a/cmd/msgvault/cmd/documents_routing_test.go +++ b/cmd/msgvault/cmd/documents_routing_test.go @@ -10,6 +10,8 @@ import ( ) func TestDocumentVectorCommandsRouteWithConfiguredRemote(t *testing.T) { + cfg := testConfigValue() + const ( apiKeyEnv = "MSGVAULT_VECTOR_TEST_KEY" apiKey = "synthetic-vector-key" @@ -40,21 +42,25 @@ func TestDocumentVectorCommandsRouteWithConfiguredRemote(t *testing.T) { assert.Empty(req.Env) } }, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + cfg = invocationFromContext(testCtx).cfg cfg.Vector.Embeddings.APIKeyEnv = apiKeyEnv t.Setenv(apiKeyEnv, apiKey) root := &cobra.Command{Use: "msgvault"} root.AddCommand(newDocumentsCmd(documentsCommandDeps{})) + root.SetContext(testCtx) root.SetArgs(test.args) - require.NoError(t, root.ExecuteContext(t.Context())) + require.NoError(t, root.ExecuteContext(testCtx)) assert.Equal(1, int(requests.Load())) }) } } func TestDocumentMutationsRouteSafelyWithConfiguredRemote(t *testing.T) { + cfg := testConfigValue() + const ( apiKeyEnv = "MSGVAULT_DOCUMENT_TEST_KEY" apiKey = "synthetic-document-key" @@ -128,7 +134,8 @@ func TestDocumentMutationsRouteSafelyWithConfiguredRemote(t *testing.T) { assert.Empty(req.Env) } }, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + cfg = invocationFromContext(testCtx).cfg documentsConfig := documentindex.DefaultDocumentsConfig() documentsConfig.APIKeyEnv = apiKeyEnv cfg.Attachments.Documents = documentsConfig @@ -136,9 +143,10 @@ func TestDocumentMutationsRouteSafelyWithConfiguredRemote(t *testing.T) { root := &cobra.Command{Use: "msgvault"} root.AddCommand(newDocumentsCmd(documentsCommandDeps{})) + root.SetContext(testCtx) root.SetArgs(test.args) - err := root.ExecuteContext(t.Context()) + err := root.ExecuteContext(testCtx) if test.localFile { require.ErrorContains(err, "run it on the daemon host with --local") assert.Equal(0, int(requests.Load())) diff --git a/cmd/msgvault/cmd/documents_test.go b/cmd/msgvault/cmd/documents_test.go index 347785849..2966fa5ab 100644 --- a/cmd/msgvault/cmd/documents_test.go +++ b/cmd/msgvault/cmd/documents_test.go @@ -32,11 +32,15 @@ import ( ) func TestProbeMistralCommandWritesCompleteSanitizedManifest(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) previousConfig := cfg t.Cleanup(func() { cfg = previousConfig }) cfg = config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Data.DataDir = t.TempDir() cfg.Attachments.Documents.Enabled = true cfg.Attachments.Documents.RetentionPosture = documentindex.RetentionStandard @@ -55,12 +59,13 @@ func TestProbeMistralCommandWritesCompleteSanitizedManifest(t *testing.T) { }, } command := newDocumentsCmd(deps) + command.SetContext(testCtx) var output bytes.Buffer command.SetOut(&output) command.SetErr(&bytes.Buffer{}) command.SetArgs([]string{"probe-mistral", "--fixtures", "synthetic-fixtures"}) - require.NoError(command.ExecuteContext(t.Context())) + require.NoError(command.ExecuteContext(testCtx)) assert.True(probeCalled) manifest, err := mistral.DecodeCapabilityManifest(bytes.NewReader(output.Bytes())) require.NoError(err) @@ -70,10 +75,14 @@ func TestProbeMistralCommandWritesCompleteSanitizedManifest(t *testing.T) { } func TestProbeMistralValidateOnlyNeedsNoProviderConfiguration(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) previousConfig := cfg t.Cleanup(func() { cfg = previousConfig }) cfg = config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Data.DataDir = t.TempDir() providerCalled := false validationCalled := false @@ -89,12 +98,13 @@ func TestProbeMistralValidateOnlyNeedsNoProviderConfiguration(t *testing.T) { }, } command := newDocumentsCmd(deps) + command.SetContext(testCtx) var output bytes.Buffer command.SetOut(&output) command.SetErr(&bytes.Buffer{}) command.SetArgs([]string{"probe-mistral", "--fixtures", "synthetic-fixtures", "--validate-only"}) - require.NoError(t, command.ExecuteContext(t.Context())) + require.NoError(t, command.ExecuteContext(testCtx)) assert.False(providerCalled) assert.True(validationCalled) assert.Contains(output.String(), "Validated 26 private Mistral fixture(s) locally") @@ -103,12 +113,16 @@ func TestProbeMistralValidateOnlyNeedsNoProviderConfiguration(t *testing.T) { } func TestDocumentsConsentBuildAndStatusUseExactAuthenticatedProfile(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) markDaemonCLISubprocessForTest(t) previousConfig := cfg t.Cleanup(func() { cfg = previousConfig }) cfg = config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Data.DataDir = t.TempDir() cfg.Attachments.Documents.Enabled = true cfg.Attachments.Documents.Conversion.CSV.Enabled = true @@ -122,7 +136,7 @@ func TestDocumentsConsentBuildAndStatusUseExactAuthenticatedProfile(t *testing.T hash := sha256.Sum256(content) digest := hex.EncodeToString(hash[:]) messageID := fixture.CreateMessage("documents-command") - require.NoError(fixture.Store.UpsertAttachmentRecord(t.Context(), messageID, store.AttachmentWrite{ + require.NoError(fixture.Store.UpsertAttachmentRecord(testCtx, messageID, store.AttachmentWrite{ Filename: "synthetic.pdf", MIMEType: "application/pdf", Size: int64(len(content)), StoragePath: digest[:2] + "/" + digest, ContentHash: digest, Role: store.AttachmentRoleStandalone, RoleSource: store.AttachmentRoleSourceImporterSemantics, @@ -135,10 +149,10 @@ func TestDocumentsConsentBuildAndStatusUseExactAuthenticatedProfile(t *testing.T newMistralProcessor: func(*documentindex.DocumentsConfig) (documentindex.MistralProcessor, error) { return processor, nil }, - openStore: func() (*store.Store, func(), error) { + openStore: func(context.Context) (*store.Store, func(), error) { return fixture.Store, func() {}, nil }, - openAttachments: func(*store.Store) (documentindex.DocumentAttachmentOpener, func() error, error) { + openAttachments: func(context.Context, *store.Store) (documentindex.DocumentAttachmentOpener, func() error, error) { attachmentOpened = true return commandAttachmentOpener{content: content}, func() error { return nil }, nil }, @@ -149,7 +163,7 @@ func TestDocumentsConsentBuildAndStatusUseExactAuthenticatedProfile(t *testing.T unconfirmedConsent.SetOut(&disclosureOutput) unconfirmedConsent.SetErr(&bytes.Buffer{}) unconfirmedConsent.SetArgs([]string{"consent-mistral", "--capabilities", manifestPath}) - require.ErrorContains(unconfirmedConsent.ExecuteContext(t.Context()), "requires --yes") + require.ErrorContains(unconfirmedConsent.ExecuteContext(testCtx), "requires --yes") assert.Contains(disclosureOutput.String(), "source text/csv is converted locally; generated application/pdf bytes are sent") assert.Contains(disclosureOutput.String(), "original application/pdf bytes and media type are sent") assert.Contains(disclosureOutput.String(), "retention=standard, training=opted-out") @@ -162,7 +176,7 @@ func TestDocumentsConsentBuildAndStatusUseExactAuthenticatedProfile(t *testing.T consent.SetOut(&consentOutput) consent.SetErr(&bytes.Buffer{}) consent.SetArgs([]string{"consent-mistral", "--capabilities", manifestPath, "--yes"}) - require.NoError(consent.ExecuteContext(t.Context())) + require.NoError(consent.ExecuteContext(testCtx)) assert.Contains(consentOutput.String(), "Recorded Mistral document consent") assert.NotContains(consentOutput.String(), manifestPath) assert.Equal(1, commandDocumentOccurrenceCount(t, fixture.Store), @@ -173,7 +187,7 @@ func TestDocumentsConsentBuildAndStatusUseExactAuthenticatedProfile(t *testing.T build.SetOut(&buildOutput) build.SetErr(&bytes.Buffer{}) build.SetArgs([]string{documentBuildSubcommand, "--capabilities", manifestPath, "--limit", "5"}) - require.ErrorContains(build.ExecuteContext(t.Context()), "requires --yes") + require.ErrorContains(build.ExecuteContext(testCtx), "requires --yes") assert.Contains(buildOutput.String(), "Document build upload preflight") assert.Contains(buildOutput.String(), "source text/csv is converted locally; generated application/pdf bytes are sent") assert.False(attachmentOpened) @@ -184,7 +198,7 @@ func TestDocumentsConsentBuildAndStatusUseExactAuthenticatedProfile(t *testing.T build.SetOut(&buildOutput) build.SetErr(&bytes.Buffer{}) build.SetArgs([]string{documentBuildSubcommand, "--capabilities", manifestPath, "--limit", "5", "--yes"}) - require.NoError(build.ExecuteContext(t.Context())) + require.NoError(build.ExecuteContext(testCtx)) assert.Contains(buildOutput.String(), "indexed 1 document(s), 1 unit(s), skipped 0, failed 0") assert.Equal(1, processor.calls) @@ -193,7 +207,7 @@ func TestDocumentsConsentBuildAndStatusUseExactAuthenticatedProfile(t *testing.T search.SetOut(&searchOutput) search.SetErr(&bytes.Buffer{}) search.SetArgs([]string{"search", "Synthetic", "--json"}) - require.NoError(search.ExecuteContext(t.Context())) + require.NoError(search.ExecuteContext(testCtx)) var searchResponse store.DocumentSearchResponse require.NoError(json.Unmarshal(searchOutput.Bytes(), &searchResponse)) require.Len(searchResponse.Results, 1) @@ -206,20 +220,20 @@ func TestDocumentsConsentBuildAndStatusUseExactAuthenticatedProfile(t *testing.T resume.SetOut(&resumeOutput) resume.SetErr(&bytes.Buffer{}) resume.SetArgs([]string{"resume", "--capabilities", manifestPath, "--limit", "5", "--yes"}) - require.NoError(resume.ExecuteContext(t.Context())) + require.NoError(resume.ExecuteContext(testCtx)) assert.Contains(resumeOutput.String(), "indexed 0 document(s)") assert.Equal(1, processor.calls) rebuild := newDocumentsCmd(deps) rebuild.SetArgs([]string{documentBuildSubcommand, "--capabilities", manifestPath, "--full-rebuild"}) - require.ErrorContains(rebuild.ExecuteContext(t.Context()), "requires --yes") + require.ErrorContains(rebuild.ExecuteContext(testCtx), "requires --yes") assert.Equal(1, processor.calls) rebuild = newDocumentsCmd(deps) var rebuildOutput bytes.Buffer rebuild.SetOut(&rebuildOutput) rebuild.SetErr(&bytes.Buffer{}) rebuild.SetArgs([]string{documentBuildSubcommand, "--capabilities", manifestPath, "--full-rebuild", "--yes"}) - require.NoError(rebuild.ExecuteContext(t.Context())) + require.NoError(rebuild.ExecuteContext(testCtx)) assert.Contains(rebuildOutput.String(), "indexed 1 document(s)") assert.Equal(2, processor.calls) search = newDocumentsCmd(deps) @@ -227,7 +241,7 @@ func TestDocumentsConsentBuildAndStatusUseExactAuthenticatedProfile(t *testing.T search.SetOut(&searchOutput) search.SetErr(&bytes.Buffer{}) search.SetArgs([]string{"search", "Replacement", "--json"}) - require.NoError(search.ExecuteContext(t.Context())) + require.NoError(search.ExecuteContext(testCtx)) require.NoError(json.Unmarshal(searchOutput.Bytes(), &searchResponse)) require.Len(searchResponse.Results, 1) assert.Equal(messageID, searchResponse.Results[0].MessageID) @@ -237,7 +251,7 @@ func TestDocumentsConsentBuildAndStatusUseExactAuthenticatedProfile(t *testing.T status.SetOut(&statusOutput) status.SetErr(&bytes.Buffer{}) status.SetArgs([]string{"status", "--capabilities", manifestPath}) - require.NoError(status.ExecuteContext(t.Context())) + require.NoError(status.ExecuteContext(testCtx)) assert.Contains(statusOutput.String(), "Exact consent: true") assert.Contains(statusOutput.String(), "Coverage: 1 ready") assert.Contains(statusOutput.String(), "Extraction accounting: 2 attempt(s), 2 successful, 0 failed") @@ -250,7 +264,7 @@ func TestDocumentsConsentBuildAndStatusUseExactAuthenticatedProfile(t *testing.T statusJSON.SetOut(&statusJSONOutput) statusJSON.SetErr(&bytes.Buffer{}) statusJSON.SetArgs([]string{"status", "--capabilities", manifestPath, "--json"}) - require.NoError(statusJSON.ExecuteContext(t.Context())) + require.NoError(statusJSON.ExecuteContext(testCtx)) var structuredStatus documentStatusOutput require.NoError(json.Unmarshal(statusJSONOutput.Bytes(), &structuredStatus)) assert.Equal(2, structuredStatus.AuthenticatedFormats) @@ -276,21 +290,21 @@ func TestDocumentsConsentBuildAndStatusUseExactAuthenticatedProfile(t *testing.T require.NotNil(structuredStatus.EstimatedSuccessfulCostUSD) assert.InDelta(0.008, *structuredStatus.EstimatedSuccessfulCostUSD, 0.000001) - profile, err := configuredDocumentProfileOnly(manifestPath) + profile, err := configuredDocumentProfileOnly(manifestPath, invocationFromContext(testCtx)) require.NoError(err) retry := newDocumentsCmd(deps) retry.SetArgs([]string{"retry", "--capabilities", manifestPath, "--hash", digest}) - require.ErrorContains(retry.ExecuteContext(t.Context()), "already current") + require.ErrorContains(retry.ExecuteContext(testCtx), "already current") retire := newDocumentsCmd(deps) retire.SetArgs([]string{"retire", profile.ID}) - require.ErrorContains(retire.ExecuteContext(t.Context()), "requires --yes") + require.ErrorContains(retire.ExecuteContext(testCtx), "requires --yes") retire = newDocumentsCmd(deps) var retireOutput bytes.Buffer retire.SetOut(&retireOutput) retire.SetErr(&bytes.Buffer{}) retire.SetArgs([]string{"retire", profile.ID, "--yes"}) - require.NoError(retire.ExecuteContext(t.Context())) + require.NoError(retire.ExecuteContext(testCtx)) assert.Contains(retireOutput.String(), "Retired document extraction profile") search = newDocumentsCmd(deps) @@ -298,19 +312,19 @@ func TestDocumentsConsentBuildAndStatusUseExactAuthenticatedProfile(t *testing.T search.SetOut(&searchOutput) search.SetErr(&bytes.Buffer{}) search.SetArgs([]string{"search", "Synthetic", "--json"}) - require.NoError(search.ExecuteContext(t.Context())) + require.NoError(search.ExecuteContext(testCtx)) require.NoError(json.Unmarshal(searchOutput.Bytes(), &searchResponse)) assert.Empty(searchResponse.Results) purge := newDocumentsCmd(deps) purge.SetArgs([]string{"purge-derived", "--hash", digest}) - require.ErrorContains(purge.ExecuteContext(t.Context()), "requires --yes") + require.ErrorContains(purge.ExecuteContext(testCtx), "requires --yes") purge = newDocumentsCmd(deps) var purgeOutput bytes.Buffer purge.SetOut(&purgeOutput) purge.SetErr(&bytes.Buffer{}) purge.SetArgs([]string{"purge-derived", "--hash", digest, "--yes"}) - require.NoError(purge.ExecuteContext(t.Context())) + require.NoError(purge.ExecuteContext(testCtx)) assert.Contains(purgeOutput.String(), "Purged 2 extraction(s) and 1 current head(s)") } @@ -363,12 +377,16 @@ func TestDocumentConsentDisclosureListsResolvedUploadRoutes(t *testing.T) { } func TestDocumentBuildRepairsHistoricalMIMERolesBeforePreflight(t *testing.T) { + cfg := testConfigValue() + markDaemonCLISubprocessForTest(t) require := require.New(t) assert := assert.New(t) previousConfig := cfg t.Cleanup(func() { cfg = previousConfig }) cfg = config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Data.DataDir = t.TempDir() cfg.Attachments.Documents.Enabled = true cfg.Attachments.Documents.RetentionPosture = documentindex.RetentionStandard @@ -398,16 +416,17 @@ func TestDocumentBuildRepairsHistoricalMIMERolesBeforePreflight(t *testing.T) { manifestPath := writeCommandCapabilityManifest(t, cfg.Attachments.Documents.MaxPagesPerDocument) deps := documentsCommandDeps{ - openStore: func() (*store.Store, func(), error) { return fixture.Store, func() {}, nil }, - openAttachments: func(*store.Store) (documentindex.DocumentAttachmentOpener, func() error, error) { + openStore: func(context.Context) (*store.Store, func(), error) { return fixture.Store, func() {}, nil }, + openAttachments: func(context.Context, *store.Store) (documentindex.DocumentAttachmentOpener, func() error, error) { return nil, func() error { return nil }, errors.New("synthetic stop after repair") }, } command := newDocumentsCmd(deps) + command.SetContext(testCtx) command.SetOut(&bytes.Buffer{}) command.SetErr(&bytes.Buffer{}) command.SetArgs([]string{documentBuildSubcommand, "--capabilities", manifestPath}) - require.ErrorContains(command.ExecuteContext(t.Context()), "requires --yes") + require.ErrorContains(command.ExecuteContext(testCtx), "requires --yes") var role, roleSource string var partKey *string @@ -422,12 +441,13 @@ func TestDocumentBuildRepairsHistoricalMIMERolesBeforePreflight(t *testing.T) { consent.SetOut(&bytes.Buffer{}) consent.SetErr(&bytes.Buffer{}) consent.SetArgs([]string{"consent-mistral", "--capabilities", manifestPath, "--yes"}) - require.NoError(consent.ExecuteContext(t.Context())) + require.NoError(consent.ExecuteContext(testCtx)) command = newDocumentsCmd(deps) + command.SetContext(testCtx) command.SetOut(&bytes.Buffer{}) command.SetErr(&bytes.Buffer{}) command.SetArgs([]string{documentBuildSubcommand, "--capabilities", manifestPath, "--yes"}) - require.ErrorContains(command.ExecuteContext(t.Context()), "synthetic stop after repair") + require.ErrorContains(command.ExecuteContext(testCtx), "synthetic stop after repair") require.NoError(fixture.Store.DB().QueryRow(fixture.Store.Rebind(` SELECT attachment_role, role_source, source_part_key @@ -442,7 +462,7 @@ func TestDocumentsSearchDoesNotRegisterUnconsentedJournalConsumer(t *testing.T) require := require.New(t) fixture := storetest.New(t) deps := documentsCommandDeps{ - openStore: func() (*store.Store, func(), error) { return fixture.Store, func() {}, nil }, + openStore: func(context.Context) (*store.Store, func(), error) { return fixture.Store, func() {}, nil }, } command := newDocumentsCmd(deps) var output bytes.Buffer @@ -462,7 +482,7 @@ func TestDocumentsSearchDoesNotRegisterUnconsentedJournalConsumer(t *testing.T) func TestDocumentsSearchLocalExplicitSemanticNeverMasqueradesAsLexical(t *testing.T) { fixture := storetest.New(t) command := newDocumentsCmd(documentsCommandDeps{ - openStore: func() (*store.Store, func(), error) { return fixture.Store, func() {}, nil }, + openStore: func(context.Context) (*store.Store, func(), error) { return fixture.Store, func() {}, nil }, }) command.SetArgs([]string{"search", "evidence", "--mode", "semantic", "--candidate-limit", "25"}) err := command.ExecuteContext(t.Context()) @@ -490,7 +510,7 @@ func TestDocumentsSearchUsesConfiguredReadClient(t *testing.T) { }, } command := newDocumentsCmd(documentsCommandDeps{ - openStore: func() (*store.Store, func(), error) { + openStore: func(context.Context) (*store.Store, func(), error) { openStoreCalled = true return nil, func() {}, errors.New("local store must not be opened") }, @@ -543,10 +563,14 @@ func TestDocumentsSearchRejectsNonPositivePersonScopeBeforeDispatch(t *testing.T } func TestDocumentsStatusUsesConfiguredReadClient(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) previousConfig := cfg t.Cleanup(func() { cfg = previousConfig }) cfg = config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Attachments.Documents.Enabled = true cfg.Attachments.Documents.RetentionPosture = documentindex.RetentionStandard cfg.Attachments.Documents.TrainingPosture = documentindex.TrainingOptedOut @@ -567,7 +591,7 @@ func TestDocumentsStatusUsesConfiguredReadClient(t *testing.T) { }, } command := newDocumentsCmd(documentsCommandDeps{ - openStore: func() (*store.Store, func(), error) { + openStore: func(context.Context) (*store.Store, func(), error) { openStoreCalled = true return nil, func() {}, errors.New("local store must not be opened") }, @@ -575,21 +599,26 @@ func TestDocumentsStatusUsesConfiguredReadClient(t *testing.T) { return reader, func() { cleanupCalled = true }, nil }, }) + command.SetContext(testCtx) var output bytes.Buffer command.SetOut(&output) command.SetErr(&bytes.Buffer{}) command.SetArgs([]string{"status", "--capabilities", manifestPath, "--json"}) - require.NoError(t, command.ExecuteContext(t.Context())) + require.NoError(t, command.ExecuteContext(testCtx)) assert.False(openStoreCalled) assert.True(cleanupCalled) assert.Contains(output.String(), `"ready_owners":3`) } func TestDocumentsBuildRefusesAPIUseBeforeExactConsent(t *testing.T) { + cfg := testConfigValue() + markDaemonCLISubprocessForTest(t) previousConfig := cfg t.Cleanup(func() { cfg = previousConfig }) cfg = config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Data.DataDir = t.TempDir() cfg.Attachments.Documents.Enabled = true cfg.Attachments.Documents.RetentionPosture = documentindex.RetentionStandard @@ -601,23 +630,28 @@ func TestDocumentsBuildRefusesAPIUseBeforeExactConsent(t *testing.T) { providerCalled = true return &commandBuildProcessor{}, nil }, - openStore: func() (*store.Store, func(), error) { return fixture.Store, func() {}, nil }, + openStore: func(context.Context) (*store.Store, func(), error) { return fixture.Store, func() {}, nil }, } manifestPath := writeCommandCapabilityManifest(t, cfg.Attachments.Documents.MaxPagesPerDocument) command := newDocumentsCmd(deps) + command.SetContext(testCtx) command.SetArgs([]string{documentBuildSubcommand, "--capabilities", manifestPath, "--yes"}) - err := command.ExecuteContext(t.Context()) + err := command.ExecuteContext(testCtx) require.ErrorContains(t, err, "requires exact consent") assert.False(t, providerCalled) } func TestDocumentFullRebuildResumesDurableTargetSnapshot(t *testing.T) { + cfg := testConfigValue() + markDaemonCLISubprocessForTest(t) require := require.New(t) assert := assert.New(t) previousConfig := cfg t.Cleanup(func() { cfg = previousConfig }) cfg = config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Data.DataDir = t.TempDir() cfg.Attachments.Documents.Enabled = true cfg.Attachments.Documents.RetentionPosture = documentindex.RetentionStandard @@ -632,7 +666,7 @@ func TestDocumentFullRebuildResumesDurableTargetSnapshot(t *testing.T) { digest := hex.EncodeToString(digestBytes[:]) contents[digest] = content messageID := fixture.CreateMessage("documents-rebuild-" + string(rune('a'+index))) - require.NoError(fixture.Store.UpsertAttachmentRecord(t.Context(), messageID, store.AttachmentWrite{ + require.NoError(fixture.Store.UpsertAttachmentRecord(testCtx, messageID, store.AttachmentWrite{ Filename: "synthetic.pdf", MIMEType: "application/pdf", Size: int64(len(content)), StoragePath: digest[:2] + "/" + digest, ContentHash: digest, Role: store.AttachmentRoleStandalone, RoleSource: store.AttachmentRoleSourceImporterSemantics, @@ -645,17 +679,17 @@ func TestDocumentFullRebuildResumesDurableTargetSnapshot(t *testing.T) { newMistralProcessor: func(*documentindex.DocumentsConfig) (documentindex.MistralProcessor, error) { return processor, nil }, - openStore: func() (*store.Store, func(), error) { return fixture.Store, func() {}, nil }, - openAttachments: func(*store.Store) (documentindex.DocumentAttachmentOpener, func() error, error) { + openStore: func(context.Context) (*store.Store, func(), error) { return fixture.Store, func() {}, nil }, + openAttachments: func(context.Context, *store.Store) (documentindex.DocumentAttachmentOpener, func() error, error) { return commandAttachmentMapOpener{contents: contents}, func() error { return nil }, nil }, } consent := newDocumentsCmd(deps) consent.SetArgs([]string{"consent-mistral", "--capabilities", manifestPath, "--yes"}) - require.NoError(consent.ExecuteContext(t.Context())) + require.NoError(consent.ExecuteContext(testCtx)) initial := newDocumentsCmd(deps) initial.SetArgs([]string{documentBuildSubcommand, "--capabilities", manifestPath, "--limit", "2", "--yes"}) - require.NoError(initial.ExecuteContext(t.Context())) + require.NoError(initial.ExecuteContext(testCtx)) assert.Equal(2, processor.calls) rebuild := newDocumentsCmd(deps) @@ -666,13 +700,13 @@ func TestDocumentFullRebuildResumesDurableTargetSnapshot(t *testing.T) { documentBuildSubcommand, "--capabilities", manifestPath, "--full-rebuild", "--yes", "--limit", "1", }) - require.NoError(rebuild.ExecuteContext(t.Context())) + require.NoError(rebuild.ExecuteContext(testCtx)) assert.Contains(rebuildOutput.String(), "1 current owner(s) remaining") assert.Equal(3, processor.calls) - profile, err := configuredDocumentProfileOnly(manifestPath) + profile, err := configuredDocumentProfileOnly(manifestPath, invocationFromContext(testCtx)) require.NoError(err) privateRebuild, err := fixture.Store.GetActiveDocumentExtractionRebuild( - t.Context(), profile.ID, "original", + testCtx, profile.ID, "original", ) require.NoError(err) status := newDocumentsCmd(deps) @@ -680,7 +714,7 @@ func TestDocumentFullRebuildResumesDurableTargetSnapshot(t *testing.T) { status.SetOut(&statusOutput) status.SetErr(&bytes.Buffer{}) status.SetArgs([]string{"status", "--capabilities", manifestPath, "--json"}) - require.NoError(status.ExecuteContext(t.Context())) + require.NoError(status.ExecuteContext(testCtx)) var structuredStatus documentStatusOutput require.NoError(json.Unmarshal(statusOutput.Bytes(), &structuredStatus)) require.NotNil(structuredStatus.ActiveRebuild) @@ -692,10 +726,10 @@ func TestDocumentFullRebuildResumesDurableTargetSnapshot(t *testing.T) { resume.SetOut(&resumeOutput) resume.SetErr(&bytes.Buffer{}) resume.SetArgs([]string{"resume", "--capabilities", manifestPath, "--limit", "1", "--yes"}) - require.NoError(resume.ExecuteContext(t.Context())) + require.NoError(resume.ExecuteContext(testCtx)) assert.Contains(resumeOutput.String(), "Full document rebuild completed") assert.Equal(4, processor.calls) - _, err = fixture.Store.GetActiveDocumentExtractionRebuild(t.Context(), profile.ID, "original") + _, err = fixture.Store.GetActiveDocumentExtractionRebuild(testCtx, profile.ID, "original") require.ErrorIs(err, store.ErrDocumentExtractionRebuildMissing) runs := operationRunsForKind(t, fixture.Store, operations.KindDocumentExtraction) require.Len(runs, 3, "incremental, rebuild, and resume are separate bounded passes") @@ -1052,11 +1086,15 @@ func TestScheduledDocumentReconcilePreservesExistingConsentWhenExtractionDisable } func TestProbeMistralCommandRequiresExplicitEnablementAndPosture(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) previousConfig := cfg t.Cleanup(func() { cfg = previousConfig }) cfg = config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx providerCalled := false deps := documentsCommandDeps{ newMistralClient: func(*documentindex.DocumentsConfig) (*mistral.Client, error) { @@ -1066,15 +1104,17 @@ func TestProbeMistralCommandRequiresExplicitEnablementAndPosture(t *testing.T) { } command := newDocumentsCmd(deps) + command.SetContext(testCtx) command.SetArgs([]string{"probe-mistral", "--fixtures", "synthetic-fixtures"}) - err := command.ExecuteContext(t.Context()) + err := command.ExecuteContext(testCtx) require.ErrorContains(err, "enabled=true") assert.False(providerCalled) cfg.Attachments.Documents.Enabled = true command = newDocumentsCmd(deps) + command.SetContext(testCtx) command.SetArgs([]string{"probe-mistral", "--fixtures", "synthetic-fixtures"}) - err = command.ExecuteContext(t.Context()) + err = command.ExecuteContext(testCtx) require.ErrorContains(err, "explicit retention_posture and training_posture") assert.False(providerCalled) } @@ -1214,3 +1254,65 @@ func commandMistralResult(markdown string) mistral.Result { Metrics: mistral.RequestMetrics{Requests: 1, Latency: time.Millisecond}, } } + +func TestInlineDocumentConsentRescansAndReportsMalformedSource(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + markDaemonCLISubprocessForTest(t) + cfg := testConfigValue() + cfg.Data.DataDir = t.TempDir() + cfg.Attachments.Documents.Enabled = true + cfg.Attachments.Documents.RetentionPosture = documentindex.RetentionStandard + cfg.Attachments.Documents.TrainingPosture = documentindex.TrainingOptedOut + testCtx := withTestConfig(t, cfg) + fixture := storetest.New(t) + content := []byte("%PDF-1.4\nsynthetic malformed PDF\n") + sum := sha256.Sum256(content) + hash := hex.EncodeToString(sum[:]) + msg := fixture.CreateMessage("inline-malformed-document") + require.NoError(fixture.Store.UpsertAttachmentRecord(t.Context(), msg, store.AttachmentWrite{Filename: "synthetic.pdf", MIMEType: "application/pdf", Size: int64(len(content)), StoragePath: hash[:2] + "/" + hash, ContentHash: hash, Role: store.AttachmentRoleInline, RoleSource: store.AttachmentRoleSourceRawMIMERepair})) + manifestPath := writeCommandCapabilityManifest(t, cfg.Attachments.Documents.MaxPagesPerDocument) + processor := &commandBuildProcessor{} + deps := documentsCommandDeps{ + newMistralProcessor: func(*documentindex.DocumentsConfig) (documentindex.MistralProcessor, error) { return processor, nil }, + openStore: func(context.Context) (*store.Store, func(), error) { return fixture.Store, func() {}, nil }, + openAttachments: func(context.Context, *store.Store) (documentindex.DocumentAttachmentOpener, func() error, error) { + return commandAttachmentOpener{content: content}, func() error { return nil }, nil + }, + openReadClient: func(context.Context) (documentReadClient, func(), error) { + return localDocumentReadClient{store: fixture.Store}, func() {}, nil + }, + } + run := func(args ...string) (string, error) { + command := newDocumentsCmd(deps) + var output bytes.Buffer + command.SetOut(&output) + command.SetErr(&bytes.Buffer{}) + command.SetArgs(args) + err := command.ExecuteContext(testCtx) + return output.String(), err + } + _, err := run("consent-mistral", "--capabilities", manifestPath, "--yes") + require.NoError(err) + // Simulate an old completed journal bootstrap that never retained inline rows. + _, err = fixture.Store.DB().Exec("DELETE FROM document_occurrences") + require.NoError(err) + cfg.Attachments.Documents.Scope.IncludeInline = true + disclosure, err := run("consent-mistral", "--capabilities", manifestPath) + require.ErrorContains(err, "requires --yes") + assert.Contains(disclosure, "standalone and inline document attachments") + _, err = run("consent-mistral", "--capabilities", manifestPath, "--yes") + require.NoError(err) + assert.Equal(1, commandDocumentOccurrenceCount(t, fixture.Store)) + _, err = run("build", "--capabilities", manifestPath, "--yes") + require.ErrorContains(err, "PDF structure is malformed") + assert.Zero(processor.calls) + human, err := run("status", "--capabilities", manifestPath) + require.NoError(err) + assert.Contains(human, "invalid_local_source") + assert.Contains(human, "PDF structure is malformed") + encoded, err := run("status", "--capabilities", manifestPath, "--json") + require.NoError(err) + assert.Contains(encoded, `"reason_code":"invalid_local_source"`) + assert.Contains(encoded, "PDF structure is malformed") +} diff --git a/cmd/msgvault/cmd/documents_vector.go b/cmd/msgvault/cmd/documents_vector.go index 32515195a..673c56a46 100644 --- a/cmd/msgvault/cmd/documents_vector.go +++ b/cmd/msgvault/cmd/documents_vector.go @@ -35,6 +35,11 @@ func newDocumentVectorsCmd(deps documentsCommandDeps) *cobra.Command { } func desiredDocumentVectorSpec(ctx context.Context, st *store.Store) (store.DocumentVectorGenerationSpec, error) { + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil { + return store.DocumentVectorGenerationSpec{}, errors.New("configuration is unavailable") + } + cfg := state.cfg if cfg == nil || !cfg.Vector.Enabled || !cfg.Attachments.Documents.Index.Embeddings.Enabled { return store.DocumentVectorGenerationSpec{}, errors.New("document embeddings are disabled; enable [vector] and [attachments.documents.index.embeddings]") } @@ -42,6 +47,11 @@ func desiredDocumentVectorSpec(ctx context.Context, st *store.Store) (store.Docu } func configuredDocumentVectorSpec(ctx context.Context, st *store.Store) (store.DocumentVectorGenerationSpec, error) { + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil { + return store.DocumentVectorGenerationSpec{}, errors.New("configuration is unavailable") + } + cfg := state.cfg if cfg == nil || !cfg.Attachments.Documents.Index.Embeddings.Enabled { return store.DocumentVectorGenerationSpec{}, errors.New("document embeddings are not configured") } @@ -68,7 +78,12 @@ func configuredDocumentVectorSpec(ctx context.Context, st *store.Store) (store.D }, nil } -func configuredDocumentVectorConsentSpec(spec store.DocumentVectorGenerationSpec) (store.DocumentVectorConsentSpec, error) { +func configuredDocumentVectorConsentSpec(spec store.DocumentVectorGenerationSpec, state *invocation) (store.DocumentVectorConsentSpec, error) { + state = invocationState(context.Background(), state) + if state == nil || state.cfg == nil { + return store.DocumentVectorConsentSpec{}, errors.New("configuration is unavailable") + } + cfg := state.cfg egressFingerprint, err := vectordocument.EgressFingerprint(spec.TargetExtractionProfileID, cfg.Vector) if err != nil { return store.DocumentVectorConsentSpec{}, err @@ -80,7 +95,12 @@ func configuredDocumentVectorConsentSpec(spec store.DocumentVectorGenerationSpec }, nil } -func configuredDocumentVectorQueryConsentSpec(spec store.DocumentVectorGenerationSpec) (store.DocumentVectorConsentSpec, error) { +func configuredDocumentVectorQueryConsentSpec(spec store.DocumentVectorGenerationSpec, state *invocation) (store.DocumentVectorConsentSpec, error) { + state = invocationState(context.Background(), state) + if state == nil || state.cfg == nil { + return store.DocumentVectorConsentSpec{}, errors.New("configuration is unavailable") + } + cfg := state.cfg egressFingerprint, err := vectordocument.QueryEgressFingerprint(spec.TargetExtractionProfileID, cfg.Vector) if err != nil { return store.DocumentVectorConsentSpec{}, err @@ -92,11 +112,11 @@ func configuredDocumentVectorQueryConsentSpec(spec store.DocumentVectorGeneratio }, nil } -func withDocumentVectorStore(deps documentsCommandDeps, fn func(*store.Store) error) error { +func withDocumentVectorStore(ctx context.Context, deps documentsCommandDeps, fn func(*store.Store) error) error { if deps.openStore == nil { return errors.New("document vector ledger is unavailable") } - st, cleanup, err := deps.openStore() + st, cleanup, err := deps.openStore(ctx) if err != nil { return err } @@ -106,7 +126,7 @@ func withDocumentVectorStore(deps documentsCommandDeps, fn func(*store.Store) er func runDocumentVectorCommandHTTP(command *cobra.Command, args []string, forwardEmbeddingKey bool) error { if forwardEmbeddingKey { - return runDaemonCLICommandHTTPFromCobraWithEnv(command, args, embeddingsForwardEnv()) + return runDaemonCLICommandHTTPFromCobraWithEnv(command, args, embeddingsForwardEnv(invocationFromCommand(command))) } return runDaemonCLICommandHTTPFromCobra(command, args) } @@ -122,7 +142,7 @@ func newDocumentVectorConsentCmd(deps documentsCommandDeps) *cobra.Command { if !isDaemonCLISubprocess() { return runDocumentVectorCommandHTTP(command, args, false) } - return withDocumentVectorStore(deps, func(st *store.Store) error { + return withDocumentVectorStore(command.Context(), deps, func(st *store.Store) error { spec, err := configuredDocumentVectorSpec(command.Context(), st) if err != nil { return err @@ -130,16 +150,16 @@ func newDocumentVectorConsentCmd(deps documentsCommandDeps) *cobra.Command { var consentSpec store.DocumentVectorConsentSpec switch purpose { case "documents": - consentSpec, err = configuredDocumentVectorConsentSpec(spec) + consentSpec, err = configuredDocumentVectorConsentSpec(spec, invocationFromCommand(command)) case "queries": - consentSpec, err = configuredDocumentVectorQueryConsentSpec(spec) + consentSpec, err = configuredDocumentVectorQueryConsentSpec(spec, invocationFromCommand(command)) default: return errors.New("document vector consent purpose must be documents or queries") } if err != nil { return err } - printDocumentVectorConsentDisclosure(command.OutOrStdout(), consentSpec) + printDocumentVectorConsentDisclosure(command.OutOrStdout(), consentSpec, invocationFromCommand(command)) if !yes { return errors.New("hosted document embedding consent requires --yes after reviewing the provider disclosure") } @@ -157,7 +177,12 @@ func newDocumentVectorConsentCmd(deps documentsCommandDeps) *cobra.Command { return command } -func printDocumentVectorConsentDisclosure(w io.Writer, spec store.DocumentVectorConsentSpec) { +func printDocumentVectorConsentDisclosure(w io.Writer, spec store.DocumentVectorConsentSpec, state *invocation) { + state = invocationState(context.Background(), state) + if state == nil || state.cfg == nil { + return + } + cfg := state.cfg authentication := "no authentication environment variable configured" if cfg.Vector.Embeddings.APIKeyEnv != "" { authentication = "environment variable " + cfg.Vector.Embeddings.APIKeyEnv @@ -203,7 +228,7 @@ func newDocumentVectorBuildCmd(deps documentsCommandDeps, resume bool) *cobra.Co if !isDaemonCLISubprocess() { return runDocumentVectorCommandHTTP(command, args, true) } - return withDocumentVectorStore(deps, func(st *store.Store) error { + return withDocumentVectorStore(command.Context(), deps, func(st *store.Store) error { return st.WithDocumentVectorOperationLock(command.Context(), func() error { if resume { generation, err := st.GetDocumentVectorGeneration(command.Context(), generationID) @@ -264,7 +289,7 @@ func requireDocumentVectorConsent(ctx context.Context, st *store.Store, spec sto } func hasDocumentVectorConsent(ctx context.Context, st *store.Store, spec store.DocumentVectorGenerationSpec) (bool, error) { - consentSpec, err := configuredDocumentVectorConsentSpec(spec) + consentSpec, err := configuredDocumentVectorConsentSpec(spec, invocationFromContext(ctx)) if err != nil { return false, err } @@ -298,7 +323,7 @@ func newDocumentVectorRetryCmd(deps documentsCommandDeps) *cobra.Command { if !isDaemonCLISubprocess() { return runDocumentVectorCommandHTTP(command, args, false) } - return withDocumentVectorStore(deps, func(st *store.Store) error { + return withDocumentVectorStore(command.Context(), deps, func(st *store.Store) error { return st.WithDocumentVectorOperationLock(command.Context(), func() error { result, err := st.ResetDocumentVectorFailures(command.Context(), generationID, afterToken, limit, time.Now()) if err != nil { @@ -327,7 +352,7 @@ func newDocumentVectorRebuildCmd(deps documentsCommandDeps) *cobra.Command { if !yes { return errors.New("document vector rebuild requires --yes") } - return withDocumentVectorStore(deps, func(st *store.Store) error { + return withDocumentVectorStore(command.Context(), deps, func(st *store.Store) error { return st.WithDocumentVectorOperationLock(command.Context(), func() error { spec, err := desiredDocumentVectorSpec(command.Context(), st) if err != nil { @@ -362,7 +387,7 @@ func newDocumentVectorRetireCmd(deps documentsCommandDeps) *cobra.Command { if !yes { return errors.New("document vector retirement requires --yes") } - return withDocumentVectorStore(deps, func(st *store.Store) error { + return withDocumentVectorStore(command.Context(), deps, func(st *store.Store) error { return st.WithDocumentVectorOperationLock(command.Context(), func() error { retired, err := st.RetireDocumentVectorGeneration(command.Context(), generationID, time.Now()) if err != nil { @@ -389,14 +414,15 @@ func newDocumentVectorStatusCmd(deps documentsCommandDeps) *cobra.Command { if !isDaemonCLISubprocess() { return runDocumentVectorCommandHTTP(command, args, false) } - if cfg == nil || !cfg.Vector.Enabled || !cfg.Attachments.Documents.Index.Embeddings.Enabled { + state := invocationFromCommand(command) + if state == nil || state.cfg == nil || !state.cfg.Vector.Enabled || !state.cfg.Attachments.Documents.Index.Embeddings.Enabled { if jsonOutput { return json.MarshalEncode(jsontext.NewEncoder(command.OutOrStdout()), map[string]bool{"enabled": false}, json.Deterministic(true)) } _, _ = fmt.Fprintln(command.OutOrStdout(), "document_vectors=disabled") return nil } - return withDocumentVectorStore(deps, func(st *store.Store) error { + return withDocumentVectorStore(command.Context(), deps, func(st *store.Store) error { spec, err := desiredDocumentVectorSpec(command.Context(), st) if errors.Is(err, store.ErrDocumentVectorInvalidGenerationState) { if jsonOutput { @@ -408,11 +434,11 @@ func newDocumentVectorStatusCmd(deps documentsCommandDeps) *cobra.Command { if err != nil { return err } - documentConsentSpec, err := configuredDocumentVectorConsentSpec(spec) + documentConsentSpec, err := configuredDocumentVectorConsentSpec(spec, invocationFromCommand(command)) if err != nil { return err } - queryConsentSpec, err := configuredDocumentVectorQueryConsentSpec(spec) + queryConsentSpec, err := configuredDocumentVectorQueryConsentSpec(spec, invocationFromCommand(command)) if err != nil { return err } diff --git a/cmd/msgvault/cmd/documents_vector_cleanup_sqlitevec_test.go b/cmd/msgvault/cmd/documents_vector_cleanup_sqlitevec_test.go index b4bce15c0..15862c78f 100644 --- a/cmd/msgvault/cmd/documents_vector_cleanup_sqlitevec_test.go +++ b/cmd/msgvault/cmd/documents_vector_cleanup_sqlitevec_test.go @@ -17,18 +17,19 @@ import ( func TestRunConfiguredDocumentVectorGenerationCleansRetiredWhenEmbeddingsDisabled(t *testing.T) { assert := assert.New(t) require := require.New(t) - fixture, spec := documentVectorCommandFixture(t) + fixture, spec, testCtx := documentVectorCommandFixture(t) + cfg := invocationFromContext(testCtx).cfg vectorPath := filepath.Join(t.TempDir(), "vectors.db") cfg.Vector.DBPath = vectorPath - generation, _, err := fixture.Store.EnsureDocumentVectorGeneration(t.Context(), spec) + generation, _, err := fixture.Store.EnsureDocumentVectorGeneration(testCtx, spec) require.NoError(err) token := strings.Repeat("9", 64) - backend, err := sqlitevec.Open(t.Context(), sqlitevec.Options{ + backend, err := sqlitevec.Open(testCtx, sqlitevec.Options{ Path: vectorPath, Dimension: spec.Dimension, }) require.NoError(err) - require.NoError(backend.DocumentBackend().PutUnpublished(t.Context(), vectordocument.GenerationID(generation.ID), spec.Dimension, []vectordocument.Embedding{{ + require.NoError(backend.DocumentBackend().PutUnpublished(testCtx, vectordocument.GenerationID(generation.ID), spec.Dimension, []vectordocument.Embedding{{ Token: token, Vector: []float32{1, 0, 0}, }})) require.NoError(backend.Close()) @@ -41,20 +42,20 @@ func TestRunConfiguredDocumentVectorGenerationCleansRetiredWhenEmbeddingsDisable "disabled-cleanup-extraction", spec.TargetExtractionProfileID, strings.Repeat("a", 64), "original", 1, "disabled-cleanup-chunk", "disabled-cleanup-checksum", 1, token) require.NoError(err) - retired, err := fixture.Store.RetireDocumentVectorGeneration(t.Context(), generation.ID, time.Now()) + retired, err := fixture.Store.RetireDocumentVectorGeneration(testCtx, generation.ID, time.Now()) require.NoError(err) require.True(retired) cfg.Vector.Enabled = false - result, err := runConfiguredDocumentVectorGeneration(t.Context(), fixture.Store, generation.ID, 1) + result, err := runConfiguredDocumentVectorGeneration(testCtx, fixture.Store, generation.ID, 1) require.NoError(err) assert.True(result.Purged) assert.True(result.Converged) - _, err = fixture.Store.GetDocumentVectorGeneration(t.Context(), generation.ID) + _, err = fixture.Store.GetDocumentVectorGeneration(testCtx, generation.ID) require.ErrorContains(err, "not found") - backend, err = sqlitevec.Open(t.Context(), sqlitevec.Options{Path: vectorPath}) + backend, err = sqlitevec.Open(testCtx, sqlitevec.Options{Path: vectorPath}) require.NoError(err) t.Cleanup(func() { _ = backend.Close() }) var remaining int diff --git a/cmd/msgvault/cmd/documents_vector_runtime.go b/cmd/msgvault/cmd/documents_vector_runtime.go index 7af7ede13..673d76d44 100644 --- a/cmd/msgvault/cmd/documents_vector_runtime.go +++ b/cmd/msgvault/cmd/documents_vector_runtime.go @@ -87,6 +87,11 @@ func documentVectorRunResultFromOperationRun(run *operations.Run) (vectordocumen } func runConfiguredDocumentVectorGeneration(ctx context.Context, st *store.Store, generationID int64, limit int) (vectordocument.ReconcileResult, error) { + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil { + return vectordocument.ReconcileResult{}, errors.New("configuration is unavailable") + } + cfg := state.cfg if limit < 1 || limit > 1000 { return vectordocument.ReconcileResult{}, errors.New("document vector operation limit must be between 1 and 1000") } @@ -118,6 +123,11 @@ func runConfiguredDocumentVectorGeneration(ctx context.Context, st *store.Store, } func openDocumentVectorCleanupBackend(ctx context.Context, st *store.Store, mainPath string) (vectordocument.Backend, func() error, error) { + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil { + return nil, nil, errors.New("configuration is unavailable") + } + cfg := state.cfg if store.IsPostgresURL(mainPath) { backend, err := pgvector.DocumentBackendForDB(st.DB()) if err != nil { @@ -140,6 +150,11 @@ func runDocumentVectorWithFeatures( ctx context.Context, st *store.Store, vf *vectorFeatures, generationID int64, limit int, scope operations.PassScope, ) (vectordocument.ReconcileResult, error) { + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil { + return vectordocument.ReconcileResult{}, errors.New("configuration is unavailable") + } + cfg := state.cfg limit = min(limit, max(1, vf.Cfg.Embeddings.BatchSize)) generation, err := st.GetDocumentVectorGeneration(ctx, generationID) if err != nil { diff --git a/cmd/msgvault/cmd/documents_vector_stub_test.go b/cmd/msgvault/cmd/documents_vector_stub_test.go index d685cbcb2..61e2d2c32 100644 --- a/cmd/msgvault/cmd/documents_vector_stub_test.go +++ b/cmd/msgvault/cmd/documents_vector_stub_test.go @@ -14,17 +14,21 @@ import ( ) func TestDocumentVectorStubKeepsStatusAvailableAndBuildActionable(t *testing.T) { + cfg := testConfigValue() + previous := cfg t.Cleanup(func() { cfg = previous }) t.Setenv(daemonCLISubprocessEnv, strconv.Itoa(os.Getppid())) cfg = config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx status := newDocumentsCmd(documentsCommandDeps{}) var output bytes.Buffer status.SetOut(&output) status.SetArgs([]string{documentVectorsSubcommand, statusValue, "--json"}) - require.NoError(t, status.ExecuteContext(t.Context())) + require.NoError(t, status.ExecuteContext(testCtx)) assert.JSONEq(t, `{"enabled":false}`, output.String()) - _, err := runConfiguredDocumentVectorGeneration(t.Context(), nil, 1, 1) + _, err := runConfiguredDocumentVectorGeneration(testCtx, nil, 1, 1) require.ErrorContains(t, err, "rebuild with sqlite_vec or pgvector support") } diff --git a/cmd/msgvault/cmd/documents_vector_test.go b/cmd/msgvault/cmd/documents_vector_test.go index 3e9c35de7..05ed0ec49 100644 --- a/cmd/msgvault/cmd/documents_vector_test.go +++ b/cmd/msgvault/cmd/documents_vector_test.go @@ -27,11 +27,12 @@ import ( func TestDocumentVectorLedgerCommandsNeverOpenRuntime(t *testing.T) { assert := assert.New(t) require := require.New(t) - fixture, spec := documentVectorCommandFixture(t) + fixture, spec, testCtx := documentVectorCommandFixture(t) + cfg := invocationFromContext(testCtx).cfg t.Setenv("SYNTHETIC_EMBEDDING_KEY", "secret-that-must-not-print") runtimeCalls := 0 deps := documentsCommandDeps{ - openStore: func() (*store.Store, func(), error) { return fixture.Store, func() {}, nil }, + openStore: func(context.Context) (*store.Store, func(), error) { return fixture.Store, func() {}, nil }, runDocumentVector: func(context.Context, *store.Store, int64, int) (vectordocument.ReconcileResult, error) { runtimeCalls++ return vectordocument.ReconcileResult{}, nil @@ -42,7 +43,7 @@ func TestDocumentVectorLedgerCommandsNeverOpenRuntime(t *testing.T) { var unconfirmedOutput bytes.Buffer unconfirmed.SetOut(&unconfirmedOutput) unconfirmed.SetArgs([]string{documentVectorsSubcommand, "consent"}) - require.ErrorContains(unconfirmed.ExecuteContext(t.Context()), "--yes") + require.ErrorContains(unconfirmed.ExecuteContext(testCtx), "--yes") assert.Contains(unconfirmedOutput.String(), "Hosted document embedding disclosure:") assert.Contains(unconfirmedOutput.String(), "Destination: https://embeddings.example.test/v1") assert.Contains(unconfirmedOutput.String(), "Authentication: environment variable SYNTHETIC_EMBEDDING_KEY") @@ -53,9 +54,9 @@ func TestDocumentVectorLedgerCommandsNeverOpenRuntime(t *testing.T) { assert.Contains(unconfirmedOutput.String(), "Docbank-prepared normalized attachment document inputs will be sent") assert.NotContains(unconfirmedOutput.String(), "Explicit semantic or hybrid document searches") assert.NotContains(unconfirmedOutput.String(), "secret-that-must-not-print") - consentSpec, err := configuredDocumentVectorConsentSpec(spec) + consentSpec, err := configuredDocumentVectorConsentSpec(spec, invocationFromContext(testCtx)) require.NoError(err) - unconfirmedConsent, err := fixture.Store.GetDocumentVectorConsent(t.Context(), consentSpec.EgressFingerprint) + unconfirmedConsent, err := fixture.Store.GetDocumentVectorConsent(testCtx, consentSpec.EgressFingerprint) require.NoError(err) assert.Nil(unconfirmedConsent) @@ -63,46 +64,46 @@ func TestDocumentVectorLedgerCommandsNeverOpenRuntime(t *testing.T) { var consentOutput bytes.Buffer consent.SetOut(&consentOutput) consent.SetArgs([]string{documentVectorsSubcommand, "consent", "--yes"}) - require.NoError(consent.ExecuteContext(t.Context())) + require.NoError(consent.ExecuteContext(testCtx)) assert.Contains(consentOutput.String(), "Hosted document embedding disclosure:") assert.Contains(consentOutput.String(), "Recorded consent for document vector egress fingerprint "+consentSpec.EgressFingerprint) assert.NotContains(consentOutput.String(), "secret-that-must-not-print") - recorded, err := fixture.Store.GetDocumentVectorConsent(t.Context(), consentSpec.EgressFingerprint) + recorded, err := fixture.Store.GetDocumentVectorConsent(testCtx, consentSpec.EgressFingerprint) require.NoError(err) require.NotNil(recorded) assert.Equal(spec, recorded.DocumentVectorGenerationSpec) assert.Equal("document_embedding", recorded.Purpose) - queryConsentSpec, err := configuredDocumentVectorQueryConsentSpec(spec) + queryConsentSpec, err := configuredDocumentVectorQueryConsentSpec(spec, invocationFromContext(testCtx)) require.NoError(err) assert.NotEqual(consentSpec.EgressFingerprint, queryConsentSpec.EgressFingerprint) queryConsent := newDocumentsCmd(deps) var queryConsentOutput bytes.Buffer queryConsent.SetOut(&queryConsentOutput) queryConsent.SetArgs([]string{documentVectorsSubcommand, "consent", "--purpose", "queries", "--yes"}) - require.NoError(queryConsent.ExecuteContext(t.Context())) + require.NoError(queryConsent.ExecuteContext(testCtx)) assert.Contains(queryConsentOutput.String(), "Explicit semantic or hybrid document searches will send query text") - recordedQuery, err := fixture.Store.GetDocumentVectorConsent(t.Context(), queryConsentSpec.EgressFingerprint) + recordedQuery, err := fixture.Store.GetDocumentVectorConsent(testCtx, queryConsentSpec.EgressFingerprint) require.NoError(err) require.NotNil(recordedQuery) assert.Equal("query_embedding", recordedQuery.Purpose) consentedEndpoint := cfg.Vector.Embeddings.Endpoint cfg.Vector.Embeddings.Endpoint = "https://hosted.example.test/v1" - changedConsentSpec, err := configuredDocumentVectorConsentSpec(spec) + changedConsentSpec, err := configuredDocumentVectorConsentSpec(spec, invocationFromContext(testCtx)) require.NoError(err) assert.NotEqual(consentSpec.EgressFingerprint, changedConsentSpec.EgressFingerprint) - require.ErrorContains(requireDocumentVectorConsent(t.Context(), fixture.Store, spec), "not consented") + require.ErrorContains(requireDocumentVectorConsent(testCtx, fixture.Store, spec), "not consented") cfg.Vector.Embeddings.Endpoint = consentedEndpoint - require.NoError(requireDocumentVectorConsent(t.Context(), fixture.Store, spec)) + require.NoError(requireDocumentVectorConsent(testCtx, fixture.Store, spec)) - generation, _, err := fixture.Store.EnsureDocumentVectorGeneration(t.Context(), spec) + generation, _, err := fixture.Store.EnsureDocumentVectorGeneration(testCtx, spec) require.NoError(err) status := newDocumentsCmd(deps) var statusOutput bytes.Buffer status.SetOut(&statusOutput) status.SetArgs([]string{documentVectorsSubcommand, statusValue}) - require.NoError(status.ExecuteContext(t.Context())) + require.NoError(status.ExecuteContext(testCtx)) assert.Contains(statusOutput.String(), "building_generation=") assert.Contains(statusOutput.String(), "state=building") assert.Contains(statusOutput.String(), "pending=0 retryable=0 terminal=0 ready_live=0 obsolete=0 cleanup_pending=0") @@ -111,26 +112,27 @@ func TestDocumentVectorLedgerCommandsNeverOpenRuntime(t *testing.T) { retry := newDocumentsCmd(deps) retry.SetOut(&bytes.Buffer{}) retry.SetArgs([]string{documentVectorsSubcommand, "retry", "--generation-id", "999", "--limit", "1"}) - require.Error(retry.ExecuteContext(t.Context())) + require.Error(retry.ExecuteContext(testCtx)) retire := newDocumentsCmd(deps) retire.SetOut(&bytes.Buffer{}) retire.SetArgs([]string{documentVectorsSubcommand, cliEmbeddingsOperationRetire, "--generation-id", fmtInt64(generation.ID), "--yes"}) - require.NoError(retire.ExecuteContext(t.Context())) + require.NoError(retire.ExecuteContext(testCtx)) assert.Zero(runtimeCalls) } func TestSetupStatusTracksBothDocumentVectorConsentPurposes(t *testing.T) { assert := assert.New(t) require := require.New(t) - fixture, _ := documentVectorCommandFixture(t) + fixture, _, testCtx := documentVectorCommandFixture(t) + cfg := invocationFromContext(testCtx).cfg cfg.Attachments.Documents.Enabled = true deps := documentsCommandDeps{ - openStore: func() (*store.Store, func(), error) { return fixture.Store, func() {}, nil }, + openStore: func(context.Context) (*store.Store, func(), error) { return fixture.Store, func() {}, nil }, } env := setupEnvironment{ lookupEnv: func(string) (string, bool) { return "synthetic-key", true }, - consent: setupConsentFromStore(t.Context(), cfg, fixture.Store), + consent: setupConsentFromStore(testCtx, cfg, fixture.Store), } lane := documentVectorsLane(cfg, env) assert.Equal(laneStatePending, lane.State) @@ -141,8 +143,8 @@ func TestSetupStatusTracksBothDocumentVectorConsentPurposes(t *testing.T) { var output bytes.Buffer command.SetOut(&output) command.SetArgs([]string{"vectors", "consent", "--purpose", purpose, "--yes"}) - require.NoError(command.ExecuteContext(t.Context()), output.String()) - env.consent = setupConsentFromStore(t.Context(), cfg, fixture.Store) + require.NoError(command.ExecuteContext(testCtx), output.String()) + env.consent = setupConsentFromStore(testCtx, cfg, fixture.Store) lane = documentVectorsLane(cfg, env) assert.Equal(consentActive, lane.ConsentPurposes["document_embedding"]) if purpose == "documents" { @@ -157,17 +159,21 @@ func TestSetupStatusTracksBothDocumentVectorConsentPurposes(t *testing.T) { } } cfg.Vector.Embeddings.Endpoint = "https://changed.example.test/v1" - env.consent = setupConsentFromStore(t.Context(), cfg, fixture.Store) + env.consent = setupConsentFromStore(testCtx, cfg, fixture.Store) lane = documentVectorsLane(cfg, env) assert.Equal(laneStatePending, lane.State) assert.Equal(map[string]string{"document_embedding": consentMissing, "query_embedding": consentMissing}, lane.ConsentPurposes) } func TestDocumentVectorStatusWorksWhenEmbeddingsAreDisabled(t *testing.T) { + cfg := testConfigValue() + markDaemonCLISubprocessForTest(t) previous := cfg t.Cleanup(func() { cfg = previous }) cfg = config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx runtimeCalls := 0 command := newDocumentsCmd(documentsCommandDeps{ runDocumentVector: func(context.Context, *store.Store, int64, int) (vectordocument.ReconcileResult, error) { @@ -175,16 +181,18 @@ func TestDocumentVectorStatusWorksWhenEmbeddingsAreDisabled(t *testing.T) { return vectordocument.ReconcileResult{}, nil }, }) + command.SetContext(testCtx) var output bytes.Buffer command.SetOut(&output) command.SetArgs([]string{documentVectorsSubcommand, statusValue, "--json"}) - require.NoError(t, command.ExecuteContext(t.Context())) + require.NoError(t, command.ExecuteContext(testCtx)) assert.JSONEq(t, `{"enabled":false}`, output.String()) assert.Zero(t, runtimeCalls) } func TestConfiguredDocumentVectorSpecRejectsInvalidDisabledEmbeddingPolicy(t *testing.T) { - fixture, _ := documentVectorCommandFixture(t) + fixture, _, testCtx := documentVectorCommandFixture(t) + cfg := invocationFromContext(testCtx).cfg base := *cfg tests := []struct { name string @@ -221,7 +229,8 @@ func TestConfiguredDocumentVectorSpecRejectsInvalidDisabledEmbeddingPolicy(t *te cfg = &changed var err error require.NotPanics(t, func() { - _, err = configuredDocumentVectorSpec(t.Context(), fixture.Store) + invocationFromContext(testCtx).cfg = &changed + _, err = configuredDocumentVectorSpec(testCtx, fixture.Store) }) require.ErrorContains(t, err, test.wantErr) }) @@ -229,10 +238,14 @@ func TestConfiguredDocumentVectorSpecRejectsInvalidDisabledEmbeddingPolicy(t *te } func TestDocumentVectorStatusWorksBeforeExtractionTargetExists(t *testing.T) { + cfg := testConfigValue() + markDaemonCLISubprocessForTest(t) previous := cfg t.Cleanup(func() { cfg = previous }) cfg = config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Vector.Enabled = true cfg.Vector.Embeddings.Endpoint = "https://embeddings.example.test/v1" cfg.Vector.Embeddings.Model = "embed-test" @@ -240,26 +253,27 @@ func TestDocumentVectorStatusWorksBeforeExtractionTargetExists(t *testing.T) { cfg.Attachments.Documents.Index.Embeddings.Enabled = true fixture := storetest.New(t) command := newDocumentsCmd(documentsCommandDeps{ - openStore: func() (*store.Store, func(), error) { return fixture.Store, func() {}, nil }, + openStore: func(context.Context) (*store.Store, func(), error) { return fixture.Store, func() {}, nil }, }) + command.SetContext(testCtx) var output bytes.Buffer command.SetOut(&output) command.SetArgs([]string{documentVectorsSubcommand, statusValue, "--json"}) - require.NoError(t, command.ExecuteContext(t.Context())) + require.NoError(t, command.ExecuteContext(testCtx)) assert.JSONEq(t, `{"enabled":true,"configured":false}`, output.String()) } func TestDocumentVectorProviderCommandsUseRuntimeAndValidateBounds(t *testing.T) { assert := assert.New(t) require := require.New(t) - fixture, spec := documentVectorCommandFixture(t) - consentSpec, err := configuredDocumentVectorConsentSpec(spec) + fixture, spec, testCtx := documentVectorCommandFixture(t) + consentSpec, err := configuredDocumentVectorConsentSpec(spec, invocationFromContext(testCtx)) require.NoError(err) - _, _, err = fixture.Store.RecordDocumentVectorConsent(t.Context(), consentSpec, time.Now()) + _, _, err = fixture.Store.RecordDocumentVectorConsent(testCtx, consentSpec, time.Now()) require.NoError(err) var calls []int64 deps := documentsCommandDeps{ - openStore: func() (*store.Store, func(), error) { return fixture.Store, func() {}, nil }, + openStore: func(context.Context) (*store.Store, func(), error) { return fixture.Store, func() {}, nil }, runDocumentVector: func(_ context.Context, _ *store.Store, generationID int64, _ int) (vectordocument.ReconcileResult, error) { calls = append(calls, generationID) return vectordocument.ReconcileResult{}, nil @@ -268,37 +282,38 @@ func TestDocumentVectorProviderCommandsUseRuntimeAndValidateBounds(t *testing.T) invalid := newDocumentsCmd(deps) invalid.SetArgs([]string{documentVectorsSubcommand, documentBuildSubcommand, "--limit", "0"}) - require.ErrorContains(invalid.ExecuteContext(t.Context()), "limit") + require.ErrorContains(invalid.ExecuteContext(testCtx), "limit") assert.Empty(calls) build := newDocumentsCmd(deps) build.SetOut(&bytes.Buffer{}) build.SetArgs([]string{documentVectorsSubcommand, documentBuildSubcommand, "--limit", "1"}) - require.NoError(build.ExecuteContext(t.Context())) + require.NoError(build.ExecuteContext(testCtx)) require.Len(calls, 1) - building, err := fixture.Store.GetBuildingDocumentVectorGeneration(t.Context()) + building, err := fixture.Store.GetBuildingDocumentVectorGeneration(testCtx) require.NoError(err) require.NotNil(building) resume := newDocumentsCmd(deps) resume.SetOut(&bytes.Buffer{}) resume.SetArgs([]string{documentVectorsSubcommand, cmdUseResume, "--generation-id", fmtInt64(building.ID), "--limit", "1"}) - require.NoError(resume.ExecuteContext(t.Context())) + require.NoError(resume.ExecuteContext(testCtx)) assert.Len(calls, 2) - require.NoError(fixture.Store.ActivateDocumentVectorGeneration(t.Context(), building.ID, time.Now())) + require.NoError(fixture.Store.ActivateDocumentVectorGeneration(testCtx, building.ID, time.Now())) rebuild := newDocumentsCmd(deps) rebuild.SetOut(&bytes.Buffer{}) rebuild.SetArgs([]string{documentVectorsSubcommand, "rebuild", "--generation-id", fmtInt64(building.ID), "--limit", "1", "--yes"}) - require.NoError(rebuild.ExecuteContext(t.Context())) + require.NoError(rebuild.ExecuteContext(testCtx)) assert.Len(calls, 3) } func TestDocumentVectorResumeRunsBoundedCleanupForRetiredGeneration(t *testing.T) { assert := assert.New(t) require := require.New(t) - fixture, spec := documentVectorCommandFixture(t) - generation, _, err := fixture.Store.EnsureDocumentVectorGeneration(t.Context(), spec) + fixture, spec, testCtx := documentVectorCommandFixture(t) + cfg := invocationFromContext(testCtx).cfg + generation, _, err := fixture.Store.EnsureDocumentVectorGeneration(testCtx, spec) require.NoError(err) token := strings.Repeat("8", 64) _, err = fixture.Store.DB().Exec(fixture.Store.Rebind(` @@ -309,13 +324,13 @@ func TestDocumentVectorResumeRunsBoundedCleanupForRetiredGeneration(t *testing.T "manual-cleanup-extraction", spec.TargetExtractionProfileID, strings.Repeat("a", 64), "original", 1, "manual-cleanup-chunk", "manual-cleanup-checksum", 1, token) require.NoError(err) - retired, err := fixture.Store.RetireDocumentVectorGeneration(t.Context(), generation.ID, time.Now()) + retired, err := fixture.Store.RetireDocumentVectorGeneration(testCtx, generation.ID, time.Now()) require.NoError(err) require.True(retired) backend := &commandDocumentVectorBackend{} deps := documentsCommandDeps{ - openStore: func() (*store.Store, func(), error) { return fixture.Store, func() {}, nil }, + openStore: func(context.Context) (*store.Store, func(), error) { return fixture.Store, func() {}, nil }, runDocumentVector: func(ctx context.Context, st *store.Store, generationID int64, limit int) (vectordocument.ReconcileResult, error) { return runDocumentVectorWithFeatures(ctx, st, &vectorFeatures{ DocumentBackend: backend, @@ -331,13 +346,13 @@ func TestDocumentVectorResumeRunsBoundedCleanupForRetiredGeneration(t *testing.T "--generation-id", fmtInt64(generation.ID), "--limit", "1", }) - require.NoError(command.ExecuteContext(t.Context())) + require.NoError(command.ExecuteContext(testCtx)) var result vectordocument.ReconcileResult require.NoError(json.Unmarshal(output.Bytes(), &result)) assert.True(result.Purged) assert.True(result.Converged) assert.Equal([][]string{{token}}, backend.deletes) - _, err = fixture.Store.GetDocumentVectorGeneration(t.Context(), generation.ID) + _, err = fixture.Store.GetDocumentVectorGeneration(testCtx, generation.ID) require.ErrorContains(err, "not found") } @@ -442,19 +457,20 @@ func (c *fakeDocumentVectorCheckpointer) CheckpointDocumentVectorBuildForFingerp func TestScheduledDocumentVectorRotationRetiresObsoleteBuildingBeforeDesiredBuild(t *testing.T) { assert := assert.New(t) require := require.New(t) - fixture, desired := documentVectorCommandFixture(t) - consentSpec, err := configuredDocumentVectorConsentSpec(desired) + fixture, desired, testCtx := documentVectorCommandFixture(t) + cfg := invocationFromContext(testCtx).cfg + consentSpec, err := configuredDocumentVectorConsentSpec(desired, invocationFromContext(testCtx)) require.NoError(err) - _, _, err = fixture.Store.RecordDocumentVectorConsent(t.Context(), consentSpec, time.Now()) + _, _, err = fixture.Store.RecordDocumentVectorConsent(testCtx, consentSpec, time.Now()) require.NoError(err) activeSpec := desired activeSpec.Fingerprint = strings.Repeat("1", 64) - active, _, err := fixture.Store.EnsureDocumentVectorGeneration(t.Context(), activeSpec) + active, _, err := fixture.Store.EnsureDocumentVectorGeneration(testCtx, activeSpec) require.NoError(err) - require.NoError(fixture.Store.ActivateDocumentVectorGeneration(t.Context(), active.ID, time.Now())) + require.NoError(fixture.Store.ActivateDocumentVectorGeneration(testCtx, active.ID, time.Now())) obsoleteSpec := desired obsoleteSpec.Fingerprint = strings.Repeat("2", 64) - obsolete, _, err := fixture.Store.EnsureDocumentVectorGeneration(t.Context(), obsoleteSpec) + obsolete, _, err := fixture.Store.EnsureDocumentVectorGeneration(testCtx, obsoleteSpec) require.NoError(err) for index := range 3 { _, err = fixture.Store.DB().Exec(fixture.Store.Rebind(` @@ -473,19 +489,19 @@ func TestScheduledDocumentVectorRotationRetiresObsoleteBuildingBeforeDesiredBuil DocumentBackend: backend, SemanticClient: client, Cfg: cfg.Vector, } - require.NoError(runScheduledDocumentVectorGeneration(t.Context(), fixture.Store, vf, 2)) - stillActive, err := fixture.Store.GetActiveDocumentVectorGeneration(t.Context()) + require.NoError(runScheduledDocumentVectorGeneration(testCtx, fixture.Store, vf, 2)) + stillActive, err := fixture.Store.GetActiveDocumentVectorGeneration(testCtx) require.NoError(err) require.NotNil(stillActive) assert.Equal(active.ID, stillActive.ID) - retired, err := fixture.Store.GetDocumentVectorGeneration(t.Context(), obsolete.ID) + retired, err := fixture.Store.GetDocumentVectorGeneration(testCtx, obsolete.ID) require.NoError(err) assert.Equal(store.DocumentVectorGenerationRetired, retired.State) require.Len(backend.deletes, 1) assert.Len(backend.deletes[0], 2) - require.NoError(runScheduledDocumentVectorGeneration(t.Context(), fixture.Store, vf, 2)) - _, err = fixture.Store.GetDocumentVectorGeneration(t.Context(), obsolete.ID) + require.NoError(runScheduledDocumentVectorGeneration(testCtx, fixture.Store, vf, 2)) + _, err = fixture.Store.GetDocumentVectorGeneration(testCtx, obsolete.ID) require.ErrorContains(err, "not found") require.Len(backend.deletes, 2) assert.Len(backend.deletes[1], 1) @@ -493,19 +509,40 @@ func TestScheduledDocumentVectorRotationRetiresObsoleteBuildingBeforeDesiredBuil require.NoError(err) assert.Equal(active.ID, stillActive.ID) - require.NoError(runScheduledDocumentVectorGeneration(t.Context(), fixture.Store, vf, 2)) - newActive, err := fixture.Store.GetActiveDocumentVectorGeneration(t.Context()) + require.NoError(runScheduledDocumentVectorGeneration(testCtx, fixture.Store, vf, 2)) + newActive, err := fixture.Store.GetActiveDocumentVectorGeneration(testCtx) require.NoError(err) require.NotNil(newActive) assert.Equal(desired, newActive.DocumentVectorGenerationSpec) assert.Zero(client.documentCalls) } +func TestRegisteredDocumentVectorJobCarriesInvocation(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + fixture, _, testCtx := documentVectorCommandFixture(t) + state := invocationFromContext(testCtx) + state.logger = testDiscardLogger() + features := &vectorFeatures{ + DocumentBackend: &commandDocumentVectorBackend{}, + SemanticClient: &commandDocumentSemanticClient{}, + Cfg: state.cfg.Vector, + } + capture := ®isteredDocumentVectorJobCapture{} + require.NoError(registerDocumentVectorJob(capture, features, fixture.Store, state)) + require.NotNil(capture.job) + require.NoError(capture.job(context.Background())) + active, err := fixture.Store.GetActiveDocumentVectorGeneration(testCtx) + require.NoError(err) + assert.Nil(active) +} + func TestScheduledDocumentVectorCleansRetiredWithoutConsentOrProvider(t *testing.T) { assert := assert.New(t) require := require.New(t) - fixture, spec := documentVectorCommandFixture(t) - generation, _, err := fixture.Store.EnsureDocumentVectorGeneration(t.Context(), spec) + fixture, spec, testCtx := documentVectorCommandFixture(t) + cfg := invocationFromContext(testCtx).cfg + generation, _, err := fixture.Store.EnsureDocumentVectorGeneration(testCtx, spec) require.NoError(err) token := strings.Repeat("9", 64) _, err = fixture.Store.DB().Exec(fixture.Store.Rebind(` @@ -516,42 +553,43 @@ func TestScheduledDocumentVectorCleansRetiredWithoutConsentOrProvider(t *testing "retired-extraction", spec.TargetExtractionProfileID, strings.Repeat("a", 64), "original", 1, "retired-chunk", "retired-checksum", 1, token) require.NoError(err) - retired, err := fixture.Store.RetireDocumentVectorGeneration(t.Context(), generation.ID, time.Now()) + retired, err := fixture.Store.RetireDocumentVectorGeneration(testCtx, generation.ID, time.Now()) require.NoError(err) require.True(retired) backend := &commandDocumentVectorBackend{} vf := &vectorFeatures{DocumentBackend: backend, Cfg: cfg.Vector} - require.NoError(runScheduledDocumentVectorGeneration(t.Context(), fixture.Store, vf, 10)) + require.NoError(runScheduledDocumentVectorGeneration(testCtx, fixture.Store, vf, 10)) assert.Equal([][]string{{token}}, backend.deletes) - _, err = fixture.Store.GetDocumentVectorGeneration(t.Context(), generation.ID) + _, err = fixture.Store.GetDocumentVectorGeneration(testCtx, generation.ID) require.ErrorContains(err, "not found") - require.NoError(runScheduledDocumentVectorGeneration(t.Context(), fixture.Store, vf, 10)) + require.NoError(runScheduledDocumentVectorGeneration(testCtx, fixture.Store, vf, 10)) } func TestScheduledDocumentVectorObservesConsentRecordedAfterRuntimeInitialization(t *testing.T) { assert := assert.New(t) require := require.New(t) - fixture, spec := documentVectorCommandFixture(t) + fixture, spec, testCtx := documentVectorCommandFixture(t) + cfg := invocationFromContext(testCtx).cfg vf := &vectorFeatures{ DocumentBackend: &commandDocumentVectorBackend{}, SemanticClient: &commandDocumentSemanticClient{}, Cfg: cfg.Vector, } - require.NoError(runScheduledDocumentVectorGeneration(t.Context(), fixture.Store, vf, 10)) - active, err := fixture.Store.GetActiveDocumentVectorGeneration(t.Context()) + require.NoError(runScheduledDocumentVectorGeneration(testCtx, fixture.Store, vf, 10)) + active, err := fixture.Store.GetActiveDocumentVectorGeneration(testCtx) require.NoError(err) assert.Nil(active) - consentSpec, err := configuredDocumentVectorConsentSpec(spec) + consentSpec, err := configuredDocumentVectorConsentSpec(spec, invocationFromContext(testCtx)) require.NoError(err) - _, _, err = fixture.Store.RecordDocumentVectorConsent(t.Context(), consentSpec, time.Now()) + _, _, err = fixture.Store.RecordDocumentVectorConsent(testCtx, consentSpec, time.Now()) require.NoError(err) - require.NoError(runScheduledDocumentVectorGeneration(t.Context(), fixture.Store, vf, 10)) - active, err = fixture.Store.GetActiveDocumentVectorGeneration(t.Context()) + require.NoError(runScheduledDocumentVectorGeneration(testCtx, fixture.Store, vf, 10)) + active, err = fixture.Store.GetActiveDocumentVectorGeneration(testCtx) require.NoError(err) require.NotNil(active) assert.Equal(spec, active.DocumentVectorGenerationSpec) @@ -560,14 +598,15 @@ func TestScheduledDocumentVectorObservesConsentRecordedAfterRuntimeInitializatio func TestScheduledDocumentVectorCleansObsoleteActiveTokensAfterCoverageIsComplete(t *testing.T) { assert := assert.New(t) require := require.New(t) - fixture, desired := documentVectorCommandFixture(t) - consentSpec, err := configuredDocumentVectorConsentSpec(desired) + fixture, desired, testCtx := documentVectorCommandFixture(t) + cfg := invocationFromContext(testCtx).cfg + consentSpec, err := configuredDocumentVectorConsentSpec(desired, invocationFromContext(testCtx)) require.NoError(err) - _, _, err = fixture.Store.RecordDocumentVectorConsent(t.Context(), consentSpec, time.Now()) + _, _, err = fixture.Store.RecordDocumentVectorConsent(testCtx, consentSpec, time.Now()) require.NoError(err) - active, _, err := fixture.Store.EnsureDocumentVectorGeneration(t.Context(), desired) + active, _, err := fixture.Store.EnsureDocumentVectorGeneration(testCtx, desired) require.NoError(err) - require.NoError(fixture.Store.ActivateDocumentVectorGeneration(t.Context(), active.ID, time.Now())) + require.NoError(fixture.Store.ActivateDocumentVectorGeneration(testCtx, active.ID, time.Now())) token := strings.Repeat("c", 64) _, err = fixture.Store.DB().Exec(fixture.Store.Rebind(` INSERT INTO document_vector_publications @@ -577,10 +616,10 @@ func TestScheduledDocumentVectorCleansObsoleteActiveTokensAfterCoverageIsComplet "deleted-extraction", desired.TargetExtractionProfileID, strings.Repeat("d", 64), "original", 1, "deleted-chunk", "deleted-checksum", 1, token) require.NoError(err) - coverage, err := fixture.Store.GetDocumentVectorCoverage(t.Context(), active.ID) + coverage, err := fixture.Store.GetDocumentVectorCoverage(testCtx, active.ID) require.NoError(err) assert.True(coverage.Complete()) - status, err := fixture.Store.GetDocumentVectorGenerationStatus(t.Context(), active.ID, "", 10) + status, err := fixture.Store.GetDocumentVectorGenerationStatus(testCtx, active.ID, "", 10) require.NoError(err) assert.Equal(int64(1), status.CleanupPending) @@ -589,10 +628,10 @@ func TestScheduledDocumentVectorCleansObsoleteActiveTokensAfterCoverageIsComplet vf := &vectorFeatures{ DocumentBackend: backend, SemanticClient: client, Cfg: cfg.Vector, } - require.NoError(runScheduledDocumentVectorGeneration(t.Context(), fixture.Store, vf, 10)) + require.NoError(runScheduledDocumentVectorGeneration(testCtx, fixture.Store, vf, 10)) require.Equal([][]string{{token}}, backend.deletes) assert.Zero(client.documentCalls) - status, err = fixture.Store.GetDocumentVectorGenerationStatus(t.Context(), active.ID, "", 10) + status, err = fixture.Store.GetDocumentVectorGenerationStatus(testCtx, active.ID, "", 10) require.NoError(err) assert.Zero(status.CleanupPending) var publications int @@ -600,7 +639,7 @@ func TestScheduledDocumentVectorCleansObsoleteActiveTokensAfterCoverageIsComplet `SELECT COUNT(*) FROM document_vector_publications WHERE generation_id = ?`), active.ID).Scan(&publications)) assert.Zero(publications) - require.NoError(runScheduledDocumentVectorGeneration(t.Context(), fixture.Store, vf, 10)) + require.NoError(runScheduledDocumentVectorGeneration(testCtx, fixture.Store, vf, 10)) assert.Len(backend.deletes, 1, "a converged replay does not re-delete finalized tokens") assert.Zero(client.documentCalls) } @@ -631,11 +670,9 @@ func (*commandDocumentVectorBackend) Search(context.Context, vectordocument.Gene return nil, nil } -func documentVectorCommandFixture(t *testing.T) (*storetest.Fixture, store.DocumentVectorGenerationSpec) { +func documentVectorCommandFixture(t *testing.T) (*storetest.Fixture, store.DocumentVectorGenerationSpec, context.Context) { t.Helper() markDaemonCLISubprocessForTest(t) - previous := cfg - t.Cleanup(func() { cfg = previous }) c := config.NewDefaultConfig() c.Vector.Enabled = true c.Vector.Embeddings.Endpoint = "https://embeddings.example.test/v1" @@ -645,7 +682,7 @@ func documentVectorCommandFixture(t *testing.T) (*storetest.Fixture, store.Docum c.Vector.Embeddings.MaxInputChars = 4096 c.Attachments.Documents.Index.Embeddings.Enabled = true c.Attachments.Documents.Index.Embeddings.Profile = "vector.embeddings" - cfg = c + cfg := c fixture := storetest.New(t) fingerprint := strings.Repeat("7", 64) profile := store.DocumentExtractionProfile{ @@ -658,9 +695,10 @@ func documentVectorCommandFixture(t *testing.T) (*storetest.Fixture, store.Docum require.NoError(t, err) _, err = fixture.Store.DB().Exec(fixture.Store.Rebind(`UPDATE document_index_state SET target_profile_id = ? WHERE singleton = 1`), profile.ID) require.NoError(t, err) - spec, err := desiredDocumentVectorSpec(t.Context(), fixture.Store) + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + spec, err := desiredDocumentVectorSpec(testCtx, fixture.Store) require.NoError(t, err) - return fixture, spec + return fixture, spec, testCtx } func fmtInt64(value int64) string { return strconv.FormatInt(value, 10) } diff --git a/cmd/msgvault/cmd/draft_lifecycle.go b/cmd/msgvault/cmd/draft_lifecycle.go new file mode 100644 index 000000000..cf124b43e --- /dev/null +++ b/cmd/msgvault/cmd/draft_lifecycle.go @@ -0,0 +1,84 @@ +package cmd + +import ( + "errors" + + "github.com/spf13/cobra" +) + +func init() { + rootCmd.AddCommand(newDraftGetCommand()) + rootCmd.AddCommand(newDraftEditCommand()) + rootCmd.AddCommand(newDraftDeleteCommand()) + rootCmd.AddCommand(newDraftRecoverCommand()) + rootCmd.AddCommand(newDraftSendAsCommand()) +} + +func newDraftGetCommand() *cobra.Command { + command := &cobra.Command{ + Use: "draft-get ( | --conversation )", + Short: "Read a managed draft, or list a chat conversation's local drafts", + Args: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 && !cmd.Flags().Changed("conversation") { + return errors.New("requires a draft ID or --conversation") + } + return cobra.MaximumNArgs(1)(cmd, args) + }, + RunE: runDaemonCLICommandHTTPFromCobra, + } + command.Flags().Int64("conversation", 0, "list local drafts for this chat conversation") + command.Flags().Bool("json", false, "emit one JSON result") + return command +} + +func newDraftEditCommand() *cobra.Command { + command := &cobra.Command{ + Use: "draft-edit ", + Short: "Replace the body of a managed draft", + Args: cobra.ExactArgs(1), + RunE: runDaemonCLICommandHTTPFromCobra, + } + command.Flags().Int64("revision", 0, "current draft revision") + command.Flags().String("body", "", "replacement plain-text body") + _ = command.MarkFlagRequired("revision") + _ = command.MarkFlagRequired("body") + command.Flags().Bool("json", false, "emit one JSON result") + return command +} + +func newDraftDeleteCommand() *cobra.Command { + command := &cobra.Command{ + Use: "draft-delete ", + Short: "Delete a managed draft", + Args: cobra.ExactArgs(1), + RunE: runDaemonCLICommandHTTPFromCobra, + } + command.Flags().Int64("revision", 0, "current draft revision") + _ = command.MarkFlagRequired("revision") + command.Flags().Bool("json", false, "emit one JSON result") + return command +} + +func newDraftRecoverCommand() *cobra.Command { + command := &cobra.Command{ + Use: "draft-recover ", + Short: "Recover an interrupted managed IMAP draft edit or delete", + Args: cobra.ExactArgs(1), + RunE: runDaemonCLICommandHTTPFromCobra, + } + command.Flags().Int64("revision", 0, "current draft revision") + _ = command.MarkFlagRequired("revision") + command.Flags().Bool("json", false, "emit one JSON result") + return command +} + +func newDraftSendAsCommand() *cobra.Command { + command := &cobra.Command{ + Use: "draft-send-as ", + Short: "List Gmail send-as identities", + Args: cobra.ExactArgs(1), + RunE: runDaemonCLICommandHTTPFromCobra, + } + command.Flags().Bool("json", false, "emit one JSON result") + return command +} diff --git a/cmd/msgvault/cmd/embed.go b/cmd/msgvault/cmd/embed.go index 371f3ad32..4aa539fb1 100644 --- a/cmd/msgvault/cmd/embed.go +++ b/cmd/msgvault/cmd/embed.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "errors" "fmt" "os" @@ -20,7 +21,10 @@ var ( embeddingsActivateYes bool ) -const embeddingsCommandName = "embeddings" +const ( + embeddingsCommandName = "embeddings" + embeddingsOptimizeWorkerName = "__optimize-worker" +) var embeddingsCmd = &cobra.Command{ Use: embeddingsCommandName, @@ -99,6 +103,11 @@ func runEmbeddingsBuild(cmd *cobra.Command, args []string) error { } func runEmbeddingsBuildLocal(cmd *cobra.Command) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg if !cfg.Vector.Enabled { return errors.New("vector search not enabled; add [vector] enabled=true to config.toml first") } @@ -117,13 +126,19 @@ func runEmbeddingsBuildHTTP(cmd *cobra.Command, args []string) error { return fmt.Errorf("set --yes after confirmation: %w", err) } } - return runDaemonCLICommandHTTPFromCobraWithEnv(cmd, args, embeddingsForwardEnv()) + return runDaemonCLICommandHTTPFromCobraWithEnv(cmd, args, embeddingsForwardEnv(invocationFromCommand(cmd))) } // embeddingsForwardEnv carries the caller's embedding API key into the // daemon-spawned subprocess, which otherwise sees only the daemon's // environment: a key exported in the user's shell would silently not apply. -func embeddingsForwardEnv() map[string]string { + +func embeddingsForwardEnv(state *invocation) map[string]string { + state = invocationState(context.Background(), state) + if state == nil || state.cfg == nil { + return nil + } + cfg := state.cfg name := cfg.Vector.Embeddings.APIKeyEnv if name == "" { return nil @@ -172,6 +187,9 @@ func init() { embeddingsCmd.AddCommand(embeddingsRetireCmd) embeddingsCmd.AddCommand(embeddingsActivateCmd) embeddingsCmd.AddCommand(embeddingsPruneCmd) + embeddingsOptimizeCmd.Flags().Bool("drop", false, "Remove the accelerator while keeping exact vectors") + embeddingsCmd.AddCommand(embeddingsOptimizeCmd) + embeddingsCmd.AddCommand(embeddingsOptimizeWorkerCmd) rootCmd.AddCommand(embeddingsCmd) rootCmd.AddCommand(embedCmd) } diff --git a/cmd/msgvault/cmd/embed_manage_sqlitevec_test.go b/cmd/msgvault/cmd/embed_manage_sqlitevec_test.go index 779eb5918..b5a250c28 100644 --- a/cmd/msgvault/cmd/embed_manage_sqlitevec_test.go +++ b/cmd/msgvault/cmd/embed_manage_sqlitevec_test.go @@ -3,7 +3,6 @@ package cmd import ( - "context" "database/sql" "path/filepath" "testing" @@ -24,7 +23,7 @@ func TestRunEmbeddingsRetire_ForceActive(t *testing.T) { require := require.New(t) assert := assert.New(t) dbPath := newEmbeddingMetadataTestDBFile(t) - withEmbeddingCommandConfig(t, dbPath) + testCtx := withEmbeddingCommandConfig(t, dbPath) oldYes := embeddingsRetireYes oldForce := embeddingsRetireForceActive @@ -37,7 +36,7 @@ func TestRunEmbeddingsRetire_ForceActive(t *testing.T) { cmd := embeddingsRetireCmd oldCtx := cmd.Context() - cmd.SetContext(context.Background()) + cmd.SetContext(testCtx) t.Cleanup(func() { cmd.SetContext(oldCtx) }) require.NoError(runEmbeddingsRetire(cmd, []string{"1"}), @@ -53,13 +52,13 @@ func TestRunEmbeddingsRetire_ForceActive(t *testing.T) { func TestFillFullCoverageUsesEmbeddingScopeForEmbeddedCount(t *testing.T) { require := require.New(t) assert := assert.New(t) - ctx := context.Background() dataDir := t.TempDir() dbPath := newEmbeddingMetadataTestDBFileAt(t, filepath.Join(dataDir, "vectors.db")) seedMainDBWithScopedFullCoverageMessages(t, dataDir) - withEmbeddingCommandConfigDataDir(t, dbPath, dataDir) + testCtx, cfg := withEmbeddingCommandConfigDataDir(t, dbPath, dataDir) cfg.Vector.Embed.Scope.MessageTypes = []string{"sms"} + ctx := testCtx backend, closeBackend, err := openEmbeddingsBackend(ctx) require.NoError(err, "open embeddings backend") t.Cleanup(closeBackend) diff --git a/cmd/msgvault/cmd/embed_pg_test.go b/cmd/msgvault/cmd/embed_pg_test.go index 402931002..41fdf45d4 100644 --- a/cmd/msgvault/cmd/embed_pg_test.go +++ b/cmd/msgvault/cmd/embed_pg_test.go @@ -68,6 +68,8 @@ func seedGenWithEmbeddingsPG(t *testing.T, pgb *pgvector.Backend, ids ...int64) // raw-SQL helper that only updated state, leaving the retired gen's vectors in // the shared HNSW graph. func TestRunEmbeddingsRetire_PG_DeletesEmbeddings(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) @@ -85,13 +87,14 @@ func TestRunEmbeddingsRetire_PG_DeletesEmbeddings(t *testing.T) { embeddingsRetireYes, embeddingsRetireForceActive = savedYes, savedForce }() cfg = &config.Config{} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) cfg.Data.DatabaseURL = dsn cfg.Vector.Embeddings.Dimension = 4 embeddingsRetireYes = true embeddingsRetireForceActive = true cmd := &cobra.Command{} - cmd.SetContext(ctx) + cmd.SetContext(testCtx) var out bytes.Buffer cmd.SetOut(&out) require.NoError( @@ -111,6 +114,8 @@ func TestRunEmbeddingsRetire_PG_DeletesEmbeddings(t *testing.T) { // of cf-2: activating a new generation auto-retires the previously-active one, // and that auto-retire must delete the demoted generation's embeddings on PG. func TestRunEmbeddingsActivate_PG_AutoRetireDeletesPrevious(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) @@ -133,6 +138,7 @@ func TestRunEmbeddingsActivate_PG_AutoRetireDeletesPrevious(t *testing.T) { embeddingsActivateYes, embeddingsActivateForce = savedYes, savedForce }() cfg = &config.Config{} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) cfg.Data.DatabaseURL = dsn cfg.Vector.Embeddings.Dimension = 4 cfg.Vector.Embeddings.Model = "test-model" @@ -140,7 +146,7 @@ func TestRunEmbeddingsActivate_PG_AutoRetireDeletesPrevious(t *testing.T) { embeddingsActivateForce = true // skip fingerprint/seeded gating in the run func cmd := &cobra.Command{} - cmd.SetContext(ctx) + cmd.SetContext(testCtx) var out bytes.Buffer cmd.SetOut(&out) require.NoError( @@ -232,6 +238,8 @@ func TestListEmbeddingGenerations_PG(t *testing.T) { // production query helpers can use. The cfg-global swap mirrors // TestSetupVectorFeatures_SucceedsOnPostgres. func TestOpenEmbeddingsMetadataDB_PG(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) @@ -245,9 +253,10 @@ func TestOpenEmbeddingsMetadataDB_PG(t *testing.T) { savedCfg := cfg defer func() { cfg = savedCfg }() cfg = &config.Config{} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) cfg.Data.DatabaseURL = dsn - mdb, rebind, closeDB, err := openEmbeddingsMetadataDB(ctx) + mdb, rebind, closeDB, err := openEmbeddingsMetadataDB(testCtx) require.NoError( err, "openEmbeddingsMetadataDB on a migrated PG schema must succeed") @@ -271,6 +280,8 @@ func TestOpenEmbeddingsMetadataDB_PG(t *testing.T) { // friendly, build-pointing error rather than leaking a raw // `relation "index_generations" does not exist (SQLSTATE 42P01)`. func TestOpenEmbeddingsMetadataDB_PG_FriendlyErrorWhenUnmigrated(t *testing.T) { + cfg := testConfigValue() + // Use a search_path scoped to ONLY the fresh isolated schema (no // "public") so to_regclass cannot resolve against tables that prior // non-isolated test runs may have left in public — the schema genuinely @@ -280,9 +291,10 @@ func TestOpenEmbeddingsMetadataDB_PG_FriendlyErrorWhenUnmigrated(t *testing.T) { savedCfg := cfg defer func() { cfg = savedCfg }() cfg = &config.Config{} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) cfg.Data.DatabaseURL = dsn - _, _, closeDB, err := openEmbeddingsMetadataDB(context.Background()) + _, _, closeDB, err := openEmbeddingsMetadataDB(testCtx) if closeDB != nil { closeDB() } diff --git a/cmd/msgvault/cmd/embed_scope.go b/cmd/msgvault/cmd/embed_scope.go index 81d71daef..21da38f77 100644 --- a/cmd/msgvault/cmd/embed_scope.go +++ b/cmd/msgvault/cmd/embed_scope.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "errors" "fmt" "strings" @@ -22,7 +23,12 @@ import ( // them. Unknown identifiers are a hard error: a silently skipped account // would quietly widen the embedded corpus beyond what the operator asked // for. -func resolveEmbedScopeSourceIDs(s *store.Store) error { +func resolveEmbedScopeSourceIDs(s *store.Store, state *invocation) error { + state = invocationState(context.Background(), state) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg var ids []int64 switch { case len(embedAccounts) > 0 || len(embedCollections) > 0: @@ -131,7 +137,12 @@ func resolveEmbedAccountList(s *store.Store, accounts []string, requireIdentifie // vector.ErrScopeUnresolvable so callers latch vector search stale instead // of retrying forever against cached source IDs; transient failures (a busy // database) pass through unwrapped for retry. -func configuredEmbedBuildScope(s *store.Store) (vector.BuildScope, error) { +func configuredEmbedBuildScope(s *store.Store, state *invocation) (vector.BuildScope, error) { + state = invocationState(context.Background(), state) + if state == nil || state.cfg == nil { + return vector.BuildScope{}, errors.New("configuration is unavailable") + } + cfg := state.cfg messageTypes := cfg.Vector.Embed.Scope.MessageTypes if len(cfg.Vector.Embed.Scope.Accounts) == 0 { return vector.NewBuildScope(messageTypes, nil), nil @@ -151,10 +162,14 @@ func configuredEmbedBuildScope(s *store.Store) (vector.BuildScope, error) { // compare generation fingerprints against short-lived stores of their own. // A no-op when no accounts are configured. // -// It mutates the package-global cfg, so it may only run in short-lived -// single-goroutine CLI processes. Daemon code paths (HTTP handlers, the -// background vector init) must use resolvedVectorConfig instead. -func ensureEmbedScopeResolved() error { +// It mutates the invocation's config, so daemon code paths (HTTP handlers, +// the background vector init) must use resolvedVectorConfig instead. +func ensureEmbedScopeResolved(state *invocation) error { + state = invocationState(context.Background(), state) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg if len(cfg.Vector.Embed.Scope.Accounts) == 0 { return nil } @@ -163,7 +178,7 @@ func ensureEmbedScopeResolved() error { return fmt.Errorf("open main db for embed scope resolution: %w", err) } defer func() { _ = s.Close() }() - return resolveEmbedScopeSourceIDs(s) + return resolveEmbedScopeSourceIDs(s, state) } // resolvedVectorConfig returns a copy of the vector config with the @@ -192,7 +207,12 @@ func resolvedVectorConfig(s *store.Store, vecCfg vector.Config) (vector.Config, // openResolvedVectorConfig is resolvedVectorConfig for callers without an // open store, such as the daemon's CLI-plan HTTP handlers. -func openResolvedVectorConfig() (vector.Config, error) { +func openResolvedVectorConfig(state *invocation) (vector.Config, error) { + state = invocationState(context.Background(), state) + if state == nil || state.cfg == nil { + return vector.Config{}, errors.New("configuration is unavailable") + } + cfg := state.cfg if len(cfg.Vector.Embed.Scope.Accounts) == 0 && len(cfg.Vector.Multimodal.Scope.Accounts) == 0 { return cfg.Vector, nil } diff --git a/cmd/msgvault/cmd/embed_scope_sqlitevec_test.go b/cmd/msgvault/cmd/embed_scope_sqlitevec_test.go index 0c9d4b338..c01a41afb 100644 --- a/cmd/msgvault/cmd/embed_scope_sqlitevec_test.go +++ b/cmd/msgvault/cmd/embed_scope_sqlitevec_test.go @@ -4,7 +4,6 @@ package cmd import ( "bytes" - "context" "database/sql" "encoding/json" "net/http" @@ -100,6 +99,8 @@ func embedGenByID(t *testing.T, dataDir string) map[int64]sql.NullInt64 { // activates, and a follow-up run scoped differently is refused by the // fingerprint mismatch with the scope visible in the error. func TestRunEmbed_AccountScopedBuildActivatesScopedGeneration(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) @@ -118,6 +119,7 @@ func TestRunEmbed_AccountScopedBuildActivatesScopedGeneration(t *testing.T) { c.Vector.Embeddings.Dimension = 4 c.Data.DataDir = dataDir cfg = c + testCtx := testInvocationContext(t.Context(), c, invocationOptions{}) oldRebuild, oldYes := embedFullRebuild, embedYes oldAccounts, oldCollections := embedAccounts, embedCollections @@ -133,7 +135,7 @@ func TestRunEmbed_AccountScopedBuildActivatesScopedGeneration(t *testing.T) { newCmd := func() (*cobra.Command, *bytes.Buffer, *bytes.Buffer) { cmd := &cobra.Command{} - cmd.SetContext(context.Background()) + cmd.SetContext(testCtx) out, errOut := &bytes.Buffer{}, &bytes.Buffer{} cmd.SetOut(out) cmd.SetErr(errOut) @@ -220,7 +222,8 @@ func TestRunEmbedLivePersonGateStopsLaterBatchesAfterConfigDeletion(t *testing.T })) t.Cleanup(provider.Close) configured.Vector.Embeddings.Endpoint = provider.URL - withTestConfig(t, configured) + testCtx := withTestConfig(t, configured) + _ = testCtx require.NoError(configured.Save()) seedTwoAccountMainDB(t, dataDir) @@ -228,10 +231,10 @@ func TestRunEmbedLivePersonGateStopsLaterBatchesAfterConfigDeletion(t *testing.T require.NoError(err) profile, err := configured.Vector.SemanticPersonEmbeddingProfile() require.NoError(err) - _, err = mainStore.EnsurePersonSemanticEmbeddingProfile(t.Context(), profile) + _, err = mainStore.EnsurePersonSemanticEmbeddingProfile(testCtx, profile) require.NoError(err) _, _, err = mainStore.GrantPersonSemanticEmbeddingConsent( - t.Context(), profile.Fingerprint, "test", + testCtx, profile.Fingerprint, "test", ) require.NoError(err) for _, personSeed := range []struct { @@ -245,10 +248,10 @@ func TestRunEmbedLivePersonGateStopsLaterBatchesAfterConfigDeletion(t *testing.T "email", personSeed.email, "Observed "+personSeed.name, ) require.NoError(err) - person, _, err := mainStore.CreatePersonFromParticipantContext(t.Context(), participantID) + person, _, err := mainStore.CreatePersonFromParticipantContext(testCtx, participantID) require.NoError(err) _, err = mainStore.UpdatePersonDisplayNameContext( - t.Context(), person.ID, person.Revision, &personSeed.name, + testCtx, person.ID, person.Revision, &personSeed.name, ) require.NoError(err) } @@ -268,7 +271,8 @@ func TestRunEmbedLivePersonGateStopsLaterBatchesAfterConfigDeletion(t *testing.T embedAccounts = nil embedCollections = nil command := &cobra.Command{} - command.SetContext(t.Context()) + command.SetContext(testCtx) + command.SetContext(testCtx) stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{} command.SetOut(stdout) command.SetErr(stderr) @@ -295,7 +299,7 @@ func TestRunEmbedLivePersonGateStopsLaterBatchesAfterConfigDeletion(t *testing.T require.NoError(mainStore.Close()) embedFullRebuild = false resume := &cobra.Command{} - resume.SetContext(t.Context()) + resume.SetContext(testCtx) resumeOut, resumeErr := &bytes.Buffer{}, &bytes.Buffer{} resume.SetOut(resumeOut) resume.SetErr(resumeErr) @@ -313,6 +317,8 @@ func TestRunEmbedLivePersonGateStopsLaterBatchesAfterConfigDeletion(t *testing.T // remaining == 0 and activate an empty generation, retiring the working // index. The run must fail instead of activating. func TestRunEmbed_AccountScopedRebuildRefusesEmptyScope(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) @@ -336,6 +342,7 @@ func TestRunEmbed_AccountScopedRebuildRefusesEmptyScope(t *testing.T) { c.Vector.Embeddings.Dimension = 4 c.Data.DataDir = dataDir cfg = c + testCtx := testInvocationContext(t.Context(), c, invocationOptions{}) oldRebuild, oldYes := embedFullRebuild, embedYes oldAccounts, oldCollections := embedAccounts, embedCollections @@ -352,7 +359,7 @@ func TestRunEmbed_AccountScopedRebuildRefusesEmptyScope(t *testing.T) { embedAccounts = []string{"c@example.com"} cmd := &cobra.Command{} - cmd.SetContext(context.Background()) + cmd.SetContext(testCtx) out, errOut := &bytes.Buffer{}, &bytes.Buffer{} cmd.SetOut(out) cmd.SetErr(errOut) diff --git a/cmd/msgvault/cmd/embed_scope_test.go b/cmd/msgvault/cmd/embed_scope_test.go index bb9b111b0..51dc20b72 100644 --- a/cmd/msgvault/cmd/embed_scope_test.go +++ b/cmd/msgvault/cmd/embed_scope_test.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "strconv" "testing" @@ -40,38 +41,40 @@ func TestEmbeddingsBuildAccountFlagsForwardToDaemon(t *testing.T) { }, got) } -// withEmbedScopeGlobals swaps the config and embed-scope flag globals for a -// resolution test and restores them afterwards. -func withEmbedScopeGlobals(t *testing.T, accounts []string) { +// withEmbedScopeContext binds an invocation to the test config and scope flags. +func withEmbedScopeContext(t *testing.T, cfg *config.Config, accounts []string) context.Context { t.Helper() - oldCfg := cfg oldAccounts, oldCollections := embedAccounts, embedCollections - c := &config.Config{} - c.Vector.Embed.Scope.Accounts = accounts - cfg = c + oldScope := append([]string(nil), cfg.Vector.Embed.Scope.Accounts...) + cfg.Vector.Embed.Scope.Accounts = append([]string(nil), accounts...) embedAccounts, embedCollections = nil, nil t.Cleanup(func() { - cfg = oldCfg + cfg.Vector.Embed.Scope.Accounts = oldScope embedAccounts, embedCollections = oldAccounts, oldCollections }) + return testInvocationContext(t.Context(), cfg, invocationOptions{}) } func TestResolveEmbedScopeSourceIDs_ConfiguredAccounts(t *testing.T) { + cfg := testConfigValue() + f, accountID, _ := setupScopeFixture(t) - withEmbedScopeGlobals(t, []string{accountID}) + testCtx := withEmbedScopeContext(t, cfg, []string{accountID}) - require.NoError(t, resolveEmbedScopeSourceIDs(f.Store)) + require.NoError(t, resolveEmbedScopeSourceIDs(f.Store, invocationFromContext(testCtx))) assert.Equal(t, []int64{f.Source.ID}, cfg.Vector.Embed.Scope.SourceIDs, "configured account resolves to its source ID") } func TestResolveEmbedScopeSourceIDs_UnknownAccountFails(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) f, _, _ := setupScopeFixture(t) - withEmbedScopeGlobals(t, []string{"nobody@example.com"}) + testCtx := withEmbedScopeContext(t, cfg, []string{"nobody@example.com"}) - err := resolveEmbedScopeSourceIDs(f.Store) + err := resolveEmbedScopeSourceIDs(f.Store, invocationFromContext(testCtx)) require.Error(err, "unknown configured account must fail loudly") assert.Contains(err.Error(), "[vector.embed.scope] accounts") assert.Contains(err.Error(), "nobody@example.com") @@ -79,31 +82,37 @@ func TestResolveEmbedScopeSourceIDs_UnknownAccountFails(t *testing.T) { } func TestResolveEmbedScopeSourceIDs_NoScopeLeavesCorpusWide(t *testing.T) { + cfg := testConfigValue() + f, _, _ := setupScopeFixture(t) - withEmbedScopeGlobals(t, nil) + testCtx := withEmbedScopeContext(t, cfg, nil) - require.NoError(t, resolveEmbedScopeSourceIDs(f.Store)) + require.NoError(t, resolveEmbedScopeSourceIDs(f.Store, invocationFromContext(testCtx))) assert.Nil(t, cfg.Vector.Embed.Scope.SourceIDs) } func TestResolveEmbedScopeSourceIDs_AccountFlagOverridesConfig(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) f, accountID, _ := setupScopeFixture(t) - withEmbedScopeGlobals(t, []string{accountID}) + testCtx := withEmbedScopeContext(t, cfg, []string{accountID}) other, err := f.Store.GetOrCreateSource("gmail", "other@example.com") require.NoError(err, "GetOrCreateSource") embedAccounts = []string{other.Identifier} - require.NoError(resolveEmbedScopeSourceIDs(f.Store)) + require.NoError(resolveEmbedScopeSourceIDs(f.Store, invocationFromContext(testCtx))) assert.Equal(t, []int64{other.ID}, cfg.Vector.Embed.Scope.SourceIDs, "--account replaces the configured accounts for the run") } func TestResolveEmbedScopeSourceIDs_CollectionFlagExpands(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) f, _, collectionName := setupScopeFixture(t) - withEmbedScopeGlobals(t, nil) + testCtx := withEmbedScopeContext(t, cfg, nil) other, err := f.Store.GetOrCreateSource("gmail", "other@example.com") require.NoError(err, "GetOrCreateSource") @@ -112,22 +121,24 @@ func TestResolveEmbedScopeSourceIDs_CollectionFlagExpands(t *testing.T) { // The fixture collection covers only f.Source; "both" covers both. embedCollections = []string{collectionName, "both"} - require.NoError(resolveEmbedScopeSourceIDs(f.Store)) + require.NoError(resolveEmbedScopeSourceIDs(f.Store, invocationFromContext(testCtx))) assert.ElementsMatch(t, []int64{f.Source.ID, other.ID}, cfg.Vector.Embed.Scope.SourceIDs, "--collection expands to the union of member sources") } func TestResolveEmbedScopeSourceIDs_EmptyCollectionFailsClosed(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) f, _, _ := setupScopeFixture(t) - withEmbedScopeGlobals(t, nil) + testCtx := withEmbedScopeContext(t, cfg, nil) _, err := f.Store.CreateCollection("empty", "", []int64{f.Source.ID}) require.NoError(err, "CreateCollection") require.NoError(f.Store.RemoveSourcesFromCollection("empty", []int64{f.Source.ID}), "empty collection") embedCollections = []string{"empty"} - err = resolveEmbedScopeSourceIDs(f.Store) + err = resolveEmbedScopeSourceIDs(f.Store, invocationFromContext(testCtx)) require.Error(err, "an explicit empty collection must not widen to the full archive") assert.Contains(t, err.Error(), "has no accounts") assert.Nil(t, cfg.Vector.Embed.Scope.SourceIDs) @@ -138,10 +149,12 @@ func TestResolveEmbedScopeSourceIDs_EmptyCollectionFailsClosed(t *testing.T) { // while the shared package-global cfg stays unmutated, so concurrent daemon // goroutines can resolve without racing. func TestResolvedVectorConfigLeavesGlobalUntouched(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) f, accountID, _ := setupScopeFixture(t) - withEmbedScopeGlobals(t, []string{accountID}) + _ = withEmbedScopeContext(t, cfg, []string{accountID}) // Scope resolution is gated on the lane being enabled. cfg.Vector.Enabled = true @@ -159,17 +172,19 @@ func TestResolvedVectorConfigLeavesGlobalUntouched(t *testing.T) { // heal on retry and must latch searches stale — while a resolvable // configuration re-resolves cleanly. func TestConfiguredEmbedBuildScope_ClassifiesResolutionFailures(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) f, accountID, _ := setupScopeFixture(t) - withEmbedScopeGlobals(t, []string{accountID}) - scope, err := configuredEmbedBuildScope(f.Store) + testCtx := withEmbedScopeContext(t, cfg, []string{accountID}) + scope, err := configuredEmbedBuildScope(f.Store, invocationFromContext(testCtx)) require.NoError(err) assert.Equal([]int64{f.Source.ID}, scope.SourceIDs) cfg.Vector.Embed.Scope.Accounts = []string{"gone@example.com"} - _, err = configuredEmbedBuildScope(f.Store) + _, err = configuredEmbedBuildScope(f.Store, invocationFromContext(testCtx)) require.ErrorIs(err, vector.ErrScopeUnresolvable, "a configured account that no longer exists is a deterministic failure") assert.Contains(err.Error(), "gone@example.com") @@ -182,6 +197,8 @@ func TestConfiguredEmbedBuildScope_ClassifiesResolutionFailures(t *testing.T) { // embed the replacement account's text. Durable configuration must name the // canonical identifier; the one-run --account flag stays permissive. func TestDurableEmbedScopeRejectsDisplayNames(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) f, accountID, _ := setupScopeFixture(t) @@ -189,18 +206,18 @@ func TestDurableEmbedScopeRejectsDisplayNames(t *testing.T) { f.Store.Rebind(`UPDATE sources SET display_name = 'Work Mail' WHERE id = ?`), f.Source.ID) require.NoError(err, "set display name") - withEmbedScopeGlobals(t, []string{"Work Mail"}) - _, err = configuredEmbedBuildScope(f.Store) + testCtx := withEmbedScopeContext(t, cfg, []string{"Work Mail"}) + _, err = configuredEmbedBuildScope(f.Store, invocationFromContext(testCtx)) require.ErrorIs(err, vector.ErrScopeUnresolvable, "a display name in durable config must fail closed") assert.Contains(err.Error(), accountID, "the error names the canonical identifier to use") - err = resolveEmbedScopeSourceIDs(f.Store) + err = resolveEmbedScopeSourceIDs(f.Store, invocationFromContext(testCtx)) require.Error(err, "startup resolution must reject the display name too") // The one-run --account flag still accepts the display name. embedAccounts = []string{"Work Mail"} - require.NoError(resolveEmbedScopeSourceIDs(f.Store)) + require.NoError(resolveEmbedScopeSourceIDs(f.Store, invocationFromContext(testCtx))) assert.Equal([]int64{f.Source.ID}, cfg.Vector.Embed.Scope.SourceIDs) } @@ -209,37 +226,42 @@ func TestDurableEmbedScopeRejectsDisplayNames(t *testing.T) { // deterministically unresolvable account all report the right (detail, err) // so the API latches stale exactly when the scope no longer matches. func TestEmbedScopeDriftCheck(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) f, accountID, _ := setupScopeFixture(t) - withEmbedScopeGlobals(t, []string{accountID}) + testCtx := withEmbedScopeContext(t, cfg, []string{accountID}) + state := invocationFromContext(testCtx) initialized := vector.NewBuildScope(nil, []int64{f.Source.ID}) - check := embedScopeDriftCheck(f.Store, initialized) + check := embedScopeDriftCheck(f.Store, initialized, state) - detail, err := check(t.Context()) + detail, err := check(context.Background()) require.NoError(err) assert.Empty(detail, "a matching scope must not latch") other, err := f.Store.GetOrCreateSource("gmail", "other@example.com") require.NoError(err, "GetOrCreateSource") cfg.Vector.Embed.Scope.Accounts = []string{other.Identifier} - detail, err = check(t.Context()) + detail, err = check(context.Background()) require.NoError(err) assert.Contains(detail, "src-", "a drifted scope latches with both fingerprints") cfg.Vector.Embed.Scope.Accounts = []string{"gone@example.com"} - detail, err = check(t.Context()) + detail, err = check(context.Background()) require.NoError(err, "an unresolvable account is drift, not a retryable error") assert.Contains(detail, "gone@example.com") assert.Contains(detail, "[vector.embed.scope]") } func TestResolveEmbedScopeSourceIDs_ConfiguredNumericIDFails(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) f, _, _ := setupScopeFixture(t) - withEmbedScopeGlobals(t, []string{strconv.FormatInt(f.Source.ID, 10)}) + testCtx := withEmbedScopeContext(t, cfg, []string{strconv.FormatInt(f.Source.ID, 10)}) - err := resolveEmbedScopeSourceIDs(f.Store) + err := resolveEmbedScopeSourceIDs(f.Store, invocationFromContext(testCtx)) require.Error(err, "durable account configuration must not accept source IDs") assert.Nil(t, cfg.Vector.Embed.Scope.SourceIDs) } diff --git a/cmd/msgvault/cmd/embed_test.go b/cmd/msgvault/cmd/embed_test.go index 04c786360..26b988761 100644 --- a/cmd/msgvault/cmd/embed_test.go +++ b/cmd/msgvault/cmd/embed_test.go @@ -56,6 +56,15 @@ func TestEmbeddingsCommandRegistration(t *testing.T) { require.NoError(err) require.Equal("prune", pruneCmd.Name()) + optimizeCmd, _, err := rootCmd.Find([]string{embeddingsCommandName, "optimize"}) + require.NoError(err) + require.Equal("optimize", optimizeCmd.Name()) + + workerCmd, _, err := rootCmd.Find([]string{embeddingsCommandName, embeddingsOptimizeWorkerName}) + require.NoError(err) + require.True(workerCmd.Hidden) + require.True(skipsConfigLoad(workerCmd)) + legacyCmd, _, err := rootCmd.Find([]string{"build-embeddings"}) require.NoError(err) require.Equal("build-embeddings", legacyCmd.Name()) @@ -71,9 +80,11 @@ func TestEmbeddingsListUsesDaemonRunner(t *testing.T) { server, requests := newDaemonCLIRunnerTestServer(t, func(req daemonCLIRunTestRequest) { assert.Equal([]string{embeddingsCommandName, "list"}, req.Args, "args") }, `{"type":"stdout","data":"ID\tSTATE\n1\tactive\n"}`, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx root := &cobra.Command{Use: daemonService} + root.SetContext(testCtx) embeddings := &cobra.Command{Use: embeddingsCommandName} list := &cobra.Command{ Use: cmdUseList, @@ -98,9 +109,11 @@ func TestEmbeddingsPruneUsesDaemonRunner(t *testing.T) { server, requests := newDaemonCLIRunnerTestServer(t, func(req daemonCLIRunTestRequest) { assert.Equal([]string{embeddingsCommandName, "prune"}, req.Args, "args") }, `{"type":"stdout","data":"Pruned 2 orphan message embedding(s).\n"}`, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx root := &cobra.Command{Use: daemonService} + root.SetContext(testCtx) embeddings := &cobra.Command{Use: embeddingsCommandName} prune := &cobra.Command{ Use: "prune", @@ -133,9 +146,11 @@ func TestEmbeddingsBuildPromptsBeforeDaemonRunner(t *testing.T) { "--yes", }, req.Args, "args") }, `{"type":"stderr","data":"Building generation 2\n"}`, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx root := &cobra.Command{Use: daemonService} + root.SetContext(testCtx) embeddings := &cobra.Command{Use: embeddingsCommandName} build := newEmbeddingsBuildCmd("build") embeddings.AddCommand(build) @@ -161,9 +176,11 @@ func TestEmbeddingsResumeUsesDaemonRunner(t *testing.T) { server, requests := newDaemonCLIRunnerTestServer(t, func(req daemonCLIRunTestRequest) { assert.Equal([]string{embeddingsCommandName, "resume", "--backstop"}, req.Args, "args") }, `{"type":"stdout","data":"Scanned: 1, succeeded: 1, failed: 0, truncated: 0\n"}`, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx root := &cobra.Command{Use: daemonService} + root.SetContext(testCtx) embeddings := &cobra.Command{Use: embeddingsCommandName} resume := &cobra.Command{ Use: "resume", @@ -201,9 +218,11 @@ func TestEmbeddingsRetirePromptsBeforeDaemonRunner(t *testing.T) { }, func(req daemonCLIRunTestRequest) { assert.Equal([]string{embeddingsCommandName, "retire", "--force-active", "--yes", "2"}, req.Args, "args") }, `{"type":"stdout","data":"Generation 2 retired.\n"}`, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx root := &cobra.Command{Use: daemonService} + root.SetContext(testCtx) embeddings := &cobra.Command{Use: embeddingsCommandName} retire := &cobra.Command{ Use: "retire ", @@ -246,9 +265,11 @@ func TestEmbeddingsActivatePromptsBeforeDaemonRunner(t *testing.T) { }, func(req daemonCLIRunTestRequest) { assert.Equal([]string{embeddingsCommandName, "activate", "--force", "--yes", "3"}, req.Args, "args") }, `{"type":"stdout","data":"Generation 3 activated.\n"}`, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx root := &cobra.Command{Use: daemonService} + root.SetContext(testCtx) embeddings := &cobra.Command{Use: embeddingsCommandName} activate := &cobra.Command{ Use: "activate ", @@ -278,6 +299,8 @@ func TestEmbeddingsActivatePromptsBeforeDaemonRunner(t *testing.T) { // must leave embedBackstop exactly as the operator set it, so // `embeddings resume --backstop` actually runs a backstop pass. func TestRunEmbeddingsResume_PreservesBackstopFlag(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) // Save and restore all three globals so the test is hermetic. @@ -292,11 +315,14 @@ func TestRunEmbeddingsResume_PreservesBackstopFlag(t *testing.T) { embedBackstop = true oldCfg := cfg cfg = &config.Config{} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx t.Cleanup(func() { cfg = oldCfg }) cmd := embeddingsResumeCmd + cmd.SetContext(testCtx) oldCtx := cmd.Context() - cmd.SetContext(context.Background()) + cmd.SetContext(testCtx) t.Cleanup(func() { cmd.SetContext(oldCtx) }) // Errors because vector is not enabled — that's fine; we only assert the @@ -340,14 +366,14 @@ func TestRunEmbeddingsActivateRefusesMissingWithoutForce(t *testing.T) { // Main DB with one live, unembedded message -> coverage reports // missing=1 for generation 2. seedMainDBWithLiveMessage(t, dataDir) - withEmbeddingCommandConfigDataDir(t, dbPath, dataDir) + testCtx, _ := withEmbeddingCommandConfigDataDir(t, dbPath, dataDir) oldYes := embeddingsActivateYes embeddingsActivateYes = true t.Cleanup(func() { embeddingsActivateYes = oldYes }) cmd := embeddingsActivateCmd oldCtx := cmd.Context() - cmd.SetContext(context.Background()) + cmd.SetContext(testCtx) t.Cleanup(func() { cmd.SetContext(oldCtx) }) err := runEmbeddingsActivate(cmd, []string{"2"}) @@ -367,7 +393,7 @@ func TestRetireEmbeddingGenerationRefusesActiveWithoutForce_PreCheck(t *testing. require := require.New(t) assert := assert.New(t) dbPath := newEmbeddingMetadataTestDBFile(t) - withEmbeddingCommandConfig(t, dbPath) + testCtx := withEmbeddingCommandConfig(t, dbPath) oldYes := embeddingsRetireYes oldForce := embeddingsRetireForceActive @@ -380,7 +406,7 @@ func TestRetireEmbeddingGenerationRefusesActiveWithoutForce_PreCheck(t *testing. cmd := embeddingsRetireCmd oldCtx := cmd.Context() - cmd.SetContext(context.Background()) + cmd.SetContext(testCtx) t.Cleanup(func() { cmd.SetContext(oldCtx) }) err := runEmbeddingsRetire(cmd, []string{"1"}) @@ -473,25 +499,21 @@ INSERT INTO messages (id, conversation_id, source_id, source_message_id, message require.NoError(t, err) } -func withEmbeddingCommandConfig(t *testing.T, vecPath string) { +func withEmbeddingCommandConfig(t *testing.T, vecPath string) context.Context { t.Helper() - oldCfg := cfg c := newTestConfigForFingerprint(vecPath) c.Data.DataDir = filepath.Dir(vecPath) - cfg = c - t.Cleanup(func() { cfg = oldCfg }) + return testInvocationContext(t.Context(), c, invocationOptions{}) } // withEmbeddingCommandConfigDataDir is like withEmbeddingCommandConfig but // also sets Data.DataDir so DatabaseDSN() resolves to a real main DB (used // by the coverage gate). -func withEmbeddingCommandConfigDataDir(t *testing.T, vecPath, dataDir string) { +func withEmbeddingCommandConfigDataDir(t *testing.T, vecPath, dataDir string) (context.Context, *config.Config) { t.Helper() - oldCfg := cfg c := newTestConfigForFingerprint(vecPath) c.Data.DataDir = dataDir - cfg = c - t.Cleanup(func() { cfg = oldCfg }) + return testInvocationContext(t.Context(), c, invocationOptions{}), c } func newTestConfigForFingerprint(vecPath string) *config.Config { @@ -520,6 +542,8 @@ func mustGetEmbeddingGeneration(ctx context.Context, t *testing.T, db *sql.DB, g } func TestEmbeddingsBuildForwardsAPIKeyEnvToDaemonRunner(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) oldFull, oldYes := embedFullRebuild, embedYes @@ -532,10 +556,11 @@ func TestEmbeddingsBuildForwardsAPIKeyEnvToDaemonRunner(t *testing.T) { assert.Equal([]string{embeddingsCommandName, "build"}, req.Args, "args") assert.Equal("secret-token", req.Env[keyEnv], "caller API key forwarded to subprocess env") }, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL, cfg) cfg.Vector.Embeddings.APIKeyEnv = keyEnv root := &cobra.Command{Use: daemonService} + root.SetContext(testCtx) embeddings := &cobra.Command{Use: embeddingsCommandName} build := &cobra.Command{ Use: "build", @@ -550,13 +575,17 @@ func TestEmbeddingsBuildForwardsAPIKeyEnvToDaemonRunner(t *testing.T) { } func TestEmbeddingsForwardEnvSkipsUnsetKey(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = &config.Config{} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx - assert.Nil(embeddingsForwardEnv(), "no api_key_env configured") + assert.Nil(embeddingsForwardEnv(invocationFromContext(testCtx)), "no api_key_env configured") cfg.Vector.Embeddings.APIKeyEnv = "MSGVAULT_TEST_EMBED_KEY_UNSET" - assert.Nil(embeddingsForwardEnv(), "configured env var not set in caller environment") + assert.Nil(embeddingsForwardEnv(invocationFromContext(testCtx)), "configured env var not set in caller environment") } diff --git a/cmd/msgvault/cmd/embed_vector.go b/cmd/msgvault/cmd/embed_vector.go index 471dc715f..f38c3ab3a 100644 --- a/cmd/msgvault/cmd/embed_vector.go +++ b/cmd/msgvault/cmd/embed_vector.go @@ -24,10 +24,15 @@ import ( ) func runEmbed(cmd *cobra.Command) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg ctx := cmd.Context() out := cmd.OutOrStdout() errOut := cmd.ErrOrStderr() - release, err := acquireDirectSQLiteWriteLock(cfg) + release, err := acquireDirectSQLiteWriteLock(cfg, state) if err != nil { return err } @@ -53,7 +58,7 @@ func runEmbed(cmd *cobra.Command) error { // --account/--collection flags (or [vector.embed.scope] accounts) // become source IDs here, and unknown identifiers fail the run loudly // rather than silently widening the embedded corpus. - if err := resolveEmbedScopeSourceIDs(s); err != nil { + if err := resolveEmbedScopeSourceIDs(s, state); err != nil { return err } @@ -96,11 +101,14 @@ func runEmbed(cmd *cobra.Command) error { vecPath = filepath.Join(cfg.Data.DataDir, "vectors.db") } sb, err := sqlitevec.Open(ctx, sqlitevec.Options{ - Path: vecPath, - MainPath: cfg.DatabaseDSN(), - Dimension: cfg.Vector.Embeddings.Dimension, - MainDB: s.DB(), - BuildScope: cfg.Vector.Embed.Scope.BuildScope(), + Path: vecPath, + MainPath: cfg.DatabaseDSN(), + Dimension: cfg.Vector.Embeddings.Dimension, + MainDB: s.DB(), + BuildScope: cfg.Vector.Embed.Scope.BuildScope(), + ANNOversample: cfg.Vector.Search.ANNOversample, + ANNNProbe: cfg.Vector.Search.ANNNProbe, + AcceleratorMode: cfg.Vector.Search.SQLiteAccelerator, }) if err != nil { return fmt.Errorf("open vectors.db: %w", err) @@ -151,7 +159,7 @@ func runEmbed(cmd *cobra.Command) error { } totalPending := int(missing) personGate := vector.NewPinnedExactSemanticPersonEmbeddingGate( - cfg.Vector, currentSemanticPersonVectorConfigSource(), s, + cfg.Vector, currentSemanticPersonVectorConfigSource(state), s, ) credentialSnapshot, err := providercredentials.Read(cfg.TokensDir()) if err != nil { diff --git a/cmd/msgvault/cmd/embed_vector_pg_test.go b/cmd/msgvault/cmd/embed_vector_pg_test.go index 180926ac5..3bb576d90 100644 --- a/cmd/msgvault/cmd/embed_vector_pg_test.go +++ b/cmd/msgvault/cmd/embed_vector_pg_test.go @@ -20,6 +20,8 @@ import ( // messages table (clean exit path). Skips when MSGVAULT_TEST_DB is unset // or not a postgres DSN. func TestRunEmbed_PG_OpenAndZeroPending(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) @@ -63,6 +65,8 @@ func TestRunEmbed_PG_OpenAndZeroPending(t *testing.T) { savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = &config.Config{} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Vector.Enabled = true cfg.Vector.Embeddings.Model = "test-model" cfg.Vector.Embeddings.Dimension = 4 diff --git a/cmd/msgvault/cmd/embed_vector_test.go b/cmd/msgvault/cmd/embed_vector_test.go index d8261a111..af306ff15 100644 --- a/cmd/msgvault/cmd/embed_vector_test.go +++ b/cmd/msgvault/cmd/embed_vector_test.go @@ -27,6 +27,7 @@ import ( "go.kenn.io/msgvault/internal/store" "go.kenn.io/msgvault/internal/vector" "go.kenn.io/msgvault/internal/vector/embed" + "go.kenn.io/msgvault/internal/vector/hybrid" "go.kenn.io/msgvault/internal/vector/personsearch" "go.kenn.io/msgvault/internal/vector/sqlitevec" ) @@ -357,7 +358,8 @@ func setupVectorFeaturesFixture( apply(c) } require.NoError(t, c.Save()) - withTestConfig(t, c) + testCtx := withTestConfig(t, c) + _ = testCtx s, err := store.Open(mainPath) require.NoError(t, err) @@ -366,10 +368,10 @@ func setupVectorFeaturesFixture( if c.Vector.People.Enabled { semanticProfile, err := c.Vector.SemanticPersonEmbeddingProfile() require.NoError(t, err) - _, err = s.EnsurePersonSemanticEmbeddingProfile(t.Context(), semanticProfile) + _, err = s.EnsurePersonSemanticEmbeddingProfile(testCtx, semanticProfile) require.NoError(t, err) _, _, err = s.GrantPersonSemanticEmbeddingConsent( - t.Context(), semanticProfile.Fingerprint, "test", + testCtx, semanticProfile.Fingerprint, "test", ) require.NoError(t, err) } @@ -380,22 +382,22 @@ func setupVectorFeaturesFixture( RetentionPosture: "standard", TrainingPosture: "opted-out", AllowedMediaTypes: []string{"application/pdf"}, PolicyJSON: []byte(`{"policy":1}`), } - _, err = s.EnsureDocumentExtractionProfile(t.Context(), profile) + _, err = s.EnsureDocumentExtractionProfile(testCtx, profile) require.NoError(t, err) _, err = s.DB().Exec(s.Rebind(`UPDATE document_index_state SET target_profile_id = ? WHERE singleton = 1`), profile.ID) require.NoError(t, err) - spec, err := configuredDocumentVectorSpec(t.Context(), s) + spec, err := configuredDocumentVectorSpec(testCtx, s) require.NoError(t, err) - documentConsent, err := configuredDocumentVectorConsentSpec(spec) + documentConsent, err := configuredDocumentVectorConsentSpec(spec, invocationFromContext(testCtx)) require.NoError(t, err) - queryConsent, err := configuredDocumentVectorQueryConsentSpec(spec) + queryConsent, err := configuredDocumentVectorQueryConsentSpec(spec, invocationFromContext(testCtx)) require.NoError(t, err) for _, consentSpec := range []store.DocumentVectorConsentSpec{documentConsent, queryConsent} { - _, _, err = s.RecordDocumentVectorConsent(t.Context(), consentSpec, time.Now()) + _, _, err = s.RecordDocumentVectorConsent(testCtx, consentSpec, time.Now()) require.NoError(t, err) } - vf, err := setupVectorFeatures(t.Context(), s, mainPath, readOnly) + vf, err := setupVectorFeatures(testCtx, s, mainPath, readOnly) require.NoError(t, err) require.NotNil(t, vf) t.Cleanup(func() { _ = vf.Close() }) @@ -571,7 +573,9 @@ func TestSetupVectorFeaturesUsesStoredCredentialSnapshotWithoutEnvironment(t *te })) t.Cleanup(provider.Close) var storedETag string + var tokensDir string vf := setupVectorFeaturesFixture(t, vector.APIFormatOpenAI, false, func(c *config.Config) { + tokensDir = c.TokensDir() c.Vector.Embeddings.Endpoint = provider.URL c.Vector.Embeddings.APIKeyEnv = "TEXT_EMBEDDING_KEY" empty, err := providercredentials.Read(c.TokensDir()) @@ -581,7 +585,7 @@ func TestSetupVectorFeaturesUsesStoredCredentialSnapshotWithoutEnvironment(t *te require.NoError(t, err) storedETag = stored.ETag }) - _, err := providercredentials.Put(cfg.TokensDir(), storedETag, + _, err := providercredentials.Put(tokensDir, storedETag, providercredentials.VectorEmbeddingsID, provider.URL, "stored-after-startup") require.NoError(t, err) @@ -1142,3 +1146,69 @@ func TestNewProgressPrinter_DoesNotBypassThrottleAfterInitialTotal(t *testing.T) lines := strings.Split(strings.TrimRight(buf.String(), "\n"), "\n") require.Len(t, lines, 1, "progress emitted %d lines, want 1 throttled line after initial total:\n%s", len(lines), buf.String()) } + +// Pin a pre-Kit generation, including its stored vector. Adopting the HTTP +// client must neither schedule a rebuild nor replace the persisted identity. +func TestKitMigrationReusesExistingGenerationForIndexAndSearch(t *testing.T) { + ctx := t.Context() + backend := openTestBackend(t) + const fingerprint = "legacy-model:4:p1-111111:c32768:e1" + cfg := vector.Config{Embeddings: vector.EmbeddingsConfig{Model: "legacy-model", Dimension: 4}} + cfg.ApplyDefaults() + require.Equal(t, fingerprint, cfg.GenerationFingerprint()) + generation, err := backend.CreateGeneration(ctx, "legacy-model", 4, fingerprint) + require.NoError(t, err) + require.NoError(t, backend.Upsert(ctx, generation, []vector.Chunk{{MessageID: 1, Vector: []float32{1, 0, 0, 0}}})) + require.NoError(t, backend.ActivateGeneration(ctx, generation, true)) + + got, rebuilding, err := pickEmbedGeneration(ctx, backend, embedGenerationOpts{ + Model: cfg.Embeddings.Model, Dimension: 4, Fingerprint: cfg.GenerationFingerprint(), Stderr: openStderrSink(t), + }) + require.NoError(t, err) + assert.Equal(t, generation, got) + assert.False(t, rebuilding) + + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + var request struct { + Input []string `json:"input"` + } + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + http.Error(w, "invalid request", http.StatusBadRequest) + return + } + assert.Equal(t, []string{"find existing message"}, request.Input) + _, _ = w.Write([]byte(`{"data":[{"index":0,"embedding":[1,0,0,0]}]}`)) + })) + t.Cleanup(server.Close) + client := embed.NewClient(embed.Config{Endpoint: server.URL, Model: cfg.Embeddings.Model, Dimension: 4}) + engine := hybrid.NewEngine(backend, nil, client, hybrid.Config{ExpectedFingerprint: cfg.GenerationFingerprint()}) + hits, metadata, err := engine.Search(ctx, hybrid.SearchRequest{Mode: hybrid.ModeVector, FreeText: "find existing message", Limit: 10}) + require.NoError(t, err) + require.Len(t, hits, 1) + assert.Equal(t, int64(1), hits[0].MessageID) + assert.Equal(t, generation, metadata.Generation.ID) + assert.Equal(t, fingerprint, metadata.Generation.Fingerprint) + assert.Equal(t, int32(1), requests.Load(), "only the query is embedded") + building, err := backend.BuildingGeneration(ctx) + require.NoError(t, err) + assert.Nil(t, building) + + for _, change := range []struct { + name string + apply func(*vector.Config) + }{ + {"model", func(c *vector.Config) { c.Embeddings.Model = "other-model" }}, + {"recipe", func(c *vector.Config) { c.Embeddings.MaxInputChars++ }}, + } { + t.Run(change.name, func(t *testing.T) { + changed := cfg + change.apply(&changed) + _, err := vector.ResolveActiveForFingerprint(ctx, backend, changed.GenerationFingerprint()) + require.ErrorIs(t, err, vector.ErrIndexStale) + _, _, err = pickEmbedGeneration(ctx, backend, embedGenerationOpts{Model: changed.Embeddings.Model, Dimension: 4, Fingerprint: changed.GenerationFingerprint(), Stderr: openStderrSink(t)}) + require.ErrorIs(t, err, vector.ErrIndexStale) + }) + } +} diff --git a/cmd/msgvault/cmd/embeddings_accelerator_status.go b/cmd/msgvault/cmd/embeddings_accelerator_status.go new file mode 100644 index 000000000..ed0ad37e7 --- /dev/null +++ b/cmd/msgvault/cmd/embeddings_accelerator_status.go @@ -0,0 +1,36 @@ +//go:build sqlite_vec + +package cmd + +import ( + "context" + "fmt" + + "go.kenn.io/msgvault/internal/vector" + "go.kenn.io/msgvault/internal/vector/sqlitevec" +) + +func readEmbeddingAccelerator( + ctx context.Context, backend vector.Backend, generationID vector.GenerationID, +) (embeddingAcceleratorRow, error) { + concrete, ok := backend.(*sqlitevec.Backend) + if !ok { + return embeddingAcceleratorRow{}, nil + } + // EffectiveAccelerator applies the same eligibility checks search uses: + // a stored-ready accelerator search would refuse (revision or count + // drift, Vec1 version mismatch, missing physical table) displays as + // stale instead of the raw stored ready state. + status, err := concrete.EffectiveAccelerator(ctx, generationID) + if err != nil { + return embeddingAcceleratorRow{}, fmt.Errorf("read generation %d accelerator status: %w", generationID, err) + } + if status == nil { + return embeddingAcceleratorRow{State: "exact"}, nil + } + started := status.StartedAt + return embeddingAcceleratorRow{ + State: string(status.State), IndexedCount: status.IndexedCount, + StartedAt: &started, CompletedAt: status.CompletedAt, LastError: status.LastError, + }, nil +} diff --git a/cmd/msgvault/cmd/embeddings_accelerator_status_stub.go b/cmd/msgvault/cmd/embeddings_accelerator_status_stub.go new file mode 100644 index 000000000..c7ff5e86f --- /dev/null +++ b/cmd/msgvault/cmd/embeddings_accelerator_status_stub.go @@ -0,0 +1,13 @@ +//go:build !sqlite_vec + +package cmd + +import ( + "context" + + "go.kenn.io/msgvault/internal/vector" +) + +func readEmbeddingAccelerator(context.Context, vector.Backend, vector.GenerationID) (embeddingAcceleratorRow, error) { + return embeddingAcceleratorRow{}, nil +} diff --git a/cmd/msgvault/cmd/embeddings_accelerator_status_test.go b/cmd/msgvault/cmd/embeddings_accelerator_status_test.go new file mode 100644 index 000000000..48b19a875 --- /dev/null +++ b/cmd/msgvault/cmd/embeddings_accelerator_status_test.go @@ -0,0 +1,118 @@ +//go:build sqlite_vec + +package cmd + +import ( + "fmt" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/vector" + "go.kenn.io/msgvault/internal/vector/sqlitevec" + "go.kenn.io/msgvault/internal/vector/sqlitevec/vec1" +) + +// installListAcceleratorRow inserts a vector_accelerators row the way the +// optimize publish path leaves it: state='ready', counts and revision +// matching the generation, and the installed Vec1 version. +func installListAcceleratorRow( + t *testing.T, + backend *sqlitevec.Backend, + generationID vector.GenerationID, + state string, +) { + t.Helper() + tableName := fmt.Sprintf("message_ann_g%d", int64(generationID)) + _, err := backend.DB().Exec(`CREATE VIRTUAL TABLE ` + tableName + ` USING vec1(embedding)`) + require.NoError(t, err) + _, err = backend.DB().Exec(`INSERT INTO `+tableName+`(rowid, embedding) + SELECT embedding_id, embedding FROM `+sqlitevec.VectorTableName(4)+` + WHERE generation_id = ?`, int64(generationID)) + require.NoError(t, err) + var count, revision int64 + require.NoError(t, backend.DB().QueryRow(`SELECT embedding_count, vector_revision + FROM index_generations WHERE id = ?`, int64(generationID)).Scan(&count, &revision)) + now := time.Now().Unix() + _, err = backend.DB().Exec(`INSERT INTO vector_accelerators + (generation_id, kind, state, table_name, dimension, indexed_count, + last_embedding_id, source_revision, started_at, completed_at, + model_config, vec1_version) + VALUES (?, 'vec1_ivf_opq', ?, ?, ?, ?, 0, ?, ?, ?, '{}', ?)`, + int64(generationID), state, tableName, 4, count, revision, now, now, + vec1.Version) + require.NoError(t, err) +} + +// openListBackend seeds a fresh vectors.db with one generation holding two +// embedded messages and returns the backend plus the generation ID. +func openListBackend(t *testing.T) (*sqlitevec.Backend, vector.GenerationID) { + t.Helper() + backend, err := sqlitevec.Open(t.Context(), sqlitevec.Options{ + Path: filepath.Join(t.TempDir(), "vectors.db"), + Dimension: 4, + }) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, backend.Close()) }) + generationID, err := backend.CreateGeneration(t.Context(), "model", 4, "model:test") + require.NoError(t, err) + require.NoError(t, backend.Upsert(t.Context(), generationID, []vector.Chunk{ + {MessageID: 10, Vector: []float32{1, 0, 0, 0}}, + {MessageID: 11, Vector: []float32{0, 1, 0, 0}}, + })) + return backend, generationID +} + +func TestReadEmbeddingAcceleratorDisplayStates(t *testing.T) { + t.Run("absent displays exact", func(t *testing.T) { + backend, generationID := openListBackend(t) + + row, err := readEmbeddingAccelerator(t.Context(), backend, generationID) + require.NoError(t, err) + assert.Equal(t, "exact", row.State) + }) + + t.Run("building displays building", func(t *testing.T) { + backend, generationID := openListBackend(t) + installListAcceleratorRow(t, backend, generationID, "building") + + row, err := readEmbeddingAccelerator(t.Context(), backend, generationID) + require.NoError(t, err) + assert.Equal(t, "building", row.State) + }) + + t.Run("eligible stored ready displays ready", func(t *testing.T) { + backend, generationID := openListBackend(t) + installListAcceleratorRow(t, backend, generationID, "ready") + + row, err := readEmbeddingAccelerator(t.Context(), backend, generationID) + require.NoError(t, err) + assert.Equal(t, "ready", row.State) + assert.Equal(t, int64(2), row.IndexedCount) + }) + + t.Run("ineligible stored ready displays stale", func(t *testing.T) { + backend, generationID := openListBackend(t) + installListAcceleratorRow(t, backend, generationID, "ready") + + // A raw embeddings write fires the vector_revision maintenance + // trigger, making the stored-ready accelerator ineligible for + // search. The stored row still says ready; the CLI must display the + // effective state: stale. + _, err := backend.DB().Exec(`UPDATE embeddings + SET message_id = message_id WHERE generation_id = ?`, int64(generationID)) + require.NoError(t, err) + raw, err := backend.Accelerator(t.Context(), generationID) + require.NoError(t, err) + require.NotNil(t, raw) + assert.Equal(t, sqlitevec.AcceleratorReady, raw.State, + "the stored state remains ready after the trigger-driven revision bump") + + row, err := readEmbeddingAccelerator(t.Context(), backend, generationID) + require.NoError(t, err) + assert.Equal(t, "stale", row.State, + "an ineligible stored-ready accelerator must display stale, not the raw ready state") + }) +} diff --git a/cmd/msgvault/cmd/embeddings_manage.go b/cmd/msgvault/cmd/embeddings_manage.go index 257d1ac10..b721dc88f 100644 --- a/cmd/msgvault/cmd/embeddings_manage.go +++ b/cmd/msgvault/cmd/embeddings_manage.go @@ -57,6 +57,15 @@ type embeddingGenerationRow struct { EmbeddedCount int64 BlankCount int64 MissingCount int64 + Accelerator embeddingAcceleratorRow +} + +type embeddingAcceleratorRow struct { + State string + IndexedCount int64 + StartedAt *time.Time + CompletedAt *time.Time + LastError string } // fillFullCoverage populates the complete live/embedded/blank/missing split @@ -68,7 +77,11 @@ type embeddingGenerationRow struct { // live == embedded + blank + missing holds. The backend handle is passed // in by the caller (which already opened it for the generation listing). func fillFullCoverage(ctx context.Context, backend vector.Backend, scope vector.BuildScope, row *embeddingGenerationRow) error { - s, err := store.Open(cfg.DatabaseDSN()) + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + s, err := store.Open(state.cfg.DatabaseDSN()) if err != nil { return fmt.Errorf("open main db for coverage: %w", err) } @@ -94,8 +107,12 @@ func fillFullCoverage(ctx context.Context, backend vector.Backend, scope vector. // column) gets the column added before any management command reads // embed_gen via CoverageCounts. Mirrors the serve.go / runEmbed pattern. // Cheap and idempotent on an already-current schema; harmless on PG. -func ensureMainSchema() error { - s, err := store.Open(cfg.DatabaseDSN()) +func ensureMainSchema(state *invocation) error { + state = invocationState(context.Background(), state) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + s, err := store.Open(state.cfg.DatabaseDSN()) if err != nil { return fmt.Errorf("open main db: %w", err) } @@ -107,16 +124,21 @@ func ensureMainSchema() error { } func runEmbeddingsList(cmd *cobra.Command, _ []string) error { - release, err := acquireDirectSQLiteWriteLock(cfg) + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + release, err := acquireDirectSQLiteWriteLock(cfg, state) if err != nil { return err } defer release() - if err := ensureMainSchema(); err != nil { + if err := ensureMainSchema(state); err != nil { return err } - if err := ensureEmbedScopeResolved(); err != nil { + if err := ensureEmbedScopeResolved(state); err != nil { return err } db, rebind, closeDB, err := openEmbeddingsMetadataDB(cmd.Context()) @@ -146,26 +168,31 @@ func runEmbeddingsList(cmd *cobra.Command, _ []string) error { break } } - if needCoverage { + sqliteAcceleratorOnly := !store.IsPostgresURL(cfg.DatabaseDSN()) && sqlitevec.Available() + if needCoverage || sqliteAcceleratorOnly { backend, closeBackend, err := openEmbeddingsBackend(cmd.Context()) if err != nil { return err } defer closeBackend() for i := range rows { - if rows[i].State == vector.GenerationRetired { - continue + if rows[i].State != vector.GenerationRetired { + if err := fillFullCoverage(cmd.Context(), backend, cfg.Vector.Embed.Scope.BuildScope(), &rows[i]); err != nil { + return err + } } - if err := fillFullCoverage(cmd.Context(), backend, cfg.Vector.Embed.Scope.BuildScope(), &rows[i]); err != nil { + accelerator, err := readEmbeddingAccelerator(cmd.Context(), backend, rows[i].ID) + if err != nil { return err } + rows[i].Accelerator = accelerator } } w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 2, ' ', 0) - _, _ = fmt.Fprintln(w, "ID\tSTATE\tMODEL\tDIM\tLIVE\tEMBEDDED\tBLANK\tMISSING\tFINGERPRINT\tSTARTED\tCOMPLETED\tACTIVATED") + _, _ = fmt.Fprintln(w, "ID\tSTATE\tMODEL\tDIM\tLIVE\tEMBEDDED\tBLANK\tMISSING\tACCELERATOR\tANN_ROWS\tANN_STARTED\tANN_COMPLETED\tANN_ERROR\tFINGERPRINT\tSTARTED\tCOMPLETED\tACTIVATED") for _, row := range rows { - _, _ = fmt.Fprintf(w, "%d\t%s\t%s\t%d\t%d\t%d\t%d\t%d\t%s\t%s\t%s\t%s\n", + _, _ = fmt.Fprintf(w, "%d\t%s\t%s\t%d\t%d\t%d\t%d\t%d\t%s\t%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", row.ID, row.State, row.Model, @@ -174,6 +201,11 @@ func runEmbeddingsList(cmd *cobra.Command, _ []string) error { row.EmbeddedCount, row.BlankCount, row.MissingCount, + row.Accelerator.State, + row.Accelerator.IndexedCount, + formatGenerationTimePtr(row.Accelerator.StartedAt), + formatGenerationTimePtr(row.Accelerator.CompletedAt), + formatAcceleratorError(row.Accelerator.LastError), row.Fingerprint, formatGenerationTime(row.StartedAt), formatGenerationTimePtr(row.CompletedAt), @@ -186,6 +218,15 @@ func runEmbeddingsList(cmd *cobra.Command, _ []string) error { return nil } +func formatAcceleratorError(message string) string { + message = strings.NewReplacer("\t", " ", "\r", " ", "\n", " ").Replace(message) + runes := []rune(message) + if len(runes) > 160 { + return string(runes[:157]) + "..." + } + return message +} + func runEmbeddingsPruneCommand(cmd *cobra.Command, args []string) error { if !isDaemonCLISubprocess() { return runDaemonCLICommandHTTPFromCobra(cmd, args) @@ -194,13 +235,18 @@ func runEmbeddingsPruneCommand(cmd *cobra.Command, args []string) error { } func runEmbeddingsPrune(cmd *cobra.Command, _ []string) error { - release, err := acquireDirectSQLiteWriteLock(cfg) + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + release, err := acquireDirectSQLiteWriteLock(cfg, state) if err != nil { return err } defer release() - if err := ensureMainSchema(); err != nil { + if err := ensureMainSchema(state); err != nil { return err } backend, closeBackend, err := openEmbeddingsBackend(cmd.Context()) @@ -232,16 +278,21 @@ func errRetireActiveGeneration(gen vector.GenerationID) error { } func runEmbeddingsRetire(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg gen, err := parseGenerationID(args[0]) if err != nil { return err } - release, err := acquireDirectSQLiteWriteLock(cfg) + release, err := acquireDirectSQLiteWriteLock(cfg, state) if err != nil { return err } defer release() - if err := ensureMainSchema(); err != nil { + if err := ensureMainSchema(state); err != nil { return err } @@ -319,19 +370,24 @@ func runEmbeddingsRetireHTTP(cmd *cobra.Command, args []string) error { } func runEmbeddingsActivate(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg gen, err := parseGenerationID(args[0]) if err != nil { return err } - release, err := acquireDirectSQLiteWriteLock(cfg) + release, err := acquireDirectSQLiteWriteLock(cfg, state) if err != nil { return err } defer release() - if err := ensureMainSchema(); err != nil { + if err := ensureMainSchema(state); err != nil { return err } - if err := ensureEmbedScopeResolved(); err != nil { + if err := ensureEmbedScopeResolved(state); err != nil { return err } @@ -488,7 +544,7 @@ func planCLIEmbeddingsRetire( gen vector.GenerationID, forceActive bool, ) (api.CLIEmbeddingsPlanResponse, error) { - if err := ensureMainSchema(); err != nil { + if err := ensureMainSchema(invocationFromContext(ctx)); err != nil { return api.CLIEmbeddingsPlanResponse{}, err } db, rebind, closeDB, err := openEmbeddingsMetadataDB(ctx) @@ -521,13 +577,13 @@ func planCLIEmbeddingsActivate( gen vector.GenerationID, force bool, ) (api.CLIEmbeddingsPlanResponse, error) { - if err := ensureMainSchema(); err != nil { + if err := ensureMainSchema(invocationFromContext(ctx)); err != nil { return api.CLIEmbeddingsPlanResponse{}, err } // This runs on daemon HTTP handler goroutines, so the account scope is // resolved into a per-request config copy — mutating the shared global // cfg here would race with concurrent plan requests. - vecCfg, err := openResolvedVectorConfig() + vecCfg, err := openResolvedVectorConfig(invocationFromContext(ctx)) if err != nil { return api.CLIEmbeddingsPlanResponse{}, err } @@ -588,7 +644,11 @@ func requireConfiguredConvergence(ctx context.Context, vecCfg vector.Config, gen } func configuredConvergenceState(ctx context.Context, vecCfg vector.Config, gen vector.GenerationID) (scheduler.ConvergenceResult, error) { - mainStore, err := store.Open(cfg.DatabaseDSN()) + inv := invocationFromContext(ctx) + if inv == nil || inv.cfg == nil { + return scheduler.ConvergenceResult{}, errors.New("configuration is unavailable") + } + mainStore, err := store.Open(inv.cfg.DatabaseDSN()) if err != nil { return scheduler.ConvergenceResult{}, fmt.Errorf("open main db for convergence: %w", err) } @@ -599,7 +659,7 @@ func configuredConvergenceState(ctx context.Context, vecCfg vector.Config, gen v } defer closeBackend() personGate := vector.NewPinnedExactSemanticPersonEmbeddingGate( - vecCfg, currentSemanticPersonVectorConfigSource(), mainStore, + vecCfg, currentSemanticPersonVectorConfigSource(inv), mainStore, ) checker, err := newConvergenceChecker(vecCfg, mainStore, backend, personGate) if err != nil { @@ -629,6 +689,11 @@ func remainingCoverageHint(gen vector.GenerationID, remaining int64) string { // rebind converts ? placeholders to $1, $2, … for PostgreSQL; it is the // identity function for SQLite so all query helpers can use it unconditionally. func openEmbeddingsMetadataDB(ctx context.Context) (*sql.DB, func(string) string, func(), error) { + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil { + return nil, nil, nil, errors.New("configuration is unavailable") + } + cfg := state.cfg dsn := cfg.DatabaseDSN() if store.IsPostgresURL(dsn) { // Use the store-level PG opener so that connection runtime params @@ -690,6 +755,11 @@ func openEmbeddingsMetadataDB(ctx context.Context) (*sql.DB, func(string) string // Returns the backend and a close callback. On a build without the relevant // vector tag the package stubs' Open returns ErrNotBuilt. func openEmbeddingsBackend(ctx context.Context) (vector.Backend, func(), error) { + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil { + return nil, nil, errors.New("configuration is unavailable") + } + cfg := state.cfg dsn := cfg.DatabaseDSN() if store.IsPostgresURL(dsn) { db, cleanup, err := store.OpenPostgresDB(dsn) @@ -746,11 +816,14 @@ func openEmbeddingsBackend(ctx context.Context) (vector.Backend, func(), error) return nil, nil, fmt.Errorf("open main db for embeddings backend: %w", err) } b, err := sqlitevec.Open(ctx, sqlitevec.Options{ - Path: vecPath, - MainPath: dsn, - Dimension: cfg.Vector.Embeddings.Dimension, - MainDB: mainStore.DB(), - BuildScope: cfg.Vector.Embed.Scope.BuildScope(), + Path: vecPath, + MainPath: dsn, + Dimension: cfg.Vector.Embeddings.Dimension, + MainDB: mainStore.DB(), + BuildScope: cfg.Vector.Embed.Scope.BuildScope(), + ANNOversample: cfg.Vector.Search.ANNOversample, + ANNNProbe: cfg.Vector.Search.ANNNProbe, + AcceleratorMode: cfg.Vector.Search.SQLiteAccelerator, }) if err != nil { _ = mainStore.Close() diff --git a/cmd/msgvault/cmd/embeddings_manage_nosqlitevec_test.go b/cmd/msgvault/cmd/embeddings_manage_nosqlitevec_test.go new file mode 100644 index 000000000..1ddc30dcb --- /dev/null +++ b/cmd/msgvault/cmd/embeddings_manage_nosqlitevec_test.go @@ -0,0 +1,72 @@ +//go:build !sqlite_vec + +package cmd + +import ( + "bytes" + "database/sql" + "path/filepath" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/vector" +) + +func TestRunEmbeddingsListRetiredOnlySucceedsWithoutSQLiteVec(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + dir := t.TempDir() + vectorPath := filepath.Join(dir, "vectors.db") + c := config.NewDefaultConfig() + c.HomeDir = dir + c.Data.DataDir = dir + c.Vector.Enabled = true + c.Vector.DBPath = vectorPath + c.Vector.Embeddings.Model = "test-model" + c.Vector.Embeddings.Dimension = 4 + testCtx := withTestConfig(t, c) + _ = testCtx + + mainStore, err := store.Open(c.DatabaseDSN()) + require.NoError(err) + require.NoError(mainStore.InitSchema()) + require.NoError(mainStore.Close()) + + vectorsDB, err := sql.Open("sqlite3", vectorPath) + require.NoError(err) + _, err = vectorsDB.Exec(`CREATE TABLE index_generations ( + id INTEGER PRIMARY KEY, + model TEXT NOT NULL, + dimension INTEGER NOT NULL, + fingerprint TEXT NOT NULL, + state TEXT NOT NULL, + started_at INTEGER NOT NULL, + completed_at INTEGER, + activated_at INTEGER, + message_count INTEGER NOT NULL DEFAULT 0, + seeded_at INTEGER + )`) + require.NoError(err) + _, err = vectorsDB.Exec(`INSERT INTO index_generations + (id, model, dimension, fingerprint, state, started_at, message_count) + VALUES (1, 'test-model', 4, 'test:4', ?, 1700000000, 0)`, + string(vector.GenerationRetired)) + require.NoError(err) + require.NoError(vectorsDB.Close()) + + var output bytes.Buffer + command := &cobra.Command{Use: "list"} + command.SetContext(testCtx) + command.SetContext(testCtx) + command.SetOut(&output) + require.NoError(runEmbeddingsList(command, nil), + "retired-only listing must not require the sqlite_vec backend") + assert.Contains(output.String(), "test-model") + assert.Contains(output.String(), string(vector.GenerationRetired)) + assert.Contains(output.String(), "test:4") +} diff --git a/cmd/msgvault/cmd/embeddings_manage_test.go b/cmd/msgvault/cmd/embeddings_manage_test.go index d7c71f624..21d23bb8e 100644 --- a/cmd/msgvault/cmd/embeddings_manage_test.go +++ b/cmd/msgvault/cmd/embeddings_manage_test.go @@ -5,6 +5,7 @@ package cmd import ( "bytes" "context" + "io" "path/filepath" "strconv" "testing" @@ -30,18 +31,19 @@ func TestRunEmbeddingsPruneRemovesOrphansWithoutEmbeddingCalls(t *testing.T) { c.Vector.DBPath = vectorPath c.Vector.Embeddings.Model = "test-model" c.Vector.Embeddings.Dimension = 4 - withTestConfig(t, c) + testCtx := withTestConfig(t, c) + _ = testCtx mainStore, err := store.Open(mainPath) require.NoError(t, err) require.NoError(t, mainStore.InitSchema()) - backend, err := sqlitevec.Open(t.Context(), sqlitevec.Options{ + backend, err := sqlitevec.Open(testCtx, sqlitevec.Options{ Path: vectorPath, MainPath: mainPath, Dimension: 4, MainDB: mainStore.DB(), }) require.NoError(t, err) - generation, err := backend.CreateGeneration(t.Context(), "test-model", 4, "test:4") + generation, err := backend.CreateGeneration(testCtx, "test-model", 4, "test:4") require.NoError(t, err) - require.NoError(t, backend.Upsert(t.Context(), generation, []vector.Chunk{ + require.NoError(t, backend.Upsert(testCtx, generation, []vector.Chunk{ {MessageID: 9001, Vector: []float32{1, 0, 0, 0}}, })) require.NoError(t, backend.Close()) @@ -49,21 +51,96 @@ func TestRunEmbeddingsPruneRemovesOrphansWithoutEmbeddingCalls(t *testing.T) { var output bytes.Buffer command := &cobra.Command{Use: "prune"} - command.SetContext(t.Context()) + command.SetContext(testCtx) + command.SetContext(testCtx) command.SetOut(&output) require.NoError(t, runEmbeddingsPrune(command, nil)) assert.Equal(t, "Pruned 1 orphan message embedding(s).\n", output.String()) - reopened, err := sqlitevec.Open(t.Context(), sqlitevec.Options{ + reopened, err := sqlitevec.Open(testCtx, sqlitevec.Options{ Path: vectorPath, MainPath: mainPath, Dimension: 4, }) require.NoError(t, err) t.Cleanup(func() { _ = reopened.Close() }) - stats, err := reopened.Stats(t.Context(), generation) + stats, err := reopened.Stats(testCtx, generation) require.NoError(t, err) assert.Zero(t, stats.EmbeddingCount) } +func TestRunEmbeddingsOptimizeBuildsFromStoredVectorsWithoutProvider(t *testing.T) { + dir := t.TempDir() + mainPath := filepath.Join(dir, "msgvault.db") + vectorPath := filepath.Join(dir, "vectors.db") + c := config.NewDefaultConfig() + c.HomeDir = dir + c.Data.DataDir = dir + c.Vector.Enabled = true + c.Vector.DBPath = vectorPath + c.Vector.Embeddings.Model = "test-model" + c.Vector.Embeddings.Dimension = 4 + c.Vector.Search.ANNThreads = 1 + testCtx := withTestConfig(t, c) + _ = testCtx + + mainStore, err := store.Open(mainPath) + require.NoError(t, err) + require.NoError(t, mainStore.InitSchema()) + backend, err := sqlitevec.Open(testCtx, sqlitevec.Options{ + Path: vectorPath, MainPath: mainPath, Dimension: 4, MainDB: mainStore.DB(), + }) + require.NoError(t, err) + generation, err := backend.CreateGeneration(testCtx, "test-model", 4, "test:4") + require.NoError(t, err) + chunks := make([]vector.Chunk, 512) + for i := range chunks { + chunks[i] = vector.Chunk{MessageID: int64(i + 1), Vector: []float32{float32(i % 17), 1, 2, 3}} + } + require.NoError(t, backend.Upsert(testCtx, generation, chunks)) + require.NoError(t, backend.Close()) + require.NoError(t, mainStore.Close()) + + originalWorker := runAcceleratorWorkerSubprocess + runAcceleratorWorkerSubprocess = func( + ctx context.Context, _ string, databasePath string, generationID vector.GenerationID, threads int, _ io.Writer, + ) error { + return sqlitevec.RunAcceleratorWorker(context.WithoutCancel(ctx), databasePath, generationID, threads) + } + t.Cleanup(func() { runAcceleratorWorkerSubprocess = originalWorker }) + + var stdout, stderr bytes.Buffer + command := &cobra.Command{Use: "optimize"} + command.SetContext(testCtx) + command.Flags().Bool("drop", false, "") + command.SetContext(testCtx) + command.SetOut(&stdout) + command.SetErr(&stderr) + require.NoError(t, runEmbeddingsOptimize(command, []string{strconv.FormatInt(int64(generation), 10)})) + assert.Contains(t, stdout.String(), "accelerator ready (512 vectors)") + assert.Contains(t, stderr.String(), "Training accelerator") + + reopened, err := sqlitevec.Open(testCtx, sqlitevec.Options{Path: vectorPath, Dimension: 4}) + require.NoError(t, err) + t.Cleanup(func() { _ = reopened.Close() }) + status, err := reopened.Accelerator(testCtx, generation) + require.NoError(t, err) + require.NotNil(t, status) + assert.Equal(t, sqlitevec.AcceleratorReady, status.State) + assert.Equal(t, int64(512), status.IndexedCount) + assert.NotNil(t, status.CompletedAt) + + require.NoError(t, reopened.RetireGeneration(testCtx, generation, false)) + require.NoError(t, command.Flags().Set("drop", "true")) + stdout.Reset() + require.NoError(t, runEmbeddingsOptimize(command, []string{strconv.FormatInt(int64(generation), 10)})) + assert.Contains(t, stdout.String(), "accelerator dropped") + status, err = reopened.Accelerator(testCtx, generation) + require.NoError(t, err) + assert.Nil(t, status) + stats, err := reopened.Stats(testCtx, generation) + require.NoError(t, err) + assert.Equal(t, int64(512), stats.EmbeddingCount) +} + type convergenceProgressPublisher struct { vector.DocumentPublisher @@ -158,7 +235,8 @@ func TestRunEmbeddingsActivate_ContextualRequiresConvergenceUnlessForced(t *test c.Vector.Embeddings.Endpoint = "https://example.invalid/v1" c.Vector.Embeddings.Model = "voyage-context-4" c.Vector.Embeddings.Dimension = 4 - withTestConfig(t, c) + testCtx := withTestConfig(t, c) + _ = testCtx mainStore, err := store.Open(mainPath) require.NoError(err) @@ -166,14 +244,14 @@ func TestRunEmbeddingsActivate_ContextualRequiresConvergenceUnlessForced(t *test _, err = mainStore.DB().Exec(`UPDATE embedding_change_clock SET sequence = 2 WHERE singleton = 1`) require.NoError(err) require.NoError(sqlitevec.RegisterExtension()) - backend, err := sqlitevec.Open(t.Context(), sqlitevec.Options{ + backend, err := sqlitevec.Open(testCtx, sqlitevec.Options{ Path: vectorPath, MainPath: mainPath, Dimension: 4, MainDB: mainStore.DB(), }) require.NoError(err) - gen, err := backend.CreateGeneration(t.Context(), c.Vector.Embeddings.Model, 4, c.Vector.GenerationFingerprint()) + gen, err := backend.CreateGeneration(testCtx, c.Vector.Embeddings.Model, 4, c.Vector.GenerationFingerprint()) require.NoError(err) - require.NoError(backend.AdvanceDocumentChangeWatermark(t.Context(), gen, 1)) - require.NoError(backend.SetDocumentReconcileCursor(t.Context(), gen, "done:2")) + require.NoError(backend.AdvanceDocumentChangeWatermark(testCtx, gen, 1)) + require.NoError(backend.SetDocumentReconcileCursor(testCtx, gen, "done:2")) require.NoError(backend.Close()) require.NoError(mainStore.Close()) @@ -183,7 +261,7 @@ func TestRunEmbeddingsActivate_ContextualRequiresConvergenceUnlessForced(t *test embeddingsActivateForce = false cmd := embeddingsActivateCmd previousContext := cmd.Context() - cmd.SetContext(t.Context()) + cmd.SetContext(testCtx) t.Cleanup(func() { cmd.SetContext(previousContext) }) var output bytes.Buffer cmd.SetOut(&output) @@ -219,7 +297,8 @@ func TestRunEmbeddingsActivateOpenAIBlocksMissingPersonCoverage(t *testing.T) { c.Vector.People = vector.PeopleConfig{ Enabled: true, RetentionPosture: "zero_data_retention", TrainingPosture: "no_training", } - withTestConfig(t, c) + testCtx := withTestConfig(t, c) + _ = testCtx require.NoError(c.Save()) mainStore, err := store.Open(mainPath) @@ -227,21 +306,21 @@ func TestRunEmbeddingsActivateOpenAIBlocksMissingPersonCoverage(t *testing.T) { require.NoError(mainStore.InitSchema()) semanticProfile, err := c.Vector.SemanticPersonEmbeddingProfile() require.NoError(err) - _, err = mainStore.EnsurePersonSemanticEmbeddingProfile(t.Context(), semanticProfile) + _, err = mainStore.EnsurePersonSemanticEmbeddingProfile(testCtx, semanticProfile) require.NoError(err) _, _, err = mainStore.GrantPersonSemanticEmbeddingConsent( - t.Context(), semanticProfile.Fingerprint, "test", + testCtx, semanticProfile.Fingerprint, "test", ) require.NoError(err) _, err = mainStore.DB().Exec(`INSERT INTO persons (vcard_uid, display_name) VALUES (?, ?)`, "urn:uuid:00000000-0000-0000-0000-000000000002", "Synthetic Missing Person") require.NoError(err) require.NoError(sqlitevec.RegisterExtension()) - backend, err := sqlitevec.Open(t.Context(), sqlitevec.Options{ + backend, err := sqlitevec.Open(testCtx, sqlitevec.Options{ Path: vectorPath, MainPath: mainPath, Dimension: 4, MainDB: mainStore.DB(), }) require.NoError(err) - gen, err := backend.CreateGeneration(t.Context(), c.Vector.Embeddings.Model, 4, c.Vector.GenerationFingerprint()) + gen, err := backend.CreateGeneration(testCtx, c.Vector.Embeddings.Model, 4, c.Vector.GenerationFingerprint()) require.NoError(err) require.NoError(backend.Close()) require.NoError(mainStore.Close()) @@ -251,7 +330,7 @@ func TestRunEmbeddingsActivateOpenAIBlocksMissingPersonCoverage(t *testing.T) { embeddingsActivateYes = true embeddingsActivateForce = false previousContext := embeddingsActivateCmd.Context() - embeddingsActivateCmd.SetContext(t.Context()) + embeddingsActivateCmd.SetContext(testCtx) t.Cleanup(func() { embeddingsActivateCmd.SetContext(previousContext) }) err = runEmbeddingsActivate(embeddingsActivateCmd, @@ -279,7 +358,8 @@ func setupManualContextualGeneration(t *testing.T, fingerprint string, retire bo c.Vector.Embeddings.Endpoint = "https://example.invalid/v1" c.Vector.Embeddings.Model = "voyage-context-4" c.Vector.Embeddings.Dimension = 4 - withTestConfig(t, c) + testCtx := withTestConfig(t, c) + _ = testCtx if fingerprint == "" { fingerprint = c.Vector.GenerationFingerprint() } @@ -288,14 +368,14 @@ func setupManualContextualGeneration(t *testing.T, fingerprint string, retire bo require.NoError(t, err) require.NoError(t, mainStore.InitSchema()) require.NoError(t, sqlitevec.RegisterExtension()) - backend, err := sqlitevec.Open(t.Context(), sqlitevec.Options{ + backend, err := sqlitevec.Open(testCtx, sqlitevec.Options{ Path: vectorPath, MainPath: mainPath, Dimension: 4, MainDB: mainStore.DB(), }) require.NoError(t, err) - gen, err := backend.CreateGeneration(t.Context(), c.Vector.Embeddings.Model, 4, fingerprint) + gen, err := backend.CreateGeneration(testCtx, c.Vector.Embeddings.Model, 4, fingerprint) require.NoError(t, err) if retire { - require.NoError(t, backend.RetireGeneration(t.Context(), gen, false)) + require.NoError(t, backend.RetireGeneration(testCtx, gen, false)) } require.NoError(t, backend.Close()) require.NoError(t, mainStore.Close()) @@ -305,17 +385,18 @@ func setupManualContextualGeneration(t *testing.T, fingerprint string, retire bo embeddingsActivateYes = true embeddingsActivateForce = false previousContext := embeddingsActivateCmd.Context() - embeddingsActivateCmd.SetContext(t.Context()) + embeddingsActivateCmd.SetContext(testCtx) t.Cleanup(func() { embeddingsActivateCmd.SetContext(previousContext) }) return gen } func assertManualGenerationState(t *testing.T, gen vector.GenerationID, want vector.GenerationState) { t.Helper() - db, rebind, closeDB, err := openEmbeddingsMetadataDB(t.Context()) + ctx := embeddingsActivateCmd.Context() + db, rebind, closeDB, err := openEmbeddingsMetadataDB(ctx) require.NoError(t, err) defer closeDB() - row, err := getEmbeddingGeneration(t.Context(), db, rebind, gen) + row, err := getEmbeddingGeneration(ctx, db, rebind, gen) require.NoError(t, err) assert.Equal(t, want, row.State) } diff --git a/cmd/msgvault/cmd/embeddings_optimize.go b/cmd/msgvault/cmd/embeddings_optimize.go new file mode 100644 index 000000000..bb9dcff18 --- /dev/null +++ b/cmd/msgvault/cmd/embeddings_optimize.go @@ -0,0 +1,169 @@ +//go:build sqlite_vec + +package cmd + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strconv" + + "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/vector" + "go.kenn.io/msgvault/internal/vector/sqlitevec" +) + +var embeddingsOptimizeCmd = &cobra.Command{ + Use: "optimize [generation-id]", + Short: "Build or remove the SQLite search accelerator", + Long: "Build a restartable SQLite approximate-search index from embeddings already stored locally. " + + "No text or embedding-provider request is made. When omitted, generation-id defaults to the active generation.", + Args: cobra.MaximumNArgs(1), + RunE: runEmbeddingsOptimizeCommand, +} + +type acceleratorWorkerRunner func(context.Context, string, string, vector.GenerationID, int, io.Writer) error + +var runAcceleratorWorkerSubprocess acceleratorWorkerRunner = func( + ctx context.Context, executable, databasePath string, generationID vector.GenerationID, threads int, stderr io.Writer, +) error { + // #nosec G204 -- executable is the path of the currently running msgvault binary, not user input. + worker := exec.CommandContext(ctx, executable, embeddingsCommandName, embeddingsOptimizeWorkerName, + databasePath, strconv.FormatInt(int64(generationID), 10), strconv.Itoa(threads)) + worker.Stderr = stderr + if err := worker.Run(); err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + return fmt.Errorf("accelerator worker: %w", err) + } + return nil +} + +func runEmbeddingsOptimizeCommand(cmd *cobra.Command, args []string) error { + if !isDaemonCLISubprocess() { + return runDaemonCLICommandHTTPFromCobra(cmd, args) + } + return runEmbeddingsOptimize(cmd, args) +} + +func runEmbeddingsOptimize(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + if !cfg.Vector.Enabled { + return errors.New("vector search not enabled; add [vector] enabled=true to config.toml first") + } + if store.IsPostgresURL(cfg.DatabaseDSN()) { + return errors.New("embeddings optimize is only needed for SQLite; PostgreSQL uses its native vector index") + } + release, err := acquireDirectSQLiteWriteLock(cfg, state) + if err != nil { + return err + } + defer release() + if err := ensureMainSchema(state); err != nil { + return err + } + backend, closeBackend, err := openEmbeddingsBackend(cmd.Context()) + if err != nil { + return err + } + sqliteBackend, ok := backend.(*sqlitevec.Backend) + if !ok { + closeBackend() + return errors.New("configured vector backend is not SQLite") + } + generationID, err := optimizeGenerationID(cmd.Context(), sqliteBackend, args) + if err != nil { + closeBackend() + return err + } + drop, err := cmd.Flags().GetBool("drop") + if err != nil { + closeBackend() + return fmt.Errorf("read --drop: %w", err) + } + if drop { + defer closeBackend() + if err := sqliteBackend.DropAccelerator(cmd.Context(), generationID); err != nil { + return err + } + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Generation %d accelerator dropped; exact vectors retained.\n", generationID) + return nil + } + plan, err := sqliteBackend.PrepareAccelerator(cmd.Context(), generationID, sqlitevec.OptimizeOptions{ + Threads: cfg.Vector.Search.ANNThreads, + Progress: func(progress sqlitevec.OptimizeProgress) { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Preparing accelerator: %d/%d vectors\n", progress.IndexedCount, progress.TotalCount) + }, + }) + if err != nil { + closeBackend() + return fmt.Errorf("prepare accelerator: %w", err) + } + if !plan.Applicable { + closeBackend() + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Generation %d remains on exact search: %s.\n", generationID, plan.Reason) + return nil + } + if plan.AlreadyReady { + closeBackend() + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Generation %d accelerator is already ready (%d vectors).\n", generationID, plan.IndexedCount) + return nil + } + closeBackend() + + executable, err := os.Executable() + if err != nil { + return fmt.Errorf("resolve msgvault executable: %w", err) + } + vectorPath := cfg.Vector.DBPath + if vectorPath == "" { + vectorPath = filepath.Join(cfg.Data.DataDir, "vectors.db") + } + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Training accelerator for generation %d with %d thread(s)...\n", generationID, plan.Threads) + if err := runAcceleratorWorkerSubprocess(cmd.Context(), executable, vectorPath, generationID, plan.Threads, cmd.ErrOrStderr()); err != nil { + if reopened, closeReopened, reopenErr := openEmbeddingsBackend(context.WithoutCancel(cmd.Context())); reopenErr == nil { + if concrete, isSQLite := reopened.(*sqlitevec.Backend); isSQLite { + _ = concrete.RecordAcceleratorError(context.WithoutCancel(cmd.Context()), generationID, err) + } + closeReopened() + } + return err + } + reopened, closeReopened, err := openEmbeddingsBackend(cmd.Context()) + if err != nil { + return err + } + defer closeReopened() + concrete, ok := reopened.(*sqlitevec.Backend) + if !ok { + return errors.New("configured vector backend changed while optimizing") + } + status, err := concrete.PublishAccelerator(cmd.Context(), generationID) + if err != nil { + _ = concrete.RecordAcceleratorError(context.WithoutCancel(cmd.Context()), generationID, err) + return fmt.Errorf("publish accelerator: %w", err) + } + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Generation %d accelerator ready (%d vectors).\n", generationID, status.IndexedCount) + return nil +} + +func optimizeGenerationID(ctx context.Context, backend *sqlitevec.Backend, args []string) (vector.GenerationID, error) { + if len(args) == 1 { + return parseGenerationID(args[0]) + } + active, err := backend.ActiveGeneration(ctx) + if err != nil { + return 0, fmt.Errorf("resolve active generation: %w", err) + } + return active.ID, nil +} diff --git a/cmd/msgvault/cmd/embeddings_optimize_stub.go b/cmd/msgvault/cmd/embeddings_optimize_stub.go new file mode 100644 index 000000000..5a6c3bbc9 --- /dev/null +++ b/cmd/msgvault/cmd/embeddings_optimize_stub.go @@ -0,0 +1,18 @@ +//go:build !sqlite_vec + +package cmd + +import ( + "errors" + + "github.com/spf13/cobra" +) + +var embeddingsOptimizeCmd = &cobra.Command{ + Use: "optimize [generation-id]", + Short: "Build or remove the SQLite search accelerator", + Args: cobra.MaximumNArgs(1), + RunE: func(*cobra.Command, []string) error { + return errors.New("SQLite accelerator unavailable without sqlite_vec") + }, +} diff --git a/cmd/msgvault/cmd/embeddings_optimize_worker.go b/cmd/msgvault/cmd/embeddings_optimize_worker.go new file mode 100644 index 000000000..6596dede7 --- /dev/null +++ b/cmd/msgvault/cmd/embeddings_optimize_worker.go @@ -0,0 +1,35 @@ +//go:build sqlite_vec + +package cmd + +import ( + "context" + "fmt" + "strconv" + + "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/vector" + "go.kenn.io/msgvault/internal/vector/sqlitevec" +) + +var embeddingsOptimizeWorkerCmd = &cobra.Command{ + Use: embeddingsOptimizeWorkerName + " ", + Args: cobra.ExactArgs(3), + Hidden: true, + RunE: func(cmd *cobra.Command, args []string) error { + generationID, err := strconv.ParseInt(args[1], 10, 64) + if err != nil || generationID <= 0 { + return fmt.Errorf("invalid accelerator generation %q", args[1]) + } + threads, err := strconv.Atoi(args[2]) + if err != nil || threads <= 0 { + return fmt.Errorf("invalid accelerator worker count %q", args[2]) + } + // The parent owns cancellation by killing this disposable process. + // Passing Cobra's canceled context into sqlite3 would call + // sqlite3_interrupt while Vec1 is inside a native training loop. + return sqlitevec.RunAcceleratorWorker( + context.WithoutCancel(cmd.Context()), args[0], vector.GenerationID(generationID), threads, + ) + }, +} diff --git a/cmd/msgvault/cmd/embeddings_optimize_worker_stub.go b/cmd/msgvault/cmd/embeddings_optimize_worker_stub.go new file mode 100644 index 000000000..23b4bbac5 --- /dev/null +++ b/cmd/msgvault/cmd/embeddings_optimize_worker_stub.go @@ -0,0 +1,17 @@ +//go:build !sqlite_vec + +package cmd + +import ( + "errors" + + "github.com/spf13/cobra" +) + +var embeddingsOptimizeWorkerCmd = &cobra.Command{ + Use: embeddingsOptimizeWorkerName, + Hidden: true, + RunE: func(*cobra.Command, []string) error { + return errors.New("SQLite accelerator worker unavailable without sqlite_vec") + }, +} diff --git a/cmd/msgvault/cmd/employment_test.go b/cmd/msgvault/cmd/employment_test.go index 1991e0243..586f94bf5 100644 --- a/cmd/msgvault/cmd/employment_test.go +++ b/cmd/msgvault/cmd/employment_test.go @@ -2,6 +2,7 @@ package cmd import ( "bytes" + "context" "encoding/json" "net/http" "net/http/httptest" @@ -32,8 +33,9 @@ func TestEmploymentAddSendsPartialDatesAndOmitsUndecidedPrimary(t *testing.T) { assert.NoError(err) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) - output := runEmploymentCommand(t, employmentAddCmd, []string{"--person", "3", "--organization", "4", "--title", "Staff Engineer", "--start", "2019-04"}) + testCtx := withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) + _ = testCtx + output := runEmploymentCommand(testCtx, t, employmentAddCmd, []string{"--person", "3", "--organization", "4", "--title", "Staff Engineer", "--start", "2019-04"}) require.NoError(decodeErr) assert.Equal(json.Number("3"), body["person_id"]) assert.Equal(json.Number("4"), body["organization_id"]) @@ -80,8 +82,9 @@ func TestEmploymentEndReadsRevisionAndSendsEndDate(t *testing.T) { } })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) - output := runEmploymentCommand(t, employmentEndCmd, []string{"9", "--end", "2026-06"}) + testCtx := withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) + _ = testCtx + output := runEmploymentCommand(testCtx, t, employmentEndCmd, []string{"9", "--end", "2026-06"}) require.NoError(decodeErr) assert.Equal(`"employment-9-r2"`, ifMatch) assert.Equal("2026-06", body["end_date"]) @@ -114,8 +117,9 @@ func TestEmploymentSetPreservesFieldsNotPassedAsFlags(t *testing.T) { } })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) - output := runEmploymentCommand(t, employmentSetCmd, []string{"9", "--title", "Principal Engineer"}) + testCtx := withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) + _ = testCtx + output := runEmploymentCommand(testCtx, t, employmentSetCmd, []string{"9", "--title", "Principal Engineer"}) require.NoError(decodeErr) assert.Equal(`"employment-9-r2"`, ifMatch) assert.Equal("Principal Engineer", body["title"]) @@ -153,8 +157,9 @@ func TestEmploymentSetEndingAPrimaryEmploymentDemotesIt(t *testing.T) { } })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) - runEmploymentCommand(t, employmentSetCmd, []string{"9", "--end", "2026-06"}) + testCtx := withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) + _ = testCtx + runEmploymentCommand(testCtx, t, employmentSetCmd, []string{"9", "--end", "2026-06"}) require.NoError(decodeErr) assert.Equal(false, body["is_current"]) assert.Equal(false, body["is_primary"], @@ -170,8 +175,9 @@ func TestEmploymentSetRejectsPrimaryCombinedWithNotCurrent(t *testing.T) { assert.NoError(err) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) + testCtx := withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) command := cloneEmploymentCommand(employmentSetCmd) + command.SetContext(testCtx) var output bytes.Buffer command.SetOut(&output) command.SetErr(&output) @@ -192,8 +198,9 @@ func TestEmploymentListPrintsHistoryAndProjection(t *testing.T) { assert.NoError(err) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) - output := runEmploymentCommand(t, employmentListCmd, []string{"--person", "3"}) + testCtx := withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) + _ = testCtx + output := runEmploymentCommand(testCtx, t, employmentListCmd, []string{"--person", "3"}) assert.Contains(output, "Staff Engineer") assert.Contains(output, "Junior Engineer") assert.Contains(output, "2018-06") @@ -223,8 +230,9 @@ func TestEmploymentSetReactivatesEndedEmploymentWithClearEnd(t *testing.T) { } })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) - runEmploymentCommand(t, employmentSetCmd, []string{"9", "--current", "--clear-end"}) + testCtx := withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) + _ = testCtx + runEmploymentCommand(testCtx, t, employmentSetCmd, []string{"9", "--current", "--clear-end"}) require.NoError(decodeErr) assert.Equal(true, body["is_current"], "an ended employment must be reactivatable") @@ -240,7 +248,7 @@ func TestEmploymentSetRejectsEmptyAndConflictingStateFlags(t *testing.T) { assert.NoError(t, err) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) + testCtx := withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) for _, test := range []struct { name string args []string @@ -257,6 +265,7 @@ func TestEmploymentSetRejectsEmptyAndConflictingStateFlags(t *testing.T) { } { t.Run(test.name, func(t *testing.T) { command := cloneEmploymentCommand(employmentSetCmd) + command.SetContext(testCtx) var output bytes.Buffer command.SetOut(&output) command.SetErr(&output) @@ -280,8 +289,9 @@ func TestEmploymentListByOrganizationShowsPersonColumn(t *testing.T) { assert.NoError(err) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) - output := runEmploymentCommand(t, employmentListCmd, + testCtx := withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) + _ = testCtx + output := runEmploymentCommand(testCtx, t, employmentListCmd, []string{"--organization", "4", "--limit", "25", "--offset", "50"}) assert.Contains(rawQuery, "limit=25", "pagination must reach the server") assert.Contains(rawQuery, "offset=50") @@ -315,12 +325,13 @@ func cloneEmploymentCommand(template *cobra.Command) *cobra.Command { }) return command } -func runEmploymentCommand(t *testing.T, template *cobra.Command, args []string) string { +func runEmploymentCommand(ctx context.Context, t *testing.T, template *cobra.Command, args []string) string { t.Helper() saved := employmentJSON employmentJSON = false t.Cleanup(func() { employmentJSON = saved }) command := cloneEmploymentCommand(template) + command.SetContext(ctx) var output bytes.Buffer command.SetOut(&output) command.SetErr(&output) diff --git a/cmd/msgvault/cmd/eval.go b/cmd/msgvault/cmd/eval.go new file mode 100644 index 000000000..c750e4c28 --- /dev/null +++ b/cmd/msgvault/cmd/eval.go @@ -0,0 +1,1627 @@ +//go:build sqlite_vec + +package cmd + +import ( + "context" + "database/sql" + "encoding/json/v2" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "slices" + "sort" + "strconv" + "strings" + "text/tabwriter" + "time" + + "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/eval" + "go.kenn.io/msgvault/internal/providercredentials" + "go.kenn.io/msgvault/internal/query" + "go.kenn.io/msgvault/internal/search" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/vector" + "go.kenn.io/msgvault/internal/vector/hybrid" + "go.kenn.io/msgvault/internal/vector/rerank" + "go.kenn.io/msgvault/internal/vector/sqlitevec" +) + +var ( + evalQrels string + evalTopics string + evalModes string + evalDocKey string + evalLimit int + evalJSON bool +) + +var evalCmd = &cobra.Command{ + Use: "eval", + Short: "Evaluate retrieval quality against relevance judgments (qrels)", + Long: `Measure retrieval quality over a set of labeled queries. + +Runs each topic through one or more search modes (fts, vector, hybrid) against +the local archive and scores the ranking against relevance judgments using +standard IR metrics: precision@10, nDCG@10, recall@100, MAP and MRR. This makes +the effect of an indexing, embedding, or fusion change measurable rather than +guessed. MAP and MRR are reported as MAP@n / MRR@n: they take no cutoff of +their own, but the ranking they score stops at -n, so a relevant message below +that rank is as invisible to them as it is to recall. + +Inputs (TREC-style): + --qrels judgments file, one per line: " " + (rel >= 1 means relevant; the iter column is ignored) + --topics queries file, tab-separated: "\t[\t]" + Each qid must appear once — it is the join key to --qrels, so a + repeat would score the same judgments twice and weight that query + twice over in every average; the file is rejected instead. + The optional third column labels the question's shape (e.g. + "pointed" for answerable-from-one-message, "spanning" for + requires-synthesizing-across-messages); when present, results + are also broken down per category. Two-column files work as-is. + +Doc ids in --qrels are matched against each message's source_message_id by +default (--doc-key=message), or its conversation's source_conversation_id +(--doc-key=conversation). Pick the one your judgments actually reference: an +mbox import, for example, keys each imported document by conversation, so +message-keyed qrels would score a flat zero against it. When the judged unit +is the conversation, a thread is counted once, at its best-ranked message — +and retrieval over-fetches messages so that -n really does yield up to n +distinct threads. Reported latency therefore includes that over-fetch. + +Both ids are assigned by the source and are unique only within it, while a +qrels doc id records no source at all. A run therefore stops before scoring +anything if the archive holds an id shared by two connected accounts: merging +two accounts' documents under one key would let an unjudged account's message +inherit a judged one's relevance. Accounts with disjoint id spaces (a mailbox +and a chat archive, say) score normally. + +Metric depths follow -n: the standard P@10 / nDCG@10 / R@100 are reported when +the run retrieves at least that deep, and are clamped to -n below it (a run +that only ever looks 20 deep has no recall@100, and labelling one would invite +a false comparison). MAP and MRR are always at -n, since the truncated ranking +is the whole of what they see. The column headers always name the depth +actually used. + +Each mode runs the same code production search runs. fts is the +relevance-ranked (BM25, subject-weighted) store path behind +/api/v1/search?mode=fts — not a chronological listing — so it honours the same +deletion scope and the same substring address-filter semantics real searches +do; vector and hybrid go through the hybrid engine with the fusion parameters +from your config. Query embedding follows [vector.embeddings] api_format, so an +index built through the Voyage contextual endpoint is queried through it too — +comparing "voyage-context-4" against an OpenAI-compatible model is a matter of +pointing the command at each config in turn. + +Every run reports the embedding model, api format, index settings and index +size that produced it, plus per-query latency, because a quality number is not +comparable — or even interpretable — without them. Those numbers describe what +was searched, not what merely exists: the corpus count covers live messages +only (dedup-hidden duplicates and messages deleted from their source account +are excluded, as they are from every search here), and the vector count covers +the active generation only, not the retired ones vectors.db still holds. That +generation is resolved once at the start of the run; each query still searches +whatever is active at query time, so a rebuild or activation you trigger while +an eval is in flight can leave the reported vector count stale for topics +scored after the swap. +Anything that went wrong without being fatal — unparseable judgment lines, +topics whose query string did not parse or parsed to no search criteria at +all, hits that could not be hydrated from the archive, rankings cut short by +the fusion pool, topics no mode could score — is reported under "Diagnostics" +rather than silently folded into the scores. So is partial qrels coverage: a +topic the judgments never mention cannot be scored, and the diagnostics say +how many of the topics file that leaves the headline numbers standing on. + +Note on topic phrasing: it is an experimental variable, not a constant. FTS5 +matches on AND semantics, so a verbose natural-language topic requires every +one of its words to appear in a message and will usually score near zero, +while its keyword reduction scores well. Dense retrieval can move the other +way. Compare runs only across the same topics file. + +Example: + msgvault eval --qrels qrels.txt --topics topics.tsv --modes fts,vector,hybrid -n 100`, + Args: cobra.NoArgs, + RunE: runEval, +} + +func init() { + rootCmd.AddCommand(evalCmd) + evalCmd.Flags().StringVar(&evalQrels, "qrels", "", "Path to TREC-format relevance judgments (required)") + evalCmd.Flags().StringVar(&evalTopics, "topics", "", "Path to topics TSV: \\t (required)") + evalCmd.Flags().StringVar(&evalModes, "modes", "fts,vector,hybrid", "Comma-separated search modes to evaluate") + // The registry's key set is fixed even though its entries are built per + // run, so rendering the usage string from a throwaway registry is safe. + evalCmd.Flags().StringVar(&evalDocKey, "doc-key", "message", "Which id qrels reference: "+docKeyNames(newDocKeyRegistry())) + evalCmd.Flags().IntVarP(&evalLimit, "limit", "n", 100, "Distinct documents retrieved per query") + evalCmd.Flags().BoolVar(&evalJSON, flagJSON, false, "Output as JSON") + evalCmd.Flags().StringVar(&evalRerankJev, "rerank-jev", "", "Opt in to TypeSafe Jev reranking; sends the query and bounded message text (per-candidate,batched)") + evalCmd.Flags().IntVar(&evalRerankTop, "rerank-top", 30, "Maximum messages to send to Jev per retrieved ranking") + evalCmd.Flags().IntVar(&evalRerankMaxRequests, "rerank-max-requests", 1000, "Maximum TypeSafe requests for this eval invocation") + evalCmd.Flags().Float64Var(&evalRerankInputUSDPerM, "rerank-input-usd-per-million", 0, "Required Jev input price in USD per million tokens") + evalCmd.Flags().Float64Var(&evalRerankOutputUSDPerM, "rerank-output-usd-per-million", 0, "Required Jev output price in USD per million tokens") + _ = evalCmd.MarkFlagRequired("qrels") + _ = evalCmd.MarkFlagRequired("topics") +} + +// errNoFreeText marks a topic that vector and hybrid modes structurally cannot +// answer: it parsed to filters only, so there is nothing to embed. It is a +// property of one topic, not of the run, so it is reported per cell instead of +// aborting and discarding every score computed so far. +var errNoFreeText = errors.New("topic has no free-text terms to embed") + +// evalHit is the doc-key-relevant projection of one retrieved message. +// +// The retrieval paths disagree about what a hit is: the store's +// relevance-ranked FTS path returns store.APIMessage, while the vector path +// hydrates hybrid hits into query.MessageSummary. A --doc-key has to mean the +// same thing whichever engine produced the hit, so both paths project into +// this one struct and the doc-key registry is defined over it alone. +type evalHit struct { + // MessageID is the archive's own row id. No registered doc-key uses it + // yet; it is carried because the judged-unit extension the registry + // documents (a reconstructed-thread id resolved through an external + // mapping) resolves from an id, not from text. + MessageID int64 + SourceMessageID string + SourceConversationID string +} + +// hitFromAPIMessage projects a store-path (relevance-ranked FTS) result. +func hitFromAPIMessage(m store.APIMessage) evalHit { + return evalHit{ + MessageID: m.ID, + SourceMessageID: m.SourceMessageID, + SourceConversationID: m.SourceConversationID, + } +} + +// hitFromSummary projects a hydrated vector/hybrid result. +func hitFromSummary(m query.MessageSummary) evalHit { + return evalHit{ + MessageID: m.ID, + SourceMessageID: m.SourceMessageID, + SourceConversationID: m.SourceConversationID, + } +} + +// evalIDColumn locates, in the archive, the column a doc-key's ids are read +// from. It exists so the cross-source collision check +// (requireDisjointSourceIDs) can be written once, over any key, without having +// to know what a particular key means. +type evalIDColumn struct { + // table is the alias the collision query gives the id's table: "m" for + // messages, "c" for the conversations reached through join. + table string + // column is the unqualified column name, and doubles as the name the + // error text shows a user. + column string + // join is the extra FROM clause needed to reach table, empty when the id + // lives on messages itself. + join string +} + +// expr renders the qualified column for use in the collision query. +func (c evalIDColumn) expr() string { return c.table + "." + c.column } + +// docKeySpec describes one --doc-key value. +type docKeySpec struct { + // extract pulls, from a retrieved hit, the stable document id that qrels + // judge against. It is the only place a doc-key's meaning lives: the + // scoring core (eval.Evaluate, eval.Aggregate, eval.DedupeKeys) operates + // on the opaque string keys these return and never learns what they + // identify. + extract func(evalHit) string + // collapses is true when the judged unit is coarser than the retrieved + // one, so several hits routinely fold into a single key. Retrieval then + // has to over-fetch messages to fill the requested depth with *distinct* + // keys — see eval.OverFetchPlan and evaluator.rankedKeys. + collapses bool + // idColumn says where in the archive extract's id comes from, so the run + // can check up front that no id in it names documents in two different + // connected sources — see requireDisjointSourceIDs, which refuses to run a + // key that leaves this unset rather than skipping the check. + idColumn evalIDColumn +} + +// newDocKeyRegistry builds the --doc-key registry for one run. +// +// It is a constructor rather than a package-level map so the registry is built +// after flags are parsed. That is what makes the extension story real: a +// future --doc-key=thread, resolving a reconstructed-thread id through an +// externally supplied message-id -> thread-id mapping file, is one more entry +// here, closing over the mapping this function loaded. A map initialised at +// program start could not hold that entry — the mapping file is named by a +// flag, and its contents are unknown until runEval runs. The CLI validation, +// the scoring core and the output paths all pick a new entry up unchanged. +func newDocKeyRegistry() map[string]docKeySpec { + return map[string]docKeySpec{ + // One message, one source_message_id. Duplicates are still collapsed + // (the same message synced from two accounts), but they are rare + // enough that the depth does not need padding for them. + "message": { + extract: func(h evalHit) string { return h.SourceMessageID }, + idColumn: evalIDColumn{table: "m", column: "source_message_id"}, + }, + // Many messages share one conversation, so filling n distinct threads + // takes more than n messages. + "conversation": { + extract: func(h evalHit) string { return h.SourceConversationID }, + collapses: true, + idColumn: evalIDColumn{ + table: "c", + column: "source_conversation_id", + join: "JOIN conversations c ON c.id = m.conversation_id", + }, + }, + } +} + +// docKeyNames renders the valid --doc-key values for usage and error text. +func docKeyNames(registry map[string]docKeySpec) string { + names := make([]string, 0, len(registry)) + for n := range registry { + names = append(names, n) + } + sort.Strings(names) + return strings.Join(names, "|") +} + +// runDiagnostics collects the non-fatal anomalies of a run. Each of these was +// previously either silent or fatal, and both are wrong for an instrument: a +// stale index that quietly drops hits is precisely the failure this command +// exists to expose, and a single unanswerable topic should not throw away +// every other topic's score. +type runDiagnostics struct { + QrelsLoad eval.LoadStats `json:"qrels_load"` + TopicsLoad eval.LoadStats `json:"topics_load"` + UnhydratedHits int `json:"unhydrated_hits,omitempty"` + // DepthShortfalls counts runs that ran out of over-fetch budget with the + // engine still willing to give more. PoolShortfalls counts runs that hit + // the hybrid engine's candidate-pool ceiling instead — the engine came + // back short, but more matching messages exist beyond the pool. The two + // are kept apart because only one of them is fixable by configuration, + // and neither may be confused with "the corpus genuinely ran out", which + // is not a shortfall at all. + DepthShortfalls int `json:"depth_shortfalls,omitempty"` + PoolShortfalls int `json:"pool_shortfalls,omitempty"` + SkippedCells []string `json:"skipped_cells,omitempty"` + // UnjudgedTopics names the topics the qrels file says nothing about. + // Leaving them out of the scoring is correct — an unjudged topic has + // nothing to be scored against — but leaving it *unsaid* is not: as soon + // as one topic is judged the run reports a headline number, and a qrels + // file that matches only a handful of a large topics file therefore + // reports it over a small, self-selected subset while looking like a + // complete run. Naming them makes the coverage of a run readable from its + // own output. + UnjudgedTopics []string `json:"unjudged_topics,omitempty"` + + // kPerSignal is the fusion pool size in force for this run, used only to + // make the PoolShortfalls note actionable. Unexported so it stays out of + // the JSON diagnostics block, where it would duplicate run_config. + kPerSignal int + + // scored is how many topics actually contributed a score, set once after + // the scoring loop finishes. It is not len(TopicsLoad)-len(UnjudgedTopics): + // a judged topic can still fail to score (an empty parsed query, or every + // mode skipping it), so the unjudged-topics note must read this field + // rather than recompute a count the loop already produced correctly — + // the same number the report's own topics_evaluated is built from. + // Unexported for the same reason as kPerSignal: it would duplicate + // topics_evaluated in the JSON diagnostics block. + scored int +} + +// skip records that one topic/mode combination could not be scored. +func (d *runDiagnostics) skip(topicID, mode, reason string) { + d.SkippedCells = append(d.SkippedCells, fmt.Sprintf("topic %s / %s: %s", topicID, mode, reason)) +} + +// skipTopic records that a topic could not be scored by any mode. Unlike skip, +// this is a property of the topic itself (a query string no mode can run), so +// it is reported once rather than once per mode. +func (d *runDiagnostics) skipTopic(topicID, reason string) { + d.SkippedCells = append(d.SkippedCells, fmt.Sprintf("topic %s: %s", topicID, reason)) +} + +// unjudged records a topic this qrels file never mentions. It is not a skip: +// nothing went wrong with the topic, there is simply nothing to score it +// against. It is tracked so the run can report how much of the topics file its +// headline numbers actually cover. +func (d *runDiagnostics) unjudged(topicID string) { + d.UnjudgedTopics = append(d.UnjudgedTopics, topicID) +} + +// notes renders the diagnostics as human-readable lines, empty when the run +// was clean. +func (d *runDiagnostics) notes() []string { + var out []string + for _, l := range []struct { + kind string + stats eval.LoadStats + }{{"qrels", d.QrelsLoad}, {"topics", d.TopicsLoad}} { + if l.stats.Skipped > 0 { + out = append(out, fmt.Sprintf("%s %s: %s — skipped lines did not match the expected format", + l.kind, l.stats.Path, l.stats)) + } + } + // Partial coverage is not an error — a topics file is often larger than + // the judgments gathered for it so far — but it changes what the headline + // numbers mean, so it is stated rather than inferred from the topic count. + if n := len(d.UnjudgedTopics); n > 0 { + // d.scored, not d.TopicsLoad.Parsed-n: a judged topic can still fail + // to score (an empty parsed query, every mode skipping it), so the + // remainder after subtracting only the unjudged ones overstates what + // the run actually covers — see the scored field's own doc comment. + out = append(out, fmt.Sprintf( + "%d of %d topics had no matching qrels entry and were not scored (%s); "+ + "the reported metrics cover %d of the topics file, so they describe a subset of %s — "+ + "check that the qids in both files refer to the same queries", + n, d.TopicsLoad.Parsed, eval.FormatIDList(d.UnjudgedTopics, 10), + d.scored, d.TopicsLoad.Path)) + } + if d.UnhydratedHits > 0 { + out = append(out, fmt.Sprintf( + "%d retrieved hits could not be hydrated back to a message row and were dropped from the ranking; "+ + "this usually means the vector index references deleted or unmigrated messages (re-run `msgvault embed`)", + d.UnhydratedHits)) + } + if d.DepthShortfalls > 0 { + out = append(out, fmt.Sprintf( + "%d topic/mode runs could not fill %d distinct %s keys within the over-fetch budget (%dx -n); "+ + "their metrics are computed over a shallower list than requested", + d.DepthShortfalls, evalLimit, evalDocKey, eval.MaxOverFetchFactor)) + } + if d.PoolShortfalls > 0 { + out = append(out, fmt.Sprintf( + "%d topic/mode runs stopped short of %d distinct %s keys because the fusion candidate pool "+ + "saturated%s: the engine returned fewer hits than asked for, but more matching messages "+ + "exist beyond the pool — this is a reachability limit, not an exhausted corpus. "+ + "Raise [vector.search].k_per_signal to rank deeper, and note that doing so changes the "+ + "fusion, so only compare runs at the same setting", + d.PoolShortfalls, evalLimit, evalDocKey, kPerSignalSuffix(d.kPerSignal))) + } + out = append(out, d.SkippedCells...) + return out +} + +// kPerSignalSuffix renders the fusion pool size for a diagnostic line, or +// nothing when the run never opened the vector path (so the number is unknown +// rather than zero). +func kPerSignalSuffix(k int) string { + if k <= 0 { + return "" + } + return fmt.Sprintf(" at k_per_signal=%d", k) +} + +// ftsSearcher is the production relevance-ranked full-text path. +// +// It is deliberately the Store's search, not query.Engine.Search: those are +// two different searches. query.Engine.Search returns matches in reverse +// chronological order with no relevance component at all, so scoring it as a +// *ranking* measures the archive's date distribution rather than its retrieval +// quality. Store.SearchMessagesQueryContext is the path /api/v1/search?mode=fts +// serves, ordering by the dialect's BM25 expression (subject-weighted) before +// falling back to recency — the same messages_fts index and the same weights +// the hybrid engine's BM25 leg fuses. It also matches production on the two +// semantics that silently move scores: it honours search.DeletionScope +// (active-only by default, so source-deleted messages are excluded), and its +// from:/to:/cc: filters are substring matches rather than exact-address +// equality. +// +// *store.Store satisfies this; the interface exists so tests can drive the +// over-fetch loop without a database. +type ftsSearcher interface { + SearchMessagesQueryContext( + ctx context.Context, q *search.Query, offset, limit int, + ) ([]store.APIMessage, int64, error) +} + +// evaluator bundles the engines and config needed to turn a query string into +// a ranked list of document ids for a given search mode. +type evaluator struct { + ctx context.Context + fts ftsSearcher + qeng query.Engine + heng *hybrid.Engine + key docKeySpec + limit int + prov eval.RunConfig + diag *runDiagnostics + captureHits bool + lastHits map[string]evalHit +} + +// fetchResult is one attempt at pulling raw hits out of a search engine. +type fetchResult struct { + // keys are the doc keys in the engine's rank order, before collapsing; + // duplicates and empty strings are expected and handled by the caller. + keys []string + // hits is aligned with keys and carries the source message for each + // key. It is used only by the opt-in rerank arm. + hits []evalHit + // raw is how many hits the engine returned. It is the count *before* key + // extraction, so "the engine gave back fewer than we asked for" — the + // signal that a deeper fetch cannot help — stays accurate even when some + // hits fail to hydrate. + raw int + // dropped is how many hits could not be hydrated back to a message row. + dropped int + // saturated reports that the engine filled its own candidate pool: it + // had at least one more candidate than it was willing to consider. It is + // what separates "the corpus ran out" from "the engine stopped looking", + // which look identical from the hit count alone. The hybrid engine's + // fused query caps each signal at k_per_signal, so it can hand back fewer + // hits than requested while the corpus still holds plenty more — see + // hybrid.ResultMeta.PoolSaturated. + saturated bool +} + +// rankedKeys turns a search engine into up to limit *distinct* doc keys, in +// ranked order. +// +// The collapse must happen before the truncation, not after: retrieving n +// messages and then collapsing them yields however many distinct threads +// happen to sit inside those n, which is not what "-n 100" claims. So for a +// collapsing doc-key this over-fetches raw hits (eval.OverFetchPlan), collapses, +// and only then cuts to the requested depth, growing the pool while the engine +// still has more to give and the depth is still unfilled. +func (e *evaluator) rankedKeys(fetch func(n int) (fetchResult, error)) ([]string, error) { + plan := eval.OverFetchPlan(e.limit, e.key.collapses) + for i, n := range plan { + res, err := fetch(n) + if err != nil { + return nil, err + } + deduped, sourceIndices := eval.DedupeRanked(res.keys) + filled := len(deduped) >= e.limit + // The engine came back short. Why it came back short decides both + // whether to retry and what to report, and the hit count alone cannot + // tell the two apart: + // + // - not saturated: it gave everything it had. The corpus is + // exhausted, a deeper fetch returns the same list, and a short + // ranking is the honest answer — not a shortfall. + // - saturated: it filled its own candidate pool and stopped. More + // matching messages exist, but no value of n reaches them, + // because the ceiling is k_per_signal, not the page size. Retrying + // deeper would only burn queries, so stop — and say so, because + // scoring this as an exhausted corpus reports a shallow ranking + // as if it were the whole of what retrieval could find. + short := res.raw < n + exhausted := short && !res.saturated + poolCapped := short && res.saturated + if filled || exhausted || poolCapped || i == len(plan)-1 { + e.diag.UnhydratedHits += res.dropped + switch { + case filled || exhausted: + // Nothing to report: the depth was met, or there was + // genuinely nothing more to retrieve. + case poolCapped: + e.diag.PoolShortfalls++ + default: + e.diag.DepthShortfalls++ + } + keys := eval.TruncateKeys(deduped, e.limit) + if e.captureHits { + e.lastHits = make(map[string]evalHit, len(keys)) + for i, key := range keys { + if i >= len(sourceIndices) || sourceIndices[i] >= len(res.hits) { + continue + } + e.lastHits[key] = res.hits[sourceIndices[i]] + } + } + return keys, nil + } + } + // Unreachable: OverFetchPlan is never empty and the loop always returns on + // its final step. + return nil, nil +} + +// rankedFTS scores the production relevance-ranked FTS path. See ftsSearcher +// for why that is the Store's search and not query.Engine.Search. +func (e *evaluator) rankedFTS(q *search.Query) ([]string, error) { + return e.rankedKeys(func(n int) (fetchResult, error) { + res, _, err := e.fts.SearchMessagesQueryContext(e.ctx, q, 0, n) + if err != nil { + return fetchResult{}, err + } + keys := make([]string, 0, len(res)) + hits := make([]evalHit, 0, len(res)) + for _, m := range res { + hit := hitFromAPIMessage(m) + keys = append(keys, e.key.extract(hit)) + hits = append(hits, hit) + } + // The store path pages a single ranked list, so a short page means + // the corpus ran out — there is no candidate pool to saturate. + return fetchResult{keys: keys, hits: hits, raw: len(res)}, nil + }) +} + +func (e *evaluator) rankedVector(mode, qstr string, q *search.Query) ([]string, error) { + // Both modes embed the free text, so a filter-only topic (`from:alice`) + // has nothing to embed and hybrid.Engine.Search would return a bare + // "empty query". Detect it here and hand runEval a recognisable error so + // it can skip this one cell instead of aborting the run. + if len(q.TextTerms) == 0 { + return nil, fmt.Errorf("%w: %q parsed to filters only", errNoFreeText, qstr) + } + // Use the engine method rather than the package function: it supplies the + // dialect's placeholder rebind, which the package function now requires. + filter, err := e.heng.BuildFilter(e.ctx, q) + if err != nil { + return nil, fmt.Errorf("build filter: %w", err) + } + subjectTerms := make([]string, 0, len(q.TextTerms)) + for _, t := range q.TextTerms { + subjectTerms = append(subjectTerms, strings.ToLower(t)) + } + freeText := strings.Join(q.TextTerms, " ") + + return e.rankedKeys(func(n int) (fetchResult, error) { + hits, meta, err := e.heng.Search(e.ctx, hybrid.SearchRequest{ + Mode: hybrid.Mode(mode), + FreeText: freeText, + Filter: filter, + Limit: n, + SubjectTerms: subjectTerms, + }) + if err != nil { + return fetchResult{}, err + } + if len(hits) == 0 { + return fetchResult{saturated: meta.PoolSaturated}, nil + } + ids := make([]int64, len(hits)) + for i, h := range hits { + ids[i] = h.MessageID + } + summaries, err := e.qeng.GetMessageSummariesByIDs(e.ctx, ids) + if err != nil { + return fetchResult{}, fmt.Errorf("map message ids: %w", err) + } + byID := make(map[int64]query.MessageSummary, len(summaries)) + for _, m := range summaries { + byID[m.ID] = m + } + // Preserve the engine's ranking order. A hit that cannot be hydrated + // is dropped — but counted, because a vector index pointing at rows + // the archive no longer has is exactly the staleness this command + // exists to surface. + out := fetchResult{ + keys: make([]string, 0, len(hits)), + hits: make([]evalHit, 0, len(hits)), + raw: len(hits), + // Carry the engine's own account of why it stopped. Without + // it, a fused query that ran out of candidate pool is + // indistinguishable from one that ran out of corpus, and + // rankedKeys would report a pool-capped ranking as complete. + saturated: meta.PoolSaturated, + } + for _, h := range hits { + m, ok := byID[h.MessageID] + if !ok { + out.dropped++ + continue + } + hit := hitFromSummary(m) + out.keys = append(out.keys, e.key.extract(hit)) + out.hits = append(out.hits, hit) + } + return out, nil + }) +} + +// ranked runs one topic through one mode. The topic is parsed once by the +// caller and handed in already validated: re-parsing per mode would let the +// same malformed filter be dropped silently three times over. +func (e *evaluator) ranked(mode, qstr string, q *search.Query) ([]string, error) { + switch mode { + case "fts": + return e.rankedFTS(q) + case "vector", "hybrid": + return e.rankedVector(mode, qstr, q) + default: + return nil, fmt.Errorf("unknown mode %q (want fts|vector|hybrid)", mode) + } +} + +// parseTopic turns one topic's query string into a validated search.Query. +// +// search.Parse never fails outright: an operator it recognises but cannot +// read — `before:invalid`, `larger:5X` — is recorded on the query and the +// filter is simply dropped, leaving a *wider* query behind. That is a +// reasonable default for an interactive search box, where the user sees the +// results and can correct the typo, but it is silent corruption for a +// benchmark: the topic still scores, against a question nobody asked. The +// production front doors (the CLI search command, /api/v1/search, +// /cli/search) all reject such a query via Query.Err(); this one skips the +// topic and says why, so one malformed line cannot quietly move a run's +// headline numbers. +// +// Parsing cleanly is not enough, though: a topic can be non-empty text and +// still parse to no search criteria at all. `subject:""` is the plain case — +// the parser drops an empty operator value rather than building a `LIKE '%%'` +// that matches everything — and the widest possible query is what is left +// behind. That is the same corruption one step further along, and it is worse, +// because the fts path answers an empty query by listing the whole live corpus +// in its default order: the topic scores whatever the archive's date +// distribution happens to give it. Production rejects the identical query +// (cmd/search.go and the /cli/search handler both test Query.IsEmpty), so it +// is skipped and reported here too. +func parseTopic(t eval.Topic, diag *runDiagnostics) (*search.Query, bool) { + q := search.Parse(t.Query) + if err := q.Err(); err != nil { + diag.skipTopic(t.ID, fmt.Sprintf("query %q did not parse: %v — scoring it would have "+ + "silently evaluated the broader query left after the bad filter was dropped", t.Query, err)) + return nil, false + } + if q.IsEmpty() { + diag.skipTopic(t.ID, fmt.Sprintf("query %q parsed to no search criteria at all — scoring it "+ + "would have ranked the whole live corpus in its default order rather than a retrieval "+ + "of this topic; production search rejects the same query as empty", t.Query)) + return nil, false + } + return q, true +} + +func runEval(cmd *cobra.Command, args []string) error { + return runEvalWithRerankerFactory(cmd, args, func(shape, key string) (evalReranker, error) { + return rerank.NewJev(shape, key) + }) +} + +func runEvalWithRerankerFactory(cmd *cobra.Command, _ []string, makeReranker evalRerankerFactory) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + registry := newDocKeyRegistry() + keySpec, ok := registry[evalDocKey] + if !ok { + return usageErr(cmd, fmt.Errorf("invalid --doc-key %q (want %s)", evalDocKey, docKeyNames(registry))) + } + // A non-positive depth is not a "use the default" signal: fts would fall + // back to an internal 100 while the vector backend would return nothing + // for k=0, so the same flag would mean two different things. Reject it. + if evalLimit <= 0 { + return usageErr(cmd, fmt.Errorf("--limit must be a positive integer, got %d", evalLimit)) + } + modes, needVec, err := parseEvalModes(evalModes) + if err != nil { + return usageErr(cmd, err) + } + rerankOptions, err := readEvalRerankOptions(cmd) + if err != nil { + return usageErr(cmd, err) + } + cutoffs := eval.CutoffsForDepth(evalLimit) + ctx := cmd.Context() + if len(rerankOptions.Shapes) > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, typesafeRunTimeout) + defer cancel() + } + + diag := &runDiagnostics{} + qrels, qrelsStats, err := eval.LoadQrels(evalQrels) + if err != nil { + return err + } + diag.QrelsLoad = qrelsStats + topics, topicsStats, err := eval.LoadTopics(evalTopics) + if err != nil { + return err + } + diag.TopicsLoad = topicsStats + + // A file in a near-miss format parses to an empty-but-valid result. Say so + // in terms of the format, because the downstream symptom ("no topics had + // judgments") reads like an id mismatch and sends people looking in the + // wrong place. + if qrelsStats.Parsed == 0 { + return fmt.Errorf("no judgments parsed from %s (%s); expected whitespace-separated "+ + "\" \" — a three-column file without the iteration column is the usual cause", + evalQrels, qrelsStats) + } + if len(topics) == 0 { + return fmt.Errorf("no topics loaded from %s (%s); expected tab-separated "+ + "\"\\t\" — spaces where tabs are expected is the usual cause", + evalTopics, topicsStats) + } + if len(rerankOptions.Shapes) > 0 { + judgedTopics := 0 + for _, topic := range topics { + if qrels.HasJudgments(topic.ID) { + judgedTopics++ + } + } + if err := validateJevRequestEstimate(judgedTopics, len(modes), rerankOptions.Shapes, + rerankOptions.Top, rerankOptions.MaxRequests); err != nil { + return usageErr(cmd, err) + } + } + // Warn on stderr so --json output stays machine-readable. + for _, l := range []struct { + kind string + stats eval.LoadStats + }{{"qrels", qrelsStats}, {"topics", topicsStats}} { + if l.stats.Suspect() { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "warning: %s %s parsed oddly (%s); check the file format\n", + l.kind, l.stats.Path, l.stats) + } + } + + // Store + query engine: serves FTS search and the rowid -> source-id + // mapping for vector/hybrid hits. Opening it also runs the schema + // migrations the vector backend relies on, and its handle is the one every + // DB read in this command goes through. + s, err := store.Open(cfg.DatabaseDSN()) + if err != nil { + return fmt.Errorf("open database: %w", err) + } + defer func() { _ = s.Close() }() + // Context-aware forms: this command's Cobra context is already + // cancellable on Ctrl-C, and a long migration or schema init on a large + // archive must actually stop when the user asks it to, not run to + // completion on a background context that can't hear the cancellation. + if err := s.InitSchemaContext(ctx); err != nil { + return fmt.Errorf("init schema: %w", err) + } + if err := runStartupMigrationsContext(ctx, s, state); err != nil { + return fmt.Errorf("startup migrations: %w", err) + } + if err := requireFTS5ForModes(modes, s.FTS5Available()); err != nil { + return err + } + // Every score this run produces rests on one archive-wide fact about the + // chosen key: that its ids name one document each. Establish it before the + // vector path is opened and before the first topic is scored, so a run that + // cannot be trusted stops instead of printing a number. + if err := requireDisjointSourceIDs(ctx, s.DB(), evalDocKey, keySpec); err != nil { + return err + } + + ev := &evaluator{ + ctx: ctx, + // The store serves --modes fts through the same relevance-ranked + // path /api/v1/search?mode=fts uses; the query engine serves the + // rowid -> source-id hydration the vector/hybrid path needs. + fts: s, + qeng: query.NewEngine(s.DB(), s.IsPostgreSQL()), + key: keySpec, + limit: evalLimit, + diag: diag, + captureHits: len(rerankOptions.Shapes) > 0, + } + + if needVec { + cleanup, err := ev.attachVector(ctx, s) + if err != nil { + return err + } + defer cleanup() + } + var rerankReport *evalRerankReport + if len(rerankOptions.Shapes) > 0 { + rerankReport = newEvalRerankReport(rerankOptions) + rerankReport.scorers = make(map[string]evalReranker, len(rerankOptions.Shapes)) + for _, shape := range rerankOptions.Shapes { + scorer, createErr := makeReranker(shape, rerankOptions.APIKey) + if createErr != nil { + return createErr + } + rerankReport.scorers[shape] = scorer + } + } + + // Record corpus size regardless of mode: recall numbers are unreadable + // without knowing how big the haystack was. + ev.prov.QrelsPath = evalQrels + ev.prov.TopicsPath = evalTopics + ev.collectCorpusStats(s.DB()) + + aggs := make(map[string]*eval.Aggregate, len(modes)) + lats := make(map[string]*eval.LatencyTracker, len(modes)) + for _, m := range modes { + aggs[m] = &eval.Aggregate{} + lats[m] = &eval.LatencyTracker{} + } + // Per-category aggregates (mode -> category), populated only for topics + // that carry a category label. Whether a question is answerable from one + // message or needs a whole thread decides which retrieval levers a run + // can even see, so when the topics file says which is which, report the + // split rather than averaging it away. + catAggs := make(map[string]map[string]*eval.Aggregate, len(modes)) + catCounts := map[string]int{} + scored := 0 + var runErr error + for _, t := range topics { + if !qrels.HasJudgments(t.ID) { + // This qrels file says nothing at all about the topic, so there is + // nothing to score it against. Record it: one judged topic is + // enough to produce a headline number, and a reader has to be able + // to see how much of the topics file that number covers. + diag.unjudged(t.ID) + continue + } + // May legitimately be empty: a topic judged but with every document + // graded non-relevant scores a real zero and belongs in the macro + // average. Dropping it would quietly raise every reported mean — see + // eval.Qrels.HasJudgments. + rel := qrels.RelevantSet(t.ID) + // Parse once, before any mode runs it: a malformed filter is a + // property of the topic, not of the mode, and must not be silently + // widened into a different question three times over. + q, ok := parseTopic(t, diag) + if !ok { + continue + } + anyMode := false + for _, m := range modes { + start := time.Now() + ev.lastHits = nil + ranked, err := ev.ranked(m, t.Query, q) + elapsed := time.Since(start) + if err != nil { + if errors.Is(err, errNoFreeText) { + // One mode cannot answer this topic. Record it and carry + // on: aborting here would throw away every score already + // computed, for every mode and every earlier topic. + diag.skip(t.ID, m, "no free-text terms to embed (filter-only topic)") + continue + } + return fmt.Errorf("topic %s, mode %s: %w", t.ID, m, err) + } + lats[m].Add(elapsed) + score := eval.Evaluate(ranked, rel, cutoffs) + aggs[m].Add(score) + if t.Category != "" { + if catAggs[m] == nil { + catAggs[m] = map[string]*eval.Aggregate{} + } + if catAggs[m][t.Category] == nil { + catAggs[m][t.Category] = &eval.Aggregate{} + } + catAggs[m][t.Category].Add(score) + } + anyMode = true + if rerankReport != nil && runErr == nil { + runErr = rerankReport.scoreRanking(ctx, s, ev.lastHits, m, t, ranked, rel, cutoffs, elapsed) + } + } + if !anyMode { + continue // no mode could score this topic + } + if t.Category != "" { + catCounts[t.Category]++ + } + scored++ + } + if rerankReport != nil { + rerankReport.reconcile(modes, aggs) + } + diag.scored = scored + if scored == 0 { + // Distinguish the two ways a run can end up with nothing: no topic + // matched a judgment, or every topic was skipped by every mode. + if len(diag.SkippedCells) > 0 { + return fmt.Errorf("no topic could be scored by any of the requested modes: %s", + strings.Join(diag.SkippedCells, "; ")) + } + return fmt.Errorf("none of the %d topics had relevance judgments in %s "+ + "(qrels: %s; topics: %s); check that the qids in both files refer to the same queries", + len(topics), evalQrels, qrelsStats, topicsStats) + } + + report := evalReport{ + modes: modes, aggs: aggs, lats: lats, catAggs: catAggs, catCounts: catCounts, + prov: ev.prov, topics: scored, cutoffs: cutoffs, diag: diag, rerank: rerankReport, + } + if evalJSON { + if reportErr := report.json(cmd.OutOrStdout()); reportErr != nil { + return reportErr + } + } else { + if reportErr := report.table(cmd.OutOrStdout()); reportErr != nil { + return reportErr + } + } + return runErr +} + +// parseEvalModes splits and validates the --modes flag, keeping each mode once +// in the order it was first named. +// +// A repeated mode is deduplicated rather than rejected. The scoring loop +// evaluates and aggregates the list entry by entry into a per-mode Aggregate, +// so `--modes fts,fts` would run every topic through fts twice, add each score +// to the same aggregate twice — doubling that mode's topic count while leaving +// its means unchanged — and double the latency work and its sample. Rejecting +// it, as a repeated qid in the topics file is rejected, would be the wrong +// shape here: two topic rows sharing a qid carry different query text, so the +// file is genuinely ambiguous and picking one silently answers a question +// nobody asked, whereas `fts,fts` has exactly one possible reading. There is +// nothing to disambiguate, so it is simply honoured once. +// +// Validation still runs per entry, before the duplicate is dropped, so a +// repeated invalid mode is still an error. Order is preserved because it is the +// order the report's rows come out in, and that belongs to the user. +func parseEvalModes(spec string) (modes []string, needVec bool, err error) { + seen := make(map[string]bool, 3) + for m := range strings.SplitSeq(spec, ",") { + m = strings.TrimSpace(m) + if m == "" { + continue + } + switch m { + case "fts": + case "vector", "hybrid": + needVec = true + default: + return nil, false, fmt.Errorf("invalid mode %q in --modes (want fts|vector|hybrid)", m) + } + if seen[m] { + continue + } + seen[m] = true + modes = append(modes, m) + } + if len(modes) == 0 { + return nil, false, errors.New("--modes is empty") + } + return modes, needVec, nil +} + +// requireFTS5ForModes stops the run before its first topic when --modes fts +// is requested against an archive without FTS5, the same "a run that cannot +// be trusted stops instead of printing a number" precondition +// requireDisjointSourceIDs establishes for --doc-key. +// +// --modes fts scores through Store.SearchMessagesQueryContext (see +// rankedFTS) expecting its production BM25 relevance ranking. A binary +// built with the sqlite_vec tag but not fts5 — or one whose FTS5 shadow +// tables failed to initialize — has that same call silently fall back to a +// LIKE scan ordered by recency instead of erroring, so the run would score a +// date sort while the report still labels the mode "fts" and claims BM25 +// ranking. --modes hybrid needs no equivalent guard: its BM25 leg runs a +// direct messages_fts MATCH inside the vector backend's fused query, which +// errors outright ("no such table: messages_fts") rather than degrading +// silently when FTS5 is unavailable. +// +// This only covers the startup state: searchMessagesQueryNoFTS documents a +// second, narrower fallback inside Store itself, forcing the same LIKE +// branch mid-run if an FTS query that started fine ever errors later +// (shadow-table corruption, an extension fault). Catching that would mean +// either threading a "did this call degrade" signal back through +// Store.SearchMessagesQueryContext — a production API several daemon +// handlers share, not an eval-only seam — or re-probing FTS5Available() +// once per topic, which cannot detect a failure the query itself hasn't hit +// yet either. Both are more machinery than a rare, already-defensive +// mid-run fallback warrants; this guard catches the case that is actually +// common (a build or archive that never had FTS5 to begin with). +func requireFTS5ForModes(modes []string, fts5Available bool) error { + if fts5Available || !slices.Contains(modes, string(hybrid.ModeFTS)) { + return nil + } + return errors.New("--modes fts needs FTS5, but this archive's FTS5 index is unavailable " + + "(binary built without the fts5 tag, or the shadow tables failed to initialize); " + + "rebuild with -tags \"fts5 sqlite_vec\" or drop fts from --modes") +} + +// attachVector wires the sqlite-vec backend and hybrid engine onto the +// evaluator (mirroring the search command's vector path) and returns a +// cleanup closure that closes the resources it opened. +// +// It reuses the caller's store handle rather than opening its own: that handle +// already carries the DSN parameters store.Open applies (busy_timeout, WAL, +// the registered driver's unicode_lower hook), and routing every DB operation +// through the Store is this repo's rule. +func (e *evaluator) attachVector(ctx context.Context, mainStore *store.Store) (func(), error) { + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil { + return nil, errors.New("configuration is unavailable") + } + cfg := state.cfg + if !cfg.Vector.Enabled { + return nil, errors.New("vector/hybrid modes need [vector].enabled = true in config") + } + mainPath := cfg.DatabaseDSN() + if store.IsPostgresURL(mainPath) { + // This command's vector path is the sqlite-vec one; a PG archive + // stores its embeddings in pgvector, alongside the messages. Fail + // clearly rather than pointing a sqlite-vec backend at a PG handle. + return nil, errors.New("vector/hybrid eval currently supports SQLite archives only; " + + "the configured database is PostgreSQL — run with --modes fts") + } + + // Resolve [vector.embed.scope] accounts to source IDs before deriving the + // build scope or the generation fingerprint, exactly as the serve/embed + // paths do. The fingerprint folds in the scope, so an unresolved config + // would compute a different one and every query would fail as "index + // stale" on any archive that scopes embedding by account. + vecCfg, err := resolvedVectorConfig(mainStore, cfg.Vector) + if err != nil { + return nil, fmt.Errorf("vector embed scope: %w", err) + } + // Validate the resolved config with the same check serve runs before it + // opens anything. It names the offending key and value — including an + // api_format this binary has no client for, which must fail here rather + // than fall back to a client that talks a different protocol to the + // endpoint that built the index. + if err := vecCfg.Validate(); err != nil { + return nil, fmt.Errorf("vector/hybrid modes need a valid [vector] config: %w", err) + } + + // Select the query client by api_format, exactly as the serve path does, + // and before anything is opened. A run scored with the OpenAI-compatible + // client against a voyage-contextual index would measure a protocol + // mismatch, not retrieval quality. Every eval call is query-time, and each + // client's EmbedQuery carries its own query role (Voyage sends + // input_type=query to /contextualizedembeddings), so no document-side + // wiring is needed here. + credentialSnapshot, err := providercredentials.Read(cfg.TokensDir()) + if err != nil { + return nil, fmt.Errorf("load provider credentials: %w", err) + } + embeddingAPIKey, err := resolveProviderCredentialFromSnapshot( + credentialSnapshot, providercredentials.VectorEmbeddingsID, + vecCfg.Embeddings.Endpoint, vecCfg.Embeddings.APIKeyEnv, + ) + if err != nil { + return nil, fmt.Errorf("resolve text embedding credential: %w", err) + } + embedClient, err := newQueryEmbeddingClient(vecCfg, embeddingAPIKey) + if err != nil { + return nil, err + } + mainDB := mainStore.DB() + + vecDBPath := vecCfg.DBPath + if vecDBPath == "" { + vecDBPath = filepath.Join(cfg.Data.DataDir, "vectors.db") + } + if err := sqlitevec.RegisterExtension(); err != nil { + return nil, fmt.Errorf("register sqlite-vec: %w", err) + } + backend, err := sqlitevec.Open(ctx, sqlitevec.Options{ + Path: vecDBPath, + MainPath: mainPath, + Dimension: vecCfg.Embeddings.Dimension, + MainDB: mainDB, + BuildScope: vecCfg.Embed.Scope.BuildScope(), + ANNOversample: vecCfg.Search.ANNOversample, + ANNNProbe: vecCfg.Search.ANNNProbe, + AcceleratorMode: vecCfg.Search.SQLiteAccelerator, + }) + if err != nil { + return nil, fmt.Errorf("open vectors.db: %w", err) + } + // Keep the resolved generation: it is the one the hybrid engine will + // search, and therefore the only one whose vector count describes this + // run's index. + active, err := vector.ResolveActiveForFingerprint(ctx, backend, vecCfg.GenerationFingerprint()) + if err != nil { + _ = backend.Close() + return nil, fmt.Errorf("resolve active generation: %w", err) + } + + e.heng = hybrid.NewEngine(backend, mainDB, embedClient, hybrid.Config{ + ExpectedFingerprint: vecCfg.GenerationFingerprint(), + RRFK: vecCfg.Search.RRFK, + KPerSignal: vecCfg.Search.KPerSignal, + SubjectBoost: vecCfg.Search.SubjectBoost, + // Without this the engine's index-scope check short-circuits to nil, + // so an out-of-scope filter would run against an index holding no + // vectors for that scope and its near-zero hit count would be scored + // as genuinely poor retrieval instead of erroring. + BuildScope: vecCfg.Embed.Scope.BuildScope(), + }) + e.collectVectorStats(mainDB, vecCfg, backend, vecDBPath, active.ID) + + return func() { _ = backend.Close() }, nil +} + +// collectCorpusStats records how big the searched archive is. It reads the +// store's own handle so it also works for --modes fts, which never opens the +// vector path. Failures are non-fatal: missing provenance should degrade the +// report, never abort a run. +// +// "How big" means how big the haystack retrieval actually searched, not how +// many rows the tables hold. A long-lived archive accumulates dedup-hidden +// duplicates and messages deleted from their source account, and no search +// this command runs returns either: the fts path resolves the default active +// deletion scope to store.LiveMessagesWhere, and the vector path drops +// source-deleted hits after the fact. Counting them would overstate the +// haystack and make recall look harder-won than it was. The predicate is +// borrowed from the store rather than restated here so the two cannot drift. +// +// Conversations are derived from those same live messages for the same +// reason: an emptied conversation row is not a thread retrieval can return, +// and with --doc-key=conversation the thread count is the denominator a +// reader will reach for. +func (e *evaluator) collectCorpusStats(db *sql.DB) { + if db == nil { + return + } + live := store.LiveMessagesWhere("", true) + _ = db.QueryRowContext(e.ctx, + "SELECT COUNT(*) FROM messages WHERE "+live).Scan(&e.prov.Messages) + _ = db.QueryRowContext(e.ctx, + "SELECT COUNT(DISTINCT conversation_id) FROM messages WHERE "+live).Scan(&e.prov.Conversations) +} + +// collidingDocKeyIDs returns doc ids that occur under more than one source in +// the live population, at most limit of them, in id order for stable output. +// +// It counts distinct source_id over the same live messages every search in this +// command draws from, so an id whose only other holder is dedup-hidden or +// deleted from its source is correctly not a collision: neither copy can be +// retrieved, so neither can be scored. +func collidingDocKeyIDs(ctx context.Context, db *sql.DB, col evalIDColumn, limit int) ([]string, error) { + // The id may be NULL (no id assigned) or empty; eval.DedupeKeys drops both + // from a ranking, so neither can collide with anything and both are + // excluded here for the same reason. + expr := col.expr() + q := fmt.Sprintf(` + SELECT %s + FROM messages m + %s + WHERE %s AND %s IS NOT NULL AND %s <> '' + GROUP BY %s + HAVING COUNT(DISTINCT m.source_id) > 1 + ORDER BY %s + LIMIT %d`, + expr, col.join, store.LiveMessagesWhere("m", true), expr, expr, expr, expr, limit) + + rows, err := db.QueryContext(ctx, q) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + var ids []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, err + } + ids = append(ids, id) + } + return ids, rows.Err() +} + +// requireDisjointSourceIDs refuses a run whose archive cannot give the chosen +// --doc-key an unambiguous doc-id space. +// +// A qrels file is flat. " " has nowhere to record +// which connected account belongs to, and neither a TREC-derived +// collection nor judgments written by hand against a personal archive carry +// one. But both ids this command can key on are assigned by the *source* — +// source_message_id by the provider or the sending mail system, +// source_conversation_id by the provider's threading — and are unique only +// within it. msgvault is a multi-source archiver, so one archive routinely +// holds several accounts, and two of them can issue the same id for unrelated +// documents (two chat accounts each numbering their first conversation "1") or +// for related ones (the same mail delivered to two mailboxes). Either way the +// eval folds two documents into one key: a hit from an unjudged account +// inherits a judged account's relevance, or two genuinely distinct documents +// collapse and the ranking quietly loses a rank. Both move the score, both move +// it upward, and neither appears anywhere in the output — which is exactly the +// class of silent corruption this command exists to expose in other people's +// indexes. +// +// The fix is a precondition rather than a new key shape. Composing the source +// id into the key — as query.EntryKeyFacts.EntryKey does for explore entries, +// production's own answer to the same uniqueness problem — would make the key +// sound, but it would also change the shape of every doc id this command +// matches on, so every qrels file already written would stop matching. And it +// would stop matching by scoring a flat zero rather than by failing, which is +// the same silent corruption one level up. +// +// The precondition is disjointness, not single-source. An archive holding a +// Gmail account and a WhatsApp account has two sources and no overlapping ids +// at all; refusing to score it would be a wall built for a hazard that is not +// there. What has to hold is that the id space the qrels address is +// unambiguous, and "no id in it names documents in two sources" is exactly +// that. It is a property of the archive rather than of what a particular topic +// happened to retrieve, so it is established once, up front, instead of +// inferred from hits that may simply have got lucky — and it is established +// before the vector path is opened, so a run that cannot be scored does not +// first pay for an index and an embedding client. +func requireDisjointSourceIDs(ctx context.Context, db *sql.DB, docKey string, spec docKeySpec) error { + if spec.idColumn.column == "" { + // A registered key whose ids do not come from an archive column cannot + // be checked here, and passing it silently would put the collision + // straight back. Fail naming the key, so adding a doc-key forces an + // answer to the question rather than allowing it to be skipped. + return fmt.Errorf("--doc-key %q has no archive column to check for cross-source id collisions; "+ + "a doc-key whose ids come from elsewhere has to establish its own single-id-space guarantee", docKey) + } + // A single connected source cannot collide with itself, and that is the + // common archive shape, so a cheap distinct-source count (backed by + // idx_messages_source) skips the GROUP BY/HAVING scan — and its join, for + // --doc-key=conversation — entirely for the run that does not need it. + var sources int + if err := db.QueryRowContext(ctx, + "SELECT COUNT(DISTINCT source_id) FROM messages WHERE "+store.LiveMessagesWhere("", true), + ).Scan(&sources); err != nil { + return fmt.Errorf("count connected sources: %w", err) + } + if sources <= 1 { + return nil + } + + // Enough ids to make the error concrete without pasting an entire + // re-imported mailbox into a terminal. + const show = 10 + ids, err := collidingDocKeyIDs(ctx, db, spec.idColumn, show+1) + if err != nil { + return fmt.Errorf("check %s for cross-source id collisions: %w", spec.idColumn.column, err) + } + if len(ids) == 0 { + return nil + } + count := strconv.Itoa(len(ids)) + if len(ids) > show { + count = fmt.Sprintf("more than %d", show) + } + return fmt.Errorf("%s document ids in this archive (%s) belong to more than one connected source, "+ + "so --doc-key=%s cannot name a single document: %s is unique only within the source that "+ + "assigned it, while a qrels doc id records no source at all. Scoring this archive would fold "+ + "those sources' hits into one key and let an unjudged account's message inherit a judged one's "+ + "relevance. Evaluate an archive whose accounts do not share ids, or key the run on the other "+ + "--doc-key if its id space is disjoint", + count, eval.FormatIDList(ids, show), docKey, spec.idColumn.column) +} + +// collectVectorStats records the embedding model, fusion parameters and index +// size in force for this run, so a score can never be read without knowing +// what produced it. +func (e *evaluator) collectVectorStats( + mainDB *sql.DB, vecCfg vector.Config, backend *sqlitevec.Backend, vecDBPath string, activeGen vector.GenerationID, +) { + e.prov.VectorEnabled = true + e.prov.EmbeddingModel = vecCfg.Embeddings.Model + e.prov.APIFormat = string(vecCfg.Embeddings.EffectiveAPIFormat()) + e.prov.Dimension = vecCfg.Embeddings.Dimension + e.prov.Endpoint = vecCfg.Embeddings.Endpoint + e.prov.Backend = vecCfg.Backend + e.prov.Fingerprint = vecCfg.GenerationFingerprint() + e.prov.RRFK = vecCfg.Search.RRFK + e.prov.KPerSignal = vecCfg.Search.KPerSignal + e.prov.SubjectBoost = vecCfg.Search.SubjectBoost + e.prov.IndexPath = vecDBPath + // The pool ceiling is what a saturation diagnostic has to name to be + // actionable, so the diagnostics carry it too. + e.diag.kPerSignal = vecCfg.Search.KPerSignal + + if fi, err := os.Stat(vecDBPath); err == nil { + e.prov.IndexSizeBytes = fi.Size() + } + // Backend.DB() is the backend's own accessor for exactly this kind of + // read-only query, so the row count goes through it rather than opening a + // second connection to the same file. + // + // Scope the count to the generation search reads. vectors.db keeps a + // retired generation's rows — vec0 partition-key isolation means retiring + // does not delete them — and a half-finished rebuild sits in the same + // table, so COUNT(*) over the whole table describes the file on disk, not + // the index this run queried. IndexSizeBytes already reports the file; + // this number has to report the index. + if vdb := backend.DB(); vdb != nil { + _ = vdb.QueryRowContext(e.ctx, + "SELECT COUNT(*) FROM embeddings WHERE generation_id = ?", int64(activeGen)). + Scan(&e.prov.IndexedVectors) + e.collectVectorCorpusStats(mainDB, vdb, vecCfg, activeGen) + } +} + +// collectVectorCorpusStats records the live population an account-scoped +// vector generation actually searches, when [vector.embed.scope] narrows it +// below the archive-wide Messages/Conversations collectCorpusStats already +// recorded. +// +// This deliberately does NOT require messages.embed_gen = gen, unlike +// Backend.EmbeddedMessageCount (production's own coverage accessor, built for +// a different question: "is this message's CURRENT content embedded"). A +// content change resets embed_gen to mark a message as needing re-embedding, +// but Backend.Search reads vectors.db purely by generation_id — it returns a +// message's stale vector until the re-embed actually runs, embed_gen or not. +// Requiring the stamp here would undercount relative to what a run can +// actually retrieve, the same "corpus" mismatch this field exists to fix in +// the other direction. So membership is: present in vectors.db for this +// generation, live, and in scope — exactly what Search can return, nothing +// narrower. +// +// Both counts are read from one query — the embedded message ids come from +// vectors.db, same as IndexedVectors reads, intersected once against +// main.db's live, scoped population — rather than a separate call per count, +// so a transient failure can only leave both at zero together, never one +// populated and the other not. +// +// Failures degrade the report rather than the run, same policy as +// collectCorpusStats: provenance is a courtesy to the reader, not a +// precondition for scoring. +func (e *evaluator) collectVectorCorpusStats( + mainDB *sql.DB, vdb *sql.DB, vecCfg vector.Config, gen vector.GenerationID, +) { + if mainDB == nil || vdb == nil { + return + } + + rows, err := vdb.QueryContext(e.ctx, + `SELECT DISTINCT message_id FROM embeddings WHERE generation_id = ?`, int64(gen)) + if err != nil { + return + } + defer func() { _ = rows.Close() }() + var ids []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return + } + ids = append(ids, id) + } + if err := rows.Err(); err != nil || len(ids) == 0 { + return + } + + blob, err := json.Marshal(ids) + if err != nil { + return + } + // BuildScope, not the raw config fields: it lowercases/trims message + // types and drops non-positive source ids the same way the embed and + // search paths already do, so a config value like "EMAIL" matches the + // lowercase message_type rows the archive actually stores instead of + // silently matching nothing. + scope := vecCfg.Embed.Scope.BuildScope() + where := `id IN (SELECT value FROM json_each(?)) + AND ` + store.LiveMessagesWhere("", true) + args := []any{string(blob)} + if len(scope.MessageTypes) > 0 { + placeholders := make([]string, len(scope.MessageTypes)) + for i, typ := range scope.MessageTypes { + placeholders[i] = "?" + args = append(args, typ) + } + where += fmt.Sprintf(" AND message_type IN (%s)", strings.Join(placeholders, ",")) + } + if len(scope.SourceIDs) > 0 { + placeholders := make([]string, len(scope.SourceIDs)) + for i, id := range scope.SourceIDs { + placeholders[i] = "?" + args = append(args, id) + } + where += fmt.Sprintf(" AND source_id IN (%s)", strings.Join(placeholders, ",")) + } + // One row, one query: reading both counts off the same scan means a + // transient failure here can only leave both fields at their zero value + // together, never one populated and the other not — a partial success + // would print a self-contradictory line no error report explains. + _ = mainDB.QueryRowContext(e.ctx, + "SELECT COUNT(DISTINCT id), COUNT(DISTINCT conversation_id) FROM messages WHERE "+where, args...). + Scan(&e.prov.VectorMessages, &e.prov.VectorConversations) +} + +// sortedCategories returns the category labels seen in a run, sorted for +// stable output. +func sortedCategories(catCounts map[string]int) []string { + cats := make([]string, 0, len(catCounts)) + for c := range catCounts { + cats = append(cats, c) + } + sort.Strings(cats) + return cats +} + +// evalReport is everything one run produced, ready to render. +type evalReport struct { + modes []string + aggs map[string]*eval.Aggregate + lats map[string]*eval.LatencyTracker + catAggs map[string]map[string]*eval.Aggregate + catCounts map[string]int + prov eval.RunConfig + topics int + cutoffs eval.Cutoffs + diag *runDiagnostics + rerank *evalRerankReport +} + +// metricHeaders names the metric columns at the depths this run actually used, +// so a clamped cutoff can never be read as the standard one. +// +// MAP and MRR are qualified too. They take no cutoff, but the ranking handed to +// them is truncated to -n, so a relevant document below that rank is invisible +// to them exactly as it is to R@n: what the run measured is MAP@n and MRR@n. +// Printing them bare would offer them for comparison against a run that +// retrieved deeper, which is the same mislabeling the clamped headers exist to +// prevent. If the depth is somehow unknown there is nothing to qualify them +// with, so they stay bare rather than claiming a depth of zero. +func (r evalReport) metricHeaders() (p, ndcg, recall, hit1, hit10, mapAt, mrr string) { + mapAt, mrr = "MAP", "MRR" + if r.cutoffs.Depth > 0 { + mapAt = fmt.Sprintf("MAP@%d", r.cutoffs.Depth) + mrr = fmt.Sprintf("MRR@%d", r.cutoffs.Depth) + } + return fmt.Sprintf("P@%d", r.cutoffs.P), + fmt.Sprintf("nDCG@%d", r.cutoffs.NDCG), + fmt.Sprintf("R@%d", r.cutoffs.Recall), + "Hit@1", fmt.Sprintf("Hit@%d", min(10, eval.HitDepth(r.cutoffs))), + mapAt, mrr +} + +func (r evalReport) table(w io.Writer) error { + if _, err := fmt.Fprintf(w, "Evaluated %d topics (doc-key=%s, n=%d)\n", r.topics, evalDocKey, evalLimit); err != nil { + return fmt.Errorf("write eval report: %w", err) + } + if !r.cutoffs.IsStandard() { + if _, err := fmt.Fprintf(w, "Metric depths are clamped to -n: the standard P@%d/nDCG@%d/R@%d need -n %d or more.\n", + eval.StandardCutoffs.P, eval.StandardCutoffs.NDCG, eval.StandardCutoffs.Recall, + eval.StandardCutoffs.Recall); err != nil { + return fmt.Errorf("write eval report: %w", err) + } + } + + // Provenance first: a score is not interpretable without it. + if _, err := fmt.Fprintln(w, "\nRun configuration"); err != nil { + return fmt.Errorf("write eval report: %w", err) + } + pw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0) + _, _ = fmt.Fprintf(pw, " topics\t%s\n", r.prov.TopicsPath) + _, _ = fmt.Fprintf(pw, " qrels\t%s\n", r.prov.QrelsPath) + _, _ = fmt.Fprintf(pw, " corpus\t%d live messages, %d conversations\n", + r.prov.Messages, r.prov.Conversations) + if r.prov.VectorEnabled { + if r.prov.VectorMessages != r.prov.Messages || r.prov.VectorConversations != r.prov.Conversations { + _, _ = fmt.Fprintf(pw, " vector corpus\t%d live messages, %d conversations "+ + "(embed.scope narrows this generation below the archive)\n", + r.prov.VectorMessages, r.prov.VectorConversations) + } + _, _ = fmt.Fprintf(pw, " embedding model\t%s (dim %d)\n", r.prov.EmbeddingModel, r.prov.Dimension) + _, _ = fmt.Fprintf(pw, " embedding api format\t%s\n", r.prov.APIFormat) + _, _ = fmt.Fprintf(pw, " embedding endpoint\t%s\n", r.prov.Endpoint) + _, _ = fmt.Fprintf(pw, " vector backend\t%s\n", r.prov.Backend) + _, _ = fmt.Fprintf(pw, " generation fingerprint\t%s\n", r.prov.Fingerprint) + _, _ = fmt.Fprintf(pw, " fusion\trrf_k=%d k_per_signal=%d subject_boost=%.2f\n", + r.prov.RRFK, r.prov.KPerSignal, r.prov.SubjectBoost) + _, _ = fmt.Fprintf(pw, " vector index\t%d vectors in the active generation, %s on disk (%s)\n", + r.prov.IndexedVectors, formatSize(r.prov.IndexSizeBytes), r.prov.IndexPath) + } else { + _, _ = fmt.Fprintf(pw, " vector index\t(not used; --modes fts only)\n") + } + if err := pw.Flush(); err != nil { + return fmt.Errorf("write eval report: %w", err) + } + + pCol, ndcgCol, rCol, hit1Col, hit10Col, mapCol, mrrCol := r.metricHeaders() + if _, err := fmt.Fprintln(w, ""); err != nil { + return fmt.Errorf("write eval report: %w", err) + } + tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0) + // "topics" is per mode, not per run: a mode that cannot answer some topic + // (a filter-only query has nothing to embed) scores fewer of them, and the + // means are only comparable if the denominators are visible. + header := []string{"MODE", "topics", pCol, ndcgCol, rCol, hit1Col, hit10Col, mapCol, mrrCol, "med ms", "p95 ms"} + _, _ = fmt.Fprintln(tw, strings.Join(header, "\t")) + rule := make([]string, len(header)) + for i, h := range header { + rule[i] = strings.Repeat("─", len([]rune(h))) + } + _, _ = fmt.Fprintln(tw, strings.Join(rule, "\t")) + for _, m := range r.modes { + s := r.aggs[m].Mean() + l := r.lats[m].Summary() + _, _ = fmt.Fprintf(tw, "%s\t%d\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.1f\t%.1f\n", + m, r.aggs[m].N, s.P, s.NDCG, s.Recall, s.Hit1, s.Hit10, s.MAP, s.MRR, l.MedianMS, l.P95MS) + } + if err := tw.Flush(); err != nil { + return fmt.Errorf("write eval report: %w", err) + } + + // Per-category breakdown, only when the topics file carries labels. + // Latency is tracked per mode, not per category, so those columns are + // omitted here. + if len(r.catCounts) > 0 { + if _, err := fmt.Fprintln(w, "\nBy query category"); err != nil { + return fmt.Errorf("write eval report: %w", err) + } + cw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0) + _, _ = fmt.Fprintf(cw, "MODE\tCATEGORY\ttopics\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", + pCol, ndcgCol, rCol, hit1Col, hit10Col, mapCol, mrrCol) + for _, m := range r.modes { + for _, c := range sortedCategories(r.catCounts) { + agg := r.catAggs[m][c] + if agg == nil { + continue + } + s := agg.Mean() + _, _ = fmt.Fprintf(cw, "%s\t%s\t%d\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\n", + m, c, agg.N, s.P, s.NDCG, s.Recall, s.Hit1, s.Hit10, s.MAP, s.MRR) + } + } + if err := cw.Flush(); err != nil { + return fmt.Errorf("write eval report: %w", err) + } + } + + if notes := r.diag.notes(); len(notes) > 0 { + if _, err := fmt.Fprintln(w, "\nDiagnostics"); err != nil { + return fmt.Errorf("write eval report: %w", err) + } + for _, n := range notes { + if _, err := fmt.Fprintf(w, " - %s\n", n); err != nil { + return fmt.Errorf("write eval report: %w", err) + } + } + } + if r.rerank != nil { + if err := r.rerank.table(w, r.cutoffs); err != nil { + return err + } + } + return nil +} + +func (r evalReport) json(w io.Writer) error { + pCol, ndcgCol, rCol, hit1Col, hit10Col, mapCol, mrrCol := r.metricHeaders() + metricsOf := func(a *eval.Aggregate) map[string]any { + s := a.Mean() + return map[string]any{ + pCol: s.P, ndcgCol: s.NDCG, rCol: s.Recall, hit1Col: s.Hit1, hit10Col: s.Hit10, + mapCol: s.MAP, mrrCol: s.MRR, + } + } + results := make(map[string]any, len(r.modes)) + for _, m := range r.modes { + entry := metricsOf(r.aggs[m]) + entry["topics"] = r.aggs[m].N + entry["latency"] = r.lats[m].Summary() + if len(r.catAggs[m]) > 0 { + byCat := make(map[string]any, len(r.catAggs[m])) + for c, a := range r.catAggs[m] { + cm := metricsOf(a) + cm["topics"] = a.N + byCat[c] = cm + } + entry["by_category"] = byCat + } + results[m] = entry + } + out := map[string]any{ + "topics_evaluated": r.topics, + "doc_key": evalDocKey, + "limit": evalLimit, + // One entry per metric, including the two whose depth is the retrieval + // depth rather than a cutoff of their own, so a consumer can read every + // metric's depth the same way instead of knowing which are special. + "cutoffs": map[string]int{ + "precision": r.cutoffs.P, "ndcg": r.cutoffs.NDCG, "recall": r.cutoffs.Recall, + "map": r.cutoffs.Depth, "mrr": r.cutoffs.Depth, + }, + "hit_cutoffs": map[string]int{ + "hit1": 1, "hit10": min(10, eval.HitDepth(r.cutoffs)), + }, + "modes": r.modes, + "run_config": r.prov, + "results": results, + "diagnostics": r.diag, + } + if len(r.catCounts) > 0 { + out["topic_categories"] = r.catCounts + } + if r.rerank != nil { + out["rerank_results"] = r.rerank.json(r.cutoffs) + } + return printJSONTo(w, out) +} diff --git a/cmd/msgvault/cmd/eval_fts_production_test.go b/cmd/msgvault/cmd/eval_fts_production_test.go new file mode 100644 index 000000000..6b517558f --- /dev/null +++ b/cmd/msgvault/cmd/eval_fts_production_test.go @@ -0,0 +1,264 @@ +//go:build fts5 && sqlite_vec + +package cmd + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/query" + "go.kenn.io/msgvault/internal/search" + "go.kenn.io/msgvault/internal/store" +) + +// seedRankingDivergenceArchive builds a small real archive in which BM25 +// relevance and reverse-chronological order disagree, and in which one +// matching message has been deleted from its source account. +// +// m1 2020-01-01 subject hit, short body — most relevant, oldest +// m2 2024-01-01 body hit, long body — least relevant, newest +// m3 2022-01-01 subject hit — deleted from source +// +// BM25 weights subject ten times body (see store.SQLiteDialect.FTSSearchClause) +// and normalizes by document length, so the ranking is m1, m3, m2 while the +// date ordering is exactly the reverse. Production FTS search returns the +// former, minus m3. +func seedRankingDivergenceArchive(t *testing.T) *store.Store { + t.Helper() + return seedRankingDivergenceArchiveIn(t, t.TempDir()) +} + +func seedRankingDivergenceArchiveIn(t *testing.T, dataDir string) *store.Store { + t.Helper() + dbPath := filepath.Join(dataDir, "msgvault.db") + s, err := store.Open(dbPath) + require.NoError(t, err, "open store") + t.Cleanup(func() { require.NoError(t, s.Close()) }) + require.NoError(t, s.InitSchema(), "init schema") + + // A body long enough that BM25's length normalization can tell the two + // documents apart, with the query term buried once inside it. + filler := strings.Repeat("quarterly figures and other unrelated correspondence text ", 60) + + _, err = s.DB().Exec(` +INSERT INTO sources (id, source_type, identifier) VALUES (1, 'gmail', 'me@example.com'); +INSERT INTO conversations (id, source_id, source_conversation_id, conversation_type) VALUES + (1, 1, 'thread-1', 'email_thread'), + (2, 1, 'thread-2', 'email_thread'), + (3, 1, 'thread-3', 'email_thread'); +INSERT INTO messages (id, conversation_id, source_id, source_message_id, message_type, subject, sent_at, size_estimate, deleted_from_source_at) VALUES + (1, 1, 1, '', 'email', 'Lease renewal terms', '2020-01-01T00:00:00Z', 100, NULL), + (2, 2, 1, '', 'email', 'Weekly digest', '2024-01-01T00:00:00Z', 100, NULL), + (3, 3, 1, '', 'email', 'Lease renewal notice','2022-01-01T00:00:00Z', 100, '2023-06-01T00:00:00Z'); +`) + require.NoError(t, err, "seed messages") + + for _, b := range []struct { + id int64 + body string + }{ + {1, "Signed and returned."}, + {2, filler + " renewal " + filler}, + {3, "Notice served."}, + } { + _, err = s.DB().Exec( + `INSERT INTO message_bodies (message_id, body_text) VALUES (?, ?)`, b.id, b.body) + require.NoError(t, err, "seed body %d", b.id) + } + + // Index through the production backfill rather than writing messages_fts + // by hand, so the test scores the documents production would have built. + indexed, err := s.BackfillFTS(nil) + require.NoError(t, err, "backfill FTS") + require.EqualValues(t, 3, indexed, "every message must be indexed") + return s +} + +func keysOfAPIMessages(msgs []store.APIMessage) []string { + out := make([]string, 0, len(msgs)) + for _, m := range msgs { + out = append(out, m.SourceMessageID) + } + return out +} + +func keysOfSummaries(msgs []query.MessageSummary) []string { + out := make([]string, 0, len(msgs)) + for _, m := range msgs { + out = append(out, m.SourceMessageID) + } + return out +} + +// TestRankedFTS_MatchesProductionRelevanceRanking is the regression for the +// mode=fts path having been scored against the wrong search. +// +// query.Engine.Search — what this command used to call — has no relevance +// component at all: it filters, then orders by sent_at DESC. Scoring that as a +// *ranking* measures the archive's date distribution, and reports it as +// retrieval quality. It also diverges from production on which messages are +// eligible at all: it leaves source-deleted messages in the result set, which +// no production search returns by default. +// +// The eval path must instead be the search production runs — the +// BM25-ranked store path behind /api/v1/search?mode=fts, whose messages_fts +// index and subject weighting are the same ones the hybrid engine's BM25 leg +// fuses, so fts and hybrid scores are comparable to each other. +func TestRankedFTS_MatchesProductionRelevanceRanking(t *testing.T) { + s := seedRankingDivergenceArchive(t) + ctx := t.Context() + const topic = "renewal" + + // The two searches genuinely disagree here — without that this test + // would pass no matter which one the eval called. + production, _, err := s.SearchMessagesQueryContext(ctx, search.Parse(topic), 0, 10) + require.NoError(t, err, "production store search") + productionKeys := keysOfAPIMessages(production) + require.Equal(t, []string{"", ""}, productionKeys, + "production ranks the subject hit first and never returns the source-deleted message") + + qeng := query.NewEngine(s.DB(), s.IsPostgreSQL()) + legacy, err := qeng.Search(ctx, search.Parse(topic), 10, 0) + require.NoError(t, err, "query engine search") + legacyKeys := keysOfSummaries(legacy) + require.Equal(t, + []string{"", "", ""}, legacyKeys, + "the chronological path inverts the ranking and includes the source-deleted message") + + ev, diag := newTestEvaluator(t, s, "message") + ev.limit = 10 + + ranked, err := ev.rankedFTS(evalTestQuery(t, topic)) + require.NoError(t, err) + + assert.Equal(t, productionKeys, ranked, + "the eval's fts mode must rank exactly as production search does") + assert.NotEqual(t, legacyKeys, ranked, + "and must no longer reproduce the chronological ordering") + assert.NotContains(t, ranked, "", + "a message deleted from its source is not something production retrieval returns") + assert.Empty(t, diag.notes(), "a clean archive produces a clean run") +} + +// TestRankedFTS_HonoursProductionAddressFilterSemantics pins the second +// divergence roborev flagged. The store path resolves from: as a substring +// match against the participant address (so `from:landlord` finds +// landlord@example.com); query.Engine.Search requires an exact address or an +// @domain pattern, and would have scored a flat zero for the same topic. +func TestRankedFTS_HonoursProductionAddressFilterSemantics(t *testing.T) { + s := seedRankingDivergenceArchive(t) + ctx := t.Context() + _, err := s.DB().Exec(` +INSERT INTO participants (id, email_address) VALUES (1, 'landlord@example.com'); +INSERT INTO message_recipients (message_id, participant_id, recipient_type) VALUES (1, 1, 'from'); +`) + require.NoError(t, err, "seed sender") + + const topic = "from:landlord renewal" + + production, _, err := s.SearchMessagesQueryContext(ctx, search.Parse(topic), 0, 10) + require.NoError(t, err, "production store search") + require.Equal(t, []string{""}, keysOfAPIMessages(production), + "production matches the address by substring") + + qeng := query.NewEngine(s.DB(), s.IsPostgreSQL()) + legacy, err := qeng.Search(ctx, search.Parse(topic), 10, 0) + require.NoError(t, err, "query engine search") + require.Empty(t, legacy, "the exact-match path finds nothing for the same topic") + + ev, _ := newTestEvaluator(t, s, "message") + ev.limit = 10 + + ranked, err := ev.rankedFTS(evalTestQuery(t, topic)) + require.NoError(t, err) + assert.Equal(t, []string{""}, ranked, + "the eval must score the hits production returns, not zero") +} + +// writeEvalFile writes one of the run's input files and returns its path. +func writeEvalFile(t *testing.T, dir, name, content string) string { + t.Helper() + path := filepath.Join(dir, name) + require.NoError(t, os.WriteFile(path, []byte(content), 0o600), "write %s", name) + return path +} + +// TestRunEval_EndToEnd_RanksByRelevanceAndSkipsAMalformedTopic drives the whole +// command against a real archive, so the two fixes are pinned where they +// actually have to hold: at the call sites, not just in the helpers. +// +// - q1 is scored through the production relevance-ranked FTS path. Its one +// judged message is the subject hit, which BM25 puts at rank 1 (MRR 1.0) +// and the old chronological path put at rank 3 (MRR 0.333). +// - q2 carries a malformed date filter. search.Parse drops it and leaves the +// broader query `renewal` behind, which would have scored exactly like q1 +// under a question nobody asked. It must be skipped and reported instead. +func TestRunEval_EndToEnd_RanksByRelevanceAndSkipsAMalformedTopic(t *testing.T) { + cfg := testConfigValue() + + dir := t.TempDir() + seedRankingDivergenceArchiveIn(t, dir) + + savedCfg := cfg + t.Cleanup(func() { cfg = savedCfg }) + cfg = config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx + cfg.Data.DataDir = dir + + savedFlags := [...]any{evalQrels, evalTopics, evalModes, evalDocKey, evalLimit, evalJSON} + t.Cleanup(func() { + evalQrels, _ = savedFlags[0].(string) + evalTopics, _ = savedFlags[1].(string) + evalModes, _ = savedFlags[2].(string) + evalDocKey, _ = savedFlags[3].(string) + evalLimit, _ = savedFlags[4].(int) + evalJSON, _ = savedFlags[5].(bool) + }) + evalQrels = writeEvalFile(t, dir, "qrels.txt", + "q1 0 1\nq2 0 1\n") + evalTopics = writeEvalFile(t, dir, "topics.tsv", + "q1\trenewal\nq2\tbefore:invalid renewal\n") + evalModes = "fts" + evalDocKey = "message" + evalLimit = 10 + evalJSON = true + + cmd := &cobra.Command{} + cmd.SetContext(testCtx) + cmd.SetContext(testCtx) + + done := captureStdout(t) + err := runEval(cmd, nil) + out := done() + require.NoError(t, err, "eval run") + + var report struct { + TopicsEvaluated int `json:"topics_evaluated"` + Results map[string]struct { + MRR float64 `json:"MRR@10"` + Topics int `json:"topics"` + } `json:"results"` + Diagnostics struct { + SkippedCells []string `json:"skipped_cells"` + } `json:"diagnostics"` + } + require.NoError(t, json.Unmarshal([]byte(out), &report), "parse report: %s", out) + + assert.Equal(t, 1, report.TopicsEvaluated, "only the well-formed topic is scored") + assert.Equal(t, 1, report.Results["fts"].Topics) + assert.InDelta(t, 1.0, report.Results["fts"].MRR, 1e-9, + "the relevance-ranked path puts the subject hit first; the chronological one scored 0.333 here") + + require.Len(t, report.Diagnostics.SkippedCells, 1, "the malformed topic must be reported, not dropped") + assert.Contains(t, report.Diagnostics.SkippedCells[0], "topic q2") + assert.Contains(t, report.Diagnostics.SkippedCells[0], "before:invalid renewal") +} diff --git a/cmd/msgvault/cmd/eval_key_test.go b/cmd/msgvault/cmd/eval_key_test.go new file mode 100644 index 000000000..cc8d35f6a --- /dev/null +++ b/cmd/msgvault/cmd/eval_key_test.go @@ -0,0 +1,91 @@ +//go:build sqlite_vec + +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/msgvault/internal/query" + "go.kenn.io/msgvault/internal/store" +) + +// TestDocKeyFuncs pins the doc-key registry: the scoring core only ever sees +// the opaque ids these extractors return, so each --doc-key value must map to +// the field its qrels actually reference. A future judged unit (for example a +// reconstructed-thread id resolved through an external mapping) is added as +// one more entry here and must not require touching this contract. +func TestDocKeyFuncs(t *testing.T) { + m := evalHit{ + MessageID: 7, + SourceMessageID: "", + SourceConversationID: "thread-42", + } + registry := newDocKeyRegistry() + + msgKey, ok := registry["message"] + require.True(t, ok) + assert.Equal(t, "", msgKey.extract(m)) + + convKey, ok := registry["conversation"] + require.True(t, ok) + assert.Equal(t, "thread-42", convKey.extract(m)) + + _, ok = registry["thread"] + assert.False(t, ok, "thread scoring is a future extension, not yet registered") +} + +// TestDocKeySpec_Collapses pins which keys need the over-fetch: a conversation +// id is shared by every message in a thread, a source message id is not. +func TestDocKeySpec_Collapses(t *testing.T) { + registry := newDocKeyRegistry() + assert.False(t, registry["message"].collapses) + assert.True(t, registry["conversation"].collapses) +} + +// TestNewDocKeyRegistry_IsPerRun backs the extensibility claim in the +// registry's doc comment: it is built by a call, not fixed at program init, so +// an entry closing over state that only exists after flags are parsed (a +// loaded message-id -> thread-id mapping, say) is possible. Two calls must +// therefore produce independent maps. +func TestNewDocKeyRegistry_IsPerRun(t *testing.T) { + first := newDocKeyRegistry() + second := newDocKeyRegistry() + require.Len(t, second, len(first)) + + first["thread"] = docKeySpec{extract: func(evalHit) string { return "x" }} + _, leaked := newDocKeyRegistry()["thread"] + assert.False(t, leaked, "a run's registry must not mutate the next run's") +} + +// TestDocKeyNames keeps usage/error text in step with the registry, sorted so +// the rendering is stable. +func TestDocKeyNames(t *testing.T) { + assert.Equal(t, "conversation|message", docKeyNames(newDocKeyRegistry())) +} + +// TestEvalHitProjections_AgreeAcrossRetrievalPaths backs the claim evalHit +// exists to make: a --doc-key means the same thing whichever engine produced +// the hit. fts hits arrive as store.APIMessage and vector/hybrid hits as +// query.MessageSummary, and if those two projected differently the same +// message would score under two different ids. +func TestEvalHitProjections_AgreeAcrossRetrievalPaths(t *testing.T) { + fromFTS := hitFromAPIMessage(store.APIMessage{ + ID: 7, + SourceMessageID: "", + SourceConversationID: "thread-42", + }) + fromVector := hitFromSummary(query.MessageSummary{ + ID: 7, + SourceMessageID: "", + SourceConversationID: "thread-42", + }) + assert.Equal(t, fromFTS, fromVector, "the same message must project identically") + + for name, spec := range newDocKeyRegistry() { + assert.Equal(t, spec.extract(fromFTS), spec.extract(fromVector), + "--doc-key=%s must not depend on which engine returned the hit", name) + } +} diff --git a/cmd/msgvault/cmd/eval_labels_test.go b/cmd/msgvault/cmd/eval_labels_test.go new file mode 100644 index 000000000..81306e570 --- /dev/null +++ b/cmd/msgvault/cmd/eval_labels_test.go @@ -0,0 +1,86 @@ +//go:build fts5 && sqlite_vec + +package cmd + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestRunEval_LabelsMAPAndMRRAtTheRetrievalDepth pins the labels against the +// depth the run actually looked to. +// +// MAP and MRR take no cutoff, so they were reported bare — but the ranking they +// score is truncated to -n before they ever see it, which makes a relevant +// message below that rank invisible to them exactly as it is to recall. A run +// at -n 5 reports MAP@5 and MRR@5; calling them "MAP" and "MRR" offers them for +// comparison against a run that retrieved a hundred deep, which is the same +// mislabeling the clamped P/nDCG/R headers already exist to prevent. +func TestRunEval_LabelsMAPAndMRRAtTheRetrievalDepth(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + dir := t.TempDir() + seedRankingDivergenceArchiveIn(t, dir) + testCtx := configureEvalRun(t, dir, "q1 0 1\n", "q1\trenewal\n") + evalLimit = 5 + + var report struct { + Cutoffs map[string]int `json:"cutoffs"` + Results map[string]map[string]any `json:"results"` + } + cmd := &cobra.Command{} + cmd.SetContext(testCtx) + done := captureStdout(t) + err := runEval(cmd, nil) + out := done() + require.NoError(err, "eval run") + require.NoError(json.Unmarshal([]byte(out), &report), "parse report: %s", out) + + scored := report.Results[evalTestMode] + assert.Contains(scored, "MAP@5", "the JSON metric key names the depth; %v", scored) + assert.Contains(scored, "MRR@5") + assert.NotContains(scored, "MAP", "an unqualified key would claim an untruncated ranking") + assert.NotContains(scored, "MRR") + // Every metric's depth is readable the same way, including the two whose + // depth is the retrieval depth rather than a cutoff of their own. + assert.Equal(map[string]int{"precision": 5, "ndcg": 5, "recall": 5, "map": 5, "mrr": 5}, + report.Cutoffs) +} + +// TestEvalReport_TableLabelsMAPAndMRRAtTheRetrievalDepth is the same rule for +// the human-readable output, which builds its header row separately from the +// JSON keys. +func TestEvalReport_TableLabelsMAPAndMRRAtTheRetrievalDepth(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + dir := t.TempDir() + seedRankingDivergenceArchiveIn(t, dir) + testCtx := configureEvalRun(t, dir, "q1 0 1\n", "q1\trenewal\n") + evalLimit = 5 + evalJSON = false + + cmd := &cobra.Command{} + cmd.SetContext(testCtx) + done := captureStdout(t) + err := runEval(cmd, nil) + out := done() + require.NoError(err, "eval run") + + header := "" + for line := range strings.SplitSeq(out, "\n") { + if strings.HasPrefix(line, "MODE") { + header = line + break + } + } + require.NotEmpty(header, "no results table in:\n%s", out) + assert.Contains(header, "MAP@5") + assert.Contains(header, "MRR@5") +} diff --git a/cmd/msgvault/cmd/eval_modes_test.go b/cmd/msgvault/cmd/eval_modes_test.go new file mode 100644 index 000000000..62dc9c31f --- /dev/null +++ b/cmd/msgvault/cmd/eval_modes_test.go @@ -0,0 +1,112 @@ +//go:build sqlite_vec + +package cmd + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/msgvault/internal/vector/hybrid" +) + +// parsedEvalModes runs the flag parser and returns the accepted modes as the +// comma-separated list they came in as, so an expectation can be written as one +// string rather than as a slice of bare mode names. +func parsedEvalModes(t *testing.T, spec string) (string, bool) { + t.Helper() + modes, needVec, err := parseEvalModes(spec) + require.NoError(t, err, "parse --modes %q", spec) + return strings.Join(modes, ","), needVec +} + +// TestParseEvalModes_DeduplicatesPreservingOrder is the regression for a +// repeated mode being evaluated twice. +// +// Each entry in the list gets its own pass through the scoring loop but shares +// one Aggregate and one LatencyTracker per mode name, so a duplicate added +// every topic's score to that mode a second time — doubling the topic count the +// report prints beside the means, and doubling the queries actually run. Order +// is the order the report's rows appear in, so the first mention keeps its +// position. +func TestParseEvalModes_DeduplicatesPreservingOrder(t *testing.T) { + assert := assert.New(t) + + modes, needVec := parsedEvalModes(t, "hybrid,fts,hybrid,fts") + assert.Equal("hybrid,fts", modes, + "each mode once, at the position it was first named") + assert.True(needVec, "the surviving hybrid entry still opens the vector path") +} + +// TestParseEvalModes_DedupesASingleRepeatedMode covers the flag as a user is +// most likely to mistype it, and pins that dropping the duplicate does not also +// drop the vector requirement it carried. +func TestParseEvalModes_DedupesASingleRepeatedMode(t *testing.T) { + assert := assert.New(t) + + modes, needVec := parsedEvalModes(t, " fts , fts ") + assert.Equal("fts", modes, "surrounding whitespace does not make a second mode") + assert.False(needVec, "fts alone never opens the vector path") + + modes, needVec = parsedEvalModes(t, "vector,vector") + assert.Equal("vector", modes) + assert.True(needVec, "the surviving entry still needs the index") +} + +// TestParseEvalModes_StillRejectsARepeatedInvalidMode keeps the dedupe from +// swallowing the validation: the switch has to run on every entry, before the +// duplicate check, or `--modes fts,bogus,bogus` would quietly report on fts +// alone. +func TestParseEvalModes_StillRejectsARepeatedInvalidMode(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + modes, _, err := parseEvalModes("fts,bogus,bogus") + require.Error(err, "an unknown mode is an error however often it appears") + assert.Empty(modes) + assert.Contains(err.Error(), `"bogus"`) +} + +// TestParseEvalModes_AcceptsTheFullSetAndRejectsAnEmptyOne pins the two ends of +// the flag the dedupe must leave alone: distinct modes all survive, and a list +// naming none of them is still rejected rather than silently scoring nothing. +func TestParseEvalModes_AcceptsTheFullSetAndRejectsAnEmptyOne(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + modes, needVec := parsedEvalModes(t, "fts,vector,hybrid") + assert.Equal("fts,vector,hybrid", modes, "distinct modes all survive, in order") + assert.True(needVec) + + _, _, err := parseEvalModes(" , ,") + require.Error(err, "a list of nothing but separators names no mode") + assert.Contains(err.Error(), "--modes is empty") +} + +// TestRequireFTS5ForModes_RejectsFTSModeWithoutFTS5 is the regression for a +// run that would otherwise silently score a LIKE-and-recency fallback while +// its own report still calls the mode "fts" and implies BM25 ranking. +func TestRequireFTS5ForModes_RejectsFTSModeWithoutFTS5(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + err := requireFTS5ForModes([]string{evalTestMode}, false) + require.Error(err, "fts without FTS5 must stop the run, not silently degrade it") + assert.Contains(err.Error(), "FTS5") + assert.Contains(err.Error(), "--modes fts") +} + +// TestRequireFTS5ForModes_AllowsWhatDoesNotNeedFTS5 pins the two ways this +// guard must stay out of the way: fts5 actually being available, and a +// --modes list that never asked for fts in the first place (vector/hybrid +// alone must not be blocked by an FTS5 outage they don't depend on). +func TestRequireFTS5ForModes_AllowsWhatDoesNotNeedFTS5(t *testing.T) { + require := require.New(t) + + require.NoError(requireFTS5ForModes([]string{evalTestMode, string(hybrid.ModeVector)}, true), + "fts is fine once FTS5 is actually available") + require.NoError(requireFTS5ForModes([]string{string(hybrid.ModeVector), string(hybrid.ModeHybrid)}, false), + "neither mode here reads through Store's FTS path") +} diff --git a/cmd/msgvault/cmd/eval_provenance_test.go b/cmd/msgvault/cmd/eval_provenance_test.go new file mode 100644 index 000000000..2c2fdca4b --- /dev/null +++ b/cmd/msgvault/cmd/eval_provenance_test.go @@ -0,0 +1,201 @@ +//go:build fts5 && sqlite_vec + +package cmd + +import ( + "context" + "database/sql" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/msgvault/internal/search" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/vector" + "go.kenn.io/msgvault/internal/vector/sqlitevec" +) + +// seedLifecycleShapedArchive builds an archive with the row shapes a long-lived +// install accumulates and a fresh benchmark corpus never has: +// +// thread-1 m1 live, m2 live +// thread-2 m3 live, m4 deleted from its source account +// thread-3 m5 dedup-hidden (deleted_at set — a losing duplicate) +// thread-4 no messages left at all +// +// Three messages are retrievable, across two threads. The tables hold five and +// four. +func seedLifecycleShapedArchive(t *testing.T) *store.Store { + t.Helper() + s, err := store.Open(filepath.Join(t.TempDir(), "msgvault.db")) + require.NoError(t, err, "open store") + t.Cleanup(func() { require.NoError(t, s.Close()) }) + require.NoError(t, s.InitSchema(), "init schema") + + _, err = s.DB().Exec(` +INSERT INTO sources (id, source_type, identifier) VALUES (1, 'gmail', 'me@example.com'); +INSERT INTO conversations (id, source_id, source_conversation_id, conversation_type) VALUES + (1, 1, 'thread-1', 'email_thread'), + (2, 1, 'thread-2', 'email_thread'), + (3, 1, 'thread-3', 'email_thread'), + (4, 1, 'thread-4', 'email_thread'); +INSERT INTO messages + (id, conversation_id, source_id, source_message_id, message_type, subject, sent_at, + size_estimate, deleted_at, deleted_from_source_at) VALUES + (1, 1, 1, '', 'email', 'Lease renewal terms', '2020-01-01T00:00:00Z', 100, NULL, NULL), + (2, 1, 1, '', 'email', 'Re: Lease renewal', '2020-01-02T00:00:00Z', 100, NULL, NULL), + (3, 2, 1, '', 'email', 'Insurance certificate','2020-02-01T00:00:00Z', 100, NULL, NULL), + (4, 2, 1, '', 'email', 'Deleted upstream', '2020-02-02T00:00:00Z', 100, NULL, '2023-06-01T00:00:00Z'), + (5, 3, 1, '', 'email', 'Duplicate copy', '2020-03-01T00:00:00Z', 100, '2023-01-01T00:00:00Z', NULL); +`) + require.NoError(t, err, "seed archive") + return s +} + +// TestCollectCorpusStats_CountsOnlyTheSearchablePopulation is the regression for +// provenance that described the tables instead of the haystack. +// +// COUNT(*) over messages and conversations includes dedup-hidden duplicates, +// messages deleted from their source, and conversations with nothing left in +// them — none of which any search this command runs can return. Reporting them +// as the corpus size inflates the denominator a reader mentally divides recall +// by, and does it invisibly: on the flat TREC corpus this branch is normally +// exercised against, every count coincides. +func TestCollectCorpusStats_CountsOnlyTheSearchablePopulation(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + s := seedLifecycleShapedArchive(t) + + // What the raw tables hold, so a change to the fixture cannot quietly + // make the assertions below trivially true. + var rawMessages, rawConversations int64 + require.NoError(s.DB().QueryRow(`SELECT COUNT(*) FROM messages`).Scan(&rawMessages)) + require.NoError(s.DB().QueryRow(`SELECT COUNT(*) FROM conversations`).Scan(&rawConversations)) + require.EqualValues(5, rawMessages) + require.EqualValues(4, rawConversations) + + ev, _ := newTestEvaluator(t, s, "message") + ev.collectCorpusStats(s.DB()) + + assert.EqualValues(3, ev.prov.Messages, + "the dedup-hidden and source-deleted rows are not part of the searched corpus") + assert.EqualValues(2, ev.prov.Conversations, + "and neither is a thread with no live message left in it") +} + +// TestCollectCorpusStats_MatchesWhatSearchCanReturn ties the count to the +// retrieval path rather than to a hand-written expectation: whatever the +// production FTS search is willing to return over the whole archive is the +// population the provenance block must be describing. +func TestCollectCorpusStats_MatchesWhatSearchCanReturn(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + s := seedLifecycleShapedArchive(t) + + // An empty query with no text terms filters on nothing but the deletion + // scope, so the result set is exactly the live population. + found, _, err := s.SearchMessagesQueryContext(t.Context(), &search.Query{}, 0, 100) + require.NoError(err, "production store search") + + threads := map[int64]struct{}{} + for _, m := range found { + threads[m.ConversationID] = struct{}{} + } + + ev, _ := newTestEvaluator(t, s, "message") + ev.collectCorpusStats(s.DB()) + + assert.EqualValues(len(found), ev.prov.Messages, + "the reported corpus must be the one search draws from") + assert.EqualValues(len(threads), ev.prov.Conversations) +} + +// seedTwoGenerations writes an index that has been rebuilt once: an older +// generation carrying three vectors, retired but retained (sqlitevec keeps a +// retired generation's rows — vec0 partition-key isolation means retiring does +// not delete them), and the active generation carrying one. Returns the total +// row count and the active generation's. +func seedTwoGenerations( + t *testing.T, dataDir, mainPath string, mainDB *sql.DB, vecCfg vector.Config, +) (total, active int64) { + t.Helper() + ctx := context.Background() + require.NoError(t, sqlitevec.RegisterExtension(), "RegisterExtension") + b, err := sqlitevec.Open(ctx, sqlitevec.Options{ + Path: filepath.Join(dataDir, "vectors.db"), + MainPath: mainPath, + Dimension: vecCfg.Embeddings.Dimension, + MainDB: mainDB, + }) + require.NoError(t, err, "open vectors.db") + defer func() { require.NoError(t, b.Close(), "close vectors.db") }() + + dim := vecCfg.Embeddings.Dimension + chunk := func(msgID int64) vector.Chunk { + v := make([]float32, dim) + for i := range v { + v[i] = float32(msgID) / 10 + } + return vector.Chunk{MessageID: msgID, Vector: v, SourceCharLen: 32} + } + newGen := func(msgIDs ...int64) { + gen, err := b.CreateGeneration(ctx, + vecCfg.Embeddings.Model, dim, vecCfg.GenerationFingerprint()) + require.NoError(t, err, "CreateGeneration") + chunks := make([]vector.Chunk, 0, len(msgIDs)) + for _, id := range msgIDs { + chunks = append(chunks, chunk(id)) + } + require.NoError(t, b.Upsert(ctx, gen, chunks), "Upsert") + // force: the coverage gate is about messages.embed_gen, which this + // test never stamps; the generation lifecycle is what is under test. + require.NoError(t, b.ActivateGeneration(ctx, gen, true), "ActivateGeneration") + } + newGen(1, 2, 3) // superseded by the next activation, rows retained + newGen(1) // active + + require.NoError(t, b.DB().QueryRowContext(ctx, + `SELECT COUNT(*) FROM embeddings`).Scan(&total), "count all embeddings") + require.NoError(t, b.DB().QueryRowContext(ctx, + `SELECT COUNT(*) FROM embeddings WHERE generation_id = + (SELECT id FROM index_generations WHERE state = 'active')`).Scan(&active), + "count active embeddings") + return total, active +} + +// TestAttachVector_CountsOnlyTheActiveGenerationsVectors is the regression for +// the index-size half of the same problem. Search reads exactly one generation +// — the active one attachVector resolves — but the reported vector count was a +// COUNT(*) over the whole embeddings table, which also holds every retired +// generation's rows and any rebuild in progress. On a rebuilt archive that +// reports an index several times the size of the one the scores came from. +func TestAttachVector_CountsOnlyTheActiveGenerationsVectors(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + dataDir := t.TempDir() + s := seedRankingDivergenceArchiveIn(t, dataDir) + + c := evalVectorConfig(t, vector.APIFormatOpenAI, "test-model") + c.Data.DataDir = dataDir + c.Vector.Embeddings.Dimension = 3 + testCtx := withTestConfig(t, c) + _ = testCtx + ctx := testCtx + + total, active := seedTwoGenerations(t, dataDir, c.DatabaseDSN(), s.DB(), c.Vector) + require.EqualValues(4, total, "the retired generation's rows are retained") + require.EqualValues(1, active, "and the active generation is the smaller one") + + ev := &evaluator{ctx: ctx, diag: &runDiagnostics{}} + cleanup, err := ev.attachVector(ctx, s) + require.NoError(err, "attachVector") + defer cleanup() + + assert.Equal(active, ev.prov.IndexedVectors, + "the run must report the index it searched, not every generation on disk") + assert.Positive(ev.prov.IndexSizeBytes, + "the whole-file measure stays available separately") +} diff --git a/cmd/msgvault/cmd/eval_qrels_test.go b/cmd/msgvault/cmd/eval_qrels_test.go new file mode 100644 index 000000000..c04260006 --- /dev/null +++ b/cmd/msgvault/cmd/eval_qrels_test.go @@ -0,0 +1,125 @@ +//go:build fts5 && sqlite_vec + +package cmd + +import ( + "context" + "encoding/json" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/msgvault/internal/config" +) + +// evalTestMode is the single search mode these end-to-end runs score. It is +// named once so the flag the run is configured with and the report key its +// assertions read can never drift apart. +const evalTestMode = "fts" + +// configureEvalRun points the eval command at one scratch directory: its +// archive as the configured data dir, and the given qrels and topics content +// written into it, for a message-keyed JSON run over the fts mode alone. The +// command's package-level config and flag variables are snapshotted and put +// back when the test ends, so a test that drives runEval directly cannot leak +// its settings into whatever runs next. +func configureEvalRun(t *testing.T, dir, qrels, topics string) context.Context { + t.Helper() + cfg := config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + cfg.Data.DataDir = dir + + savedQrels, savedTopics, savedModes := evalQrels, evalTopics, evalModes + savedDocKey, savedLimit, savedJSON := evalDocKey, evalLimit, evalJSON + t.Cleanup(func() { + evalQrels, evalTopics, evalModes = savedQrels, savedTopics, savedModes + evalDocKey, evalLimit, evalJSON = savedDocKey, savedLimit, savedJSON + }) + + evalQrels = writeEvalFile(t, dir, "qrels.txt", qrels) + evalTopics = writeEvalFile(t, dir, "topics.tsv", topics) + evalModes = evalTestMode + evalDocKey = "message" + evalLimit = 10 + evalJSON = true + return testCtx +} + +// TestRunEval_ScoresATopicJudgedEntirelyNonRelevant is the regression for +// conflating "this topic was never judged" with "this topic was judged and +// nothing was relevant". +// +// Both produce an empty relevant set, and the command used to skip on exactly +// that. But an all-non-relevant topic is a real measurement — the run looked, +// and found nothing it should have found — and it can only ever score zero. +// Dropping it therefore removes a zero from every macro average and reports a +// better run than happened. TREC semantics are to score it; only a qid the +// qrels file never mentions has nothing to score against. +// +// - q1 is judged relevant on the BM25 top hit: MRR 1.0, P@10 0.1. +// - q2 is judged, every grade 0: MRR 0, P@10 0 — and must be counted. +// - q3 appears in the topics file only, and must still be skipped. +// +// Before the fix this reported one topic at MRR 1.0; the honest answer is two +// topics at 0.5. +func TestRunEval_ScoresATopicJudgedEntirelyNonRelevant(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + dir := t.TempDir() + seedRankingDivergenceArchiveIn(t, dir) + testCtx := configureEvalRun(t, dir, + "q1 0 1\n"+ + "q2 0 0\n"+ + "q2 0 0\n", + "q1\trenewal\nq2\trenewal\nq3\trenewal\n") + + cmd := &cobra.Command{} + cmd.SetContext(testCtx) + + done := captureStdout(t) + err := runEval(cmd, nil) + out := done() + require.NoError(err, "eval run") + + var report struct { + TopicsEvaluated int `json:"topics_evaluated"` + Results map[string]struct { + MRR float64 `json:"MRR@10"` + P float64 `json:"P@10"` + Topics int `json:"topics"` + } `json:"results"` + } + require.NoError(json.Unmarshal([]byte(out), &report), "parse report: %s", out) + scored := report.Results[evalTestMode] + + assert.Equal(2, report.TopicsEvaluated, + "the all-non-relevant topic is scored; only the unjudged q3 is skipped") + assert.Equal(2, scored.Topics) + assert.InDelta(0.5, scored.MRR, 1e-9, + "q2 contributes a real zero to the macro average; skipping it reported 1.0") + assert.InDelta(0.05, scored.P, 1e-9, + "same for precision: (0.1 + 0) / 2, not 0.1") +} + +// TestRunEval_FailsWhenNoTopicIsJudged pins the other side of the split: when +// the qrels file mentions none of the topics there is genuinely nothing to +// score, and the run must still fail with the id-mismatch guidance rather than +// report an empty run full of zeroes. +func TestRunEval_FailsWhenNoTopicIsJudged(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + dir := t.TempDir() + seedRankingDivergenceArchiveIn(t, dir) + testCtx := configureEvalRun(t, dir, "other-1 0 0\n", "q1\trenewal\n") + + cmd := &cobra.Command{} + cmd.SetContext(testCtx) + + err := runEval(cmd, nil) + require.Error(err, "no topic was judged, so there is nothing to report") + assert.Contains(err.Error(), "relevance judgments") +} diff --git a/cmd/msgvault/cmd/eval_rank_test.go b/cmd/msgvault/cmd/eval_rank_test.go new file mode 100644 index 000000000..7e0456e67 --- /dev/null +++ b/cmd/msgvault/cmd/eval_rank_test.go @@ -0,0 +1,463 @@ +//go:build sqlite_vec + +package cmd + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/msgvault/internal/eval" + "go.kenn.io/msgvault/internal/query" + "go.kenn.io/msgvault/internal/query/querytest" + "go.kenn.io/msgvault/internal/search" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/vector" + "go.kenn.io/msgvault/internal/vector/hybrid" +) + +// vectorTestGeneration is the active generation the fake backend serves; the +// hybrid engine refuses to search unless the fingerprint matches its config. +var vectorTestGeneration = vector.Generation{ + ID: 1, Model: "test-model", Dimension: 4, + Fingerprint: "test-model:4", State: vector.GenerationActive, +} + +// saturatingFusingBackend is a vector.FusingBackend that returns a fixed hit +// list together with a caller-chosen saturation flag, so a test can pin what +// the eval path does with the flag the real engine computes. Only the two +// methods the hybrid search path calls are implemented; the embedded interface +// panics on anything else, which is the point — a widened call is a test bug, +// not something to silently stub. +type saturatingFusingBackend struct { + vector.Backend + + generation vector.Generation + hits []vector.FusedHit + vectorHits []vector.Hit + saturated bool + fusedCalls int +} + +func (b *saturatingFusingBackend) ActiveGeneration(context.Context) (vector.Generation, error) { + return b.generation, nil +} + +func (b *saturatingFusingBackend) FusedSearch( + context.Context, vector.FusedRequest, +) ([]vector.FusedHit, vector.SearchMetadata, error) { + b.fusedCalls++ + return b.hits, vector.SearchMetadata{PoolSaturated: b.saturated}, nil +} + +func (b *saturatingFusingBackend) Search( + context.Context, vector.GenerationID, []float32, int, vector.Filter, +) ([]vector.Hit, error) { + return b.vectorHits, nil +} + +// stubEmbedder returns a fixed query vector; the fake backend never looks at +// it, but the engine insists on embedding before it will search. +type stubEmbedder struct{} + +func (stubEmbedder) EmbedQuery(context.Context, string) ([]float32, error) { + return []float32{1, 0, 0, 0}, nil +} + +// threadedCorpus builds a ranked corpus of threads*perThread messages, ordered +// thread by thread: the first perThread hits all belong to one conversation. +// That is the shape that breaks a truncate-then-collapse ranking. +func threadedCorpus(threads, perThread int) []store.APIMessage { + out := make([]store.APIMessage, 0, threads*perThread) + var id int64 + for t := range threads { + for m := range perThread { + id++ + out = append(out, store.APIMessage{ + ID: id, + SourceMessageID: fmt.Sprintf("", t, m), + SourceConversationID: fmt.Sprintf("thread-%03d", t), + }) + } + } + return out +} + +// pagingFTS serves the first n results of corpus and records the depths it was +// asked for, so a test can assert both the answer and the work done to get it. +// It stands in for *store.Store on the production relevance-ranked FTS path. +type pagingFTS struct { + corpus []store.APIMessage + depths []int +} + +func (p *pagingFTS) SearchMessagesQueryContext( + _ context.Context, _ *search.Query, _, limit int, +) ([]store.APIMessage, int64, error) { + p.depths = append(p.depths, limit) + if limit > len(p.corpus) { + limit = len(p.corpus) + } + return p.corpus[:limit], int64(len(p.corpus)), nil +} + +func newPagingFTS(corpus []store.APIMessage) *pagingFTS { + return &pagingFTS{corpus: corpus} +} + +// evalTestLimit is the retrieval depth every ranking test uses; it matches the +// command's own -n default, so the over-fetch arithmetic in the assertions is +// the arithmetic a real run does. +const evalTestLimit = 100 + +func newTestEvaluator(t *testing.T, fts ftsSearcher, docKey string) (*evaluator, *runDiagnostics) { + t.Helper() + spec, ok := newDocKeyRegistry()[docKey] + require.True(t, ok) + diag := &runDiagnostics{} + return &evaluator{ctx: t.Context(), fts: fts, key: spec, limit: evalTestLimit, diag: diag}, diag +} + +// evalTestQuery parses a topic the way runEval does, so the ranking tests +// exercise the same already-validated query object the command builds. +func evalTestQuery(t *testing.T, qstr string) *search.Query { + t.Helper() + q := search.Parse(qstr) + require.NoError(t, q.Err()) + return q +} + +// TestRankedFTS_ConversationKeyFillsTheRequestedDepth is the regression for the +// ordering bug. The corpus has 4 messages per thread, so a raw fetch of 100 +// covers only 25 threads. Collapsing after truncating returned those 25 and +// reported them as "R@100"; the ranking must instead over-fetch, collapse, and +// hand back 100 distinct threads. +func TestRankedFTS_ConversationKeyFillsTheRequestedDepth(t *testing.T) { + corpus := threadedCorpus(200, 4) + eng := newPagingFTS(corpus) + ev, diag := newTestEvaluator(t, eng, "conversation") + + // For contrast, what the old ordering produced: collapsing the first + // --limit raw hits only ever reached a quarter of the requested depth. + firstPage := make([]string, 0, 100) + for _, m := range corpus[:100] { + firstPage = append(firstPage, m.SourceConversationID) + } + require.Len(t, eval.DedupeKeys(firstPage), 25, "truncate-then-collapse caps out at 25 threads here") + + ranked, err := ev.rankedFTS(evalTestQuery(t, "lease renewal")) + require.NoError(t, err) + + require.Len(t, ranked, 100, "-n 100 with --doc-key=conversation means 100 distinct threads") + assert.Equal(t, "thread-000", ranked[0], "best rank wins") + assert.Equal(t, "thread-099", ranked[99]) + seen := map[string]struct{}{} + for _, k := range ranked { + _, dup := seen[k] + require.False(t, dup, "collapsed ranking must not repeat a thread: %s", k) + seen[k] = struct{}{} + } + assert.Equal(t, []int{400}, eng.depths, "one over-fetch was enough here") + assert.Zero(t, diag.DepthShortfalls) +} + +// TestRankedFTS_MessageKeyDoesNotOverFetch: the message doc-key is 1:1 with +// hits, so padding the query would only inflate the latency this command +// reports. +func TestRankedFTS_MessageKeyDoesNotOverFetch(t *testing.T) { + eng := newPagingFTS(threadedCorpus(200, 5)) + ev, _ := newTestEvaluator(t, eng, "message") + + ranked, err := ev.rankedFTS(evalTestQuery(t, "lease renewal")) + require.NoError(t, err) + assert.Len(t, ranked, 100) + assert.Equal(t, []int{100}, eng.depths, "no over-fetch for a 1:1 doc-key") +} + +func TestRankedFTS_CapturesMessageIdentityWithKey(t *testing.T) { + corpus := threadedCorpus(2, 2) + eng := newPagingFTS(corpus) + ev, _ := newTestEvaluator(t, eng, "message") + ev.captureHits = true + + ranked, err := ev.rankedFTS(evalTestQuery(t, "lease renewal")) + require.NoError(t, err) + require.NotEmpty(t, ranked) + hit, ok := ev.lastHits[ranked[0]] + require.True(t, ok) + assert.Equal(t, corpus[0].ID, hit.MessageID) + assert.Equal(t, corpus[0].SourceMessageID, ranked[0]) +} + +// TestRankedFTS_GrowsThePoolUntilTheDepthIsFilled: one over-fetch is not always +// enough. With 50 messages per thread, limit*4 still only covers 8 threads, so +// the pool has to grow. +func TestRankedFTS_GrowsThePoolUntilTheDepthIsFilled(t *testing.T) { + eng := newPagingFTS(threadedCorpus(200, 50)) + ev, diag := newTestEvaluator(t, eng, "conversation") + + ranked, err := ev.rankedFTS(evalTestQuery(t, "lease renewal")) + require.NoError(t, err) + assert.Len(t, ranked, 100) + assert.Equal(t, []int{400, 1600, 6400}, eng.depths, "the pool grows geometrically") + assert.Zero(t, diag.DepthShortfalls, "the depth was filled on the last attempt") +} + +// TestRankedFTS_StopsWhenTheEngineIsExhausted: a corpus smaller than the pool +// must not trigger pointless retries, and the short answer is not a shortfall +// worth warning about — there is simply nothing more to retrieve. +func TestRankedFTS_StopsWhenTheEngineIsExhausted(t *testing.T) { + eng := newPagingFTS(threadedCorpus(6, 5)) // 30 messages, 6 threads + ev, diag := newTestEvaluator(t, eng, "conversation") + + ranked, err := ev.rankedFTS(evalTestQuery(t, "lease renewal")) + require.NoError(t, err) + assert.Len(t, ranked, 6, "six threads exist; six threads come back") + assert.Equal(t, []int{400}, eng.depths, "the engine came back short, so stop") + assert.Zero(t, diag.DepthShortfalls) + assert.Zero(t, diag.PoolShortfalls, "an unsaturated short page is an exhausted corpus, not a shortfall") +} + +// TestRankedFTS_ReportsADepthShortfall: when even the largest pool cannot fill +// the depth, the metrics are computed over a shallower list than requested and +// the run has to say so rather than passing the number off as full depth. +func TestRankedFTS_ReportsADepthShortfall(t *testing.T) { + // One single thread, deeper than the biggest pool: every fetch is + // saturated, and every fetch collapses to one key. + eng := newPagingFTS(threadedCorpus(1, 20_000)) + ev, diag := newTestEvaluator(t, eng, "conversation") + + ranked, err := ev.rankedFTS(evalTestQuery(t, "lease renewal")) + require.NoError(t, err) + assert.Equal(t, []string{"thread-000"}, ranked) + assert.Equal(t, []int{400, 1600, 6400}, eng.depths, "the pool grows to the documented ceiling and stops") + assert.Equal(t, 1, diag.DepthShortfalls) + assert.Contains(t, diag.notes()[0], "could not fill") +} + +// TestRankedKeys_SaturatedShortFetchIsNotCorpusExhaustion is the regression for +// the discarded PoolSaturated flag. A fused query caps each signal at +// k_per_signal, so it can return fewer hits than asked for while the corpus +// still holds plenty more. Reading that short page as "the corpus ran out" +// reported a pool-capped ranking as if it were everything retrieval could +// find — the one reading that makes the resulting metric silently wrong. +func TestRankedKeys_SaturatedShortFetchIsNotCorpusExhaustion(t *testing.T) { + ev, diag := newTestEvaluator(t, nil, "conversation") + diag.kPerSignal = 250 + + var asked []int + ranked, err := ev.rankedKeys(func(n int) (fetchResult, error) { + asked = append(asked, n) + // 40 hits back for a request of 400, spread over 10 threads, with + // the engine reporting that its candidate pool was full. + keys := make([]string, 0, 40) + for i := range 40 { + keys = append(keys, fmt.Sprintf("thread-%03d", i%10)) + } + return fetchResult{keys: keys, raw: len(keys), saturated: true}, nil + }) + require.NoError(t, err) + + assert.Len(t, ranked, 10, "the ten reachable threads are still scored") + assert.Equal(t, []int{400}, asked, + "a bigger page cannot get past k_per_signal, so do not burn another query on it") + assert.Equal(t, 1, diag.PoolShortfalls) + assert.Zero(t, diag.DepthShortfalls, "this is a pool ceiling, not an exhausted over-fetch budget") + + notes := diag.notes() + require.Len(t, notes, 1) + assert.Contains(t, notes[0], "candidate pool") + assert.Contains(t, notes[0], "k_per_signal=250", "the note names the setting that caused it") + assert.Contains(t, notes[0], "not an exhausted corpus", + "the whole point of the flag is to keep the two apart") +} + +// TestRankedKeys_UnsaturatedShortFetchIsExhaustion pins the other half of the +// pair: the same short page, with the engine reporting it had nothing left, +// is not a shortfall at all and must stay silent. +func TestRankedKeys_UnsaturatedShortFetchIsExhaustion(t *testing.T) { + ev, diag := newTestEvaluator(t, nil, "conversation") + + ranked, err := ev.rankedKeys(func(int) (fetchResult, error) { + return fetchResult{keys: []string{"thread-000", "thread-001"}, raw: 2}, nil + }) + require.NoError(t, err) + + assert.Equal(t, []string{"thread-000", "thread-001"}, ranked) + assert.Zero(t, diag.PoolShortfalls) + assert.Zero(t, diag.DepthShortfalls) + assert.Empty(t, diag.notes(), "an exhausted corpus is an answer, not an anomaly") +} + +// TestRankedKeys_SaturatedFullDepthIsNotAShortfall: saturation only matters +// when the depth went unfilled. A run that got everything it asked for has +// nothing to warn about, however full the engine's pool was. +func TestRankedKeys_SaturatedFullDepthIsNotAShortfall(t *testing.T) { + ev, diag := newTestEvaluator(t, nil, "conversation") + + keys := make([]string, 0, evalTestLimit) + for i := range evalTestLimit { + keys = append(keys, fmt.Sprintf("thread-%03d", i)) + } + ranked, err := ev.rankedKeys(func(int) (fetchResult, error) { + return fetchResult{keys: keys, raw: len(keys), saturated: true}, nil + }) + require.NoError(t, err) + + assert.Len(t, ranked, evalTestLimit) + assert.Zero(t, diag.PoolShortfalls) + assert.Zero(t, diag.DepthShortfalls) +} + +// TestRankedVector_CarriesPoolSaturationFromTheEngine checks the wiring the +// classification above depends on: hybrid.ResultMeta.PoolSaturated has to +// survive the trip from the engine into fetchResult. It used to be dropped on +// the floor at the call site, which no amount of correct classification +// downstream could recover from. +func TestRankedVector_CarriesPoolSaturationFromTheEngine(t *testing.T) { + // One hit for a request of 400, with the backend reporting a full pool. + backend := &saturatingFusingBackend{ + generation: vectorTestGeneration, + hits: []vector.FusedHit{{MessageID: 1, RRFScore: 0.9}}, + saturated: true, + } + qeng := &querytest.MockEngine{ + GetMessageSummariesByIDsFunc: func(_ context.Context, ids []int64) ([]query.MessageSummary, error) { + out := make([]query.MessageSummary, 0, len(ids)) + for _, id := range ids { + out = append(out, query.MessageSummary{ + ID: id, + SourceMessageID: fmt.Sprintf("", id), + SourceConversationID: "thread-000", + }) + } + return out, nil + }, + } + ev, diag := newTestEvaluator(t, nil, "conversation") + ev.captureHits = true + ev.qeng = qeng + ev.heng = hybrid.NewEngine(backend, nil, stubEmbedder{}, hybrid.Config{ + ExpectedFingerprint: vectorTestGeneration.Fingerprint, + }) + + ranked, err := ev.rankedVector("hybrid", "lease renewal", evalTestQuery(t, "lease renewal")) + require.NoError(t, err) + + assert.Equal(t, []string{"thread-000"}, ranked) + hit, ok := ev.lastHits[ranked[0]] + require.True(t, ok) + assert.Equal(t, int64(1), hit.MessageID) + assert.Equal(t, 1, backend.fusedCalls, "a saturated pool must not be retried at a deeper page") + assert.Equal(t, 1, diag.PoolShortfalls, "the engine's own saturation flag has to reach the diagnostics") + assert.Zero(t, diag.DepthShortfalls) +} + +func TestRankedVector_CapturesMessageIdentityWithKey(t *testing.T) { + backend := &saturatingFusingBackend{ + generation: vectorTestGeneration, + vectorHits: []vector.Hit{{MessageID: 7, Score: 0.9, Rank: 1}}, + } + ev, _ := newTestEvaluator(t, nil, "message") + ev.captureHits = true + ev.qeng = &querytest.MockEngine{ + GetMessageSummariesByIDsFunc: func(_ context.Context, ids []int64) ([]query.MessageSummary, error) { + return []query.MessageSummary{{ + ID: ids[0], SourceMessageID: "", SourceConversationID: "thread-007", + }}, nil + }, + } + ev.heng = hybrid.NewEngine(backend, nil, stubEmbedder{}, hybrid.Config{ + ExpectedFingerprint: vectorTestGeneration.Fingerprint, + }) + + ranked, err := ev.rankedVector("vector", "lease renewal", evalTestQuery(t, "lease renewal")) + require.NoError(t, err) + require.Equal(t, []string{""}, ranked) + hit, ok := ev.lastHits[ranked[0]] + require.True(t, ok) + assert.Equal(t, int64(7), hit.MessageID) +} + +// TestRankedVector_FilterOnlyTopicIsRecoverable: a topic that parses to filters +// only has nothing to embed. It must surface as errNoFreeText — which runEval +// turns into a skipped cell — and never as an opaque failure that aborts the +// run and discards every score computed so far. +func TestRankedVector_FilterOnlyTopicIsRecoverable(t *testing.T) { + const topic = "from:alice@example.com" + parsed := evalTestQuery(t, topic) + require.Empty(t, parsed.TextTerms, "fixture assumption: this topic is filter-only") + + // heng is deliberately nil: the check must happen before the engine is + // ever touched. + ev, _ := newTestEvaluator(t, nil, "message") + ev.qeng = &querytest.MockEngine{} + + _, err := ev.rankedVector("vector", topic, parsed) + require.Error(t, err) + require.ErrorIs(t, err, errNoFreeText) + assert.Contains(t, err.Error(), topic, "the error names the offending topic") +} + +// TestParseTopic_RejectsAMalformedFilter is the regression for silently +// widened topics. search.Parse drops an operator value it cannot read and +// carries on, so `before:invalid renewal` becomes the unfiltered query +// `renewal` — a different question, scored under the original topic's id. +func TestParseTopic_RejectsAMalformedFilter(t *testing.T) { + const bad = "before:invalid renewal" + + // What the old code would have run: the date filter is gone, and only + // the bare term survives. This is the query that must NOT be scored. + widened := search.Parse(bad) + require.Error(t, widened.Err(), "fixture assumption: this topic does not parse cleanly") + require.Nil(t, widened.BeforeDate, "the malformed filter is dropped, not honoured") + require.Equal(t, []string{"renewal"}, widened.TextTerms, "leaving a strictly broader query behind") + + diag := &runDiagnostics{} + q, ok := parseTopic(eval.Topic{ID: "q7", Query: bad}, diag) + assert.False(t, ok, "a topic that does not parse must not be scored") + assert.Nil(t, q) + + notes := diag.notes() + require.Len(t, notes, 1) + assert.Contains(t, notes[0], "topic q7", "the report names the offending topic") + assert.Contains(t, notes[0], "before", "and the operator that failed") +} + +// TestParseTopic_AcceptsAWellFormedFilter: the guard rejects malformed values, +// not filters in general. A topic with a valid date filter parses through with +// the filter intact. +func TestParseTopic_AcceptsAWellFormedFilter(t *testing.T) { + diag := &runDiagnostics{} + q, ok := parseTopic(eval.Topic{ID: "q8", Query: "before:2024-01-01 renewal"}, diag) + require.True(t, ok) + require.NotNil(t, q) + assert.NotNil(t, q.BeforeDate, "a filter that parses is kept") + assert.Equal(t, []string{"renewal"}, q.TextTerms) + assert.Empty(t, diag.notes(), "a clean topic says nothing") +} + +// TestRunDiagnostics_Notes checks that every silent failure mode this run can +// hit produces a line a user will actually read. +func TestRunDiagnostics_Notes(t *testing.T) { + assert.Empty(t, (&runDiagnostics{}).notes(), "a clean run says nothing") + + d := &runDiagnostics{UnhydratedHits: 7} + require.Len(t, d.notes(), 1) + assert.Contains(t, d.notes()[0], "7 retrieved hits could not be hydrated") + + d = &runDiagnostics{} + d.skip("q3", "hybrid", "no free-text terms to embed (filter-only topic)") + require.Len(t, d.notes(), 1) + assert.Contains(t, d.notes()[0], "topic q3 / hybrid") + + // The pool note stays readable when the run never opened the vector + // path, so an unknown k_per_signal is never rendered as "k_per_signal=0". + d = &runDiagnostics{PoolShortfalls: 2} + require.Len(t, d.notes(), 1) + assert.Contains(t, d.notes()[0], "candidate pool") + assert.NotContains(t, d.notes()[0], "k_per_signal=0") +} diff --git a/cmd/msgvault/cmd/eval_rerank.go b/cmd/msgvault/cmd/eval_rerank.go new file mode 100644 index 000000000..df4f7644d --- /dev/null +++ b/cmd/msgvault/cmd/eval_rerank.go @@ -0,0 +1,503 @@ +//go:build sqlite_vec + +package cmd + +import ( + "context" + "errors" + "fmt" + "io" + "math" + "os" + "slices" + "strconv" + "strings" + "text/tabwriter" + "time" + "unicode/utf8" + + "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/eval" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/vector/embed" + "go.kenn.io/msgvault/internal/vector/rerank" +) + +const typesafeRunTimeout = 30 * time.Minute + +var ( + evalRerankJev string + evalRerankTop int + evalRerankMaxRequests int + evalRerankInputUSDPerM float64 + evalRerankOutputUSDPerM float64 +) + +type evalRerankOptions struct { + Shapes []string + Top int + MaxRequests int + InputUSDPerM float64 + OutputUSDPerM float64 + APIKey string + Preprocess embed.PreprocessConfig +} + +type evalReranker interface { + Rerank(ctx context.Context, request rerank.Request) (rerank.Result, error) +} + +type evalRerankerFactory func(string, string) (evalReranker, error) + +type evalRerankArm struct { + Agg *eval.Aggregate + Lat *eval.LatencyTracker + Requests int + InputTokens *int64 + OutputTokens *int64 + CostUSD *float64 + UsageComplete bool + Complete bool + Status string + Error string +} + +type evalRerankReport struct { + scorers map[string]evalReranker + Shapes []string + Top int + MaxRequests int + InputUSDPerM float64 + OutputUSDPerM float64 + Model string + Endpoint string + Preprocess embed.PreprocessConfig + Complete bool + Failure string + Results map[string]map[string]*evalRerankArm +} + +func newEvalRerankReport(options evalRerankOptions) *evalRerankReport { + return &evalRerankReport{ + Shapes: slices.Clone(options.Shapes), Top: options.Top, MaxRequests: options.MaxRequests, + InputUSDPerM: options.InputUSDPerM, OutputUSDPerM: options.OutputUSDPerM, + Model: rerank.JevModel, Endpoint: rerank.JevEndpoint, Preprocess: options.Preprocess, + Complete: true, Results: make(map[string]map[string]*evalRerankArm), + } +} + +func (r *evalRerankReport) arm(mode, shape string) *evalRerankArm { + if r.Results[mode] == nil { + r.Results[mode] = make(map[string]*evalRerankArm) + } + if r.Results[mode][shape] == nil { + input, output := int64(0), int64(0) + r.Results[mode][shape] = &evalRerankArm{ + Agg: &eval.Aggregate{}, Lat: &eval.LatencyTracker{}, Complete: true, Status: "complete", + InputTokens: &input, OutputTokens: &output, CostUSD: new(0.0), UsageComplete: true, + } + } + return r.Results[mode][shape] +} + +func (a *evalRerankArm) addUsage(usage rerank.Usage, inputPrice, outputPrice float64) { + a.Requests += usage.Requests + if usage.InputTokens != nil && a.InputTokens != nil { + *a.InputTokens += *usage.InputTokens + } + if usage.OutputTokens != nil && a.OutputTokens != nil { + *a.OutputTokens += *usage.OutputTokens + } + if !usage.Complete || usage.InputTokens == nil || usage.OutputTokens == nil { + a.UsageComplete = false + a.CostUSD = nil + return + } + if a.CostUSD != nil { + *a.CostUSD += float64(*usage.InputTokens)*inputPrice/1e6 + float64(*usage.OutputTokens)*outputPrice/1e6 + } +} + +func (a *evalRerankArm) addQuality(ranked []string, rel map[string]struct{}, cutoffs eval.Cutoffs, elapsed time.Duration) { + a.Agg.Add(eval.Evaluate(ranked, rel, cutoffs)) + a.Lat.Add(elapsed) +} + +func hitColumns(cutoffs eval.Cutoffs) (string, string) { + return "Hit@1", fmt.Sprintf("Hit@%d", min(10, eval.HitDepth(cutoffs))) +} + +func (r *evalRerankReport) table(w io.Writer, cutoffs eval.Cutoffs) error { + if _, err := fmt.Fprintln(w, "\nJev reranking"); err != nil { + return fmt.Errorf("write rerank report: %w", err) + } + if _, err := fmt.Fprintf(w, " shapes\t%s\n top\t%d\n request limit\t%d\n input price\t$%.6f / million tokens\n output price\t$%.6f / million tokens\n", + strings.Join(r.Shapes, ","), r.Top, r.MaxRequests, r.InputUSDPerM, r.OutputUSDPerM); err != nil { + return fmt.Errorf("write rerank report: %w", err) + } + _, hit10 := hitColumns(cutoffs) + writer := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0) + if _, err := fmt.Fprintf(writer, "MODE\tSHAPE\tstatus\tusage complete\ttopics\tHit@1\t%s\tnDCG@%d\tp95 ms\trequests\trequests/q\tinput tokens\tinput/q\toutput tokens\toutput/q\tcost/query\n", hit10, cutoffs.NDCG); err != nil { + return fmt.Errorf("write rerank report: %w", err) + } + for _, mode := range sortedRerankModes(r.Results) { + for _, shape := range r.Shapes { + arm := r.Results[mode][shape] + if arm == nil { + if _, err := fmt.Fprintf(writer, "%s\t%s\tunrun\t-\t-\t-\t-\t-\t-\t0\t-\t0\t-\t0\t-\t-\n", mode, shape); err != nil { + return fmt.Errorf("write rerank report: %w", err) + } + continue + } + if arm.Agg == nil || arm.Agg.N == 0 || !arm.Complete || arm.Status != "complete" { + topics := 0 + if arm.Agg != nil { + topics = arm.Agg.N + } + if _, err := fmt.Fprintf(writer, "%s\t%s\t%s\t%t\t%d\t-\t-\t-\t-\t%d\t-\t%s\t-\t%s\t-\t%s\n", mode, shape, arm.Status, arm.UsageComplete, topics, arm.Requests, + formatOptionalTokenCount(arm.InputTokens), formatOptionalTokenCount(arm.OutputTokens), formatCostPerQuery(arm.CostUSD, topics)); err != nil { + return fmt.Errorf("write rerank report: %w", err) + } + continue + } + s := arm.Agg.Mean() + l := arm.Lat.Summary() + if _, err := fmt.Fprintf(writer, "%s\t%s\t%s\t%t\t%d\t%.3f\t%.3f\t%.3f\t%.1f\t%d\t%.1f\t%s\t%s\t%s\t%s\t%s\n", + mode, shape, arm.Status, arm.UsageComplete, arm.Agg.N, s.Hit1, s.Hit10, s.NDCG, l.P95MS, + arm.Requests, float64(arm.Requests)/float64(arm.Agg.N), formatOptionalTokenCount(arm.InputTokens), + formatOptionalAverage(arm.InputTokens, arm.Agg.N), formatOptionalTokenCount(arm.OutputTokens), + formatOptionalAverage(arm.OutputTokens, arm.Agg.N), formatCostPerQuery(arm.CostUSD, arm.Agg.N)); err != nil { + return fmt.Errorf("write rerank report: %w", err) + } + } + } + if err := writer.Flush(); err != nil { + return fmt.Errorf("write rerank report: %w", err) + } + if r.Failure != "" { + if _, err := fmt.Fprintf(w, " status\tincomplete: %s\n", r.Failure); err != nil { + return fmt.Errorf("write rerank report: %w", err) + } + } + return nil +} + +func sortedRerankModes(results map[string]map[string]*evalRerankArm) []string { + modes := make([]string, 0, len(results)) + for mode := range results { + modes = append(modes, mode) + } + slices.Sort(modes) + return modes +} + +func formatOptionalAverage(value *int64, queries int) string { + if value == nil { + return "unknown" + } + if queries <= 0 { + return "0" + } + return fmt.Sprintf("%.1f", float64(*value)/float64(queries)) +} + +func formatOptionalTokenCount(value *int64) string { + if value == nil { + return "unknown" + } + return strconv.FormatInt(*value, 10) +} + +func formatCostPerQuery(value *float64, queries int) string { + if value == nil { + return "unknown" + } + if queries <= 0 { + return "0" + } + return fmt.Sprintf("$%.6f", *value/float64(queries)) +} + +func (r *evalRerankReport) json(cutoffs eval.Cutoffs) map[string]any { + hit1, hit10 := hitColumns(cutoffs) + metrics := func(arm *evalRerankArm) map[string]any { + out := map[string]any{ + "status": arm.Status, "requests": arm.Requests, + "usage_complete": arm.UsageComplete, + "input_tokens": arm.InputTokens, "output_tokens": arm.OutputTokens, + "cost_usd": arm.CostUSD, + } + if arm.Error != "" { + out["error"] = arm.Error + } + if arm.Agg != nil { + out["topics"] = arm.Agg.N + } + if arm.Agg == nil || !arm.Complete || arm.Status != "complete" || arm.Agg.N == 0 { + return out + } + s := arm.Agg.Mean() + out["requests_per_query"] = float64(arm.Requests) / float64(arm.Agg.N) + out["input_tokens_per_query"] = optionalAverage(arm.InputTokens, arm.Agg.N) + out["output_tokens_per_query"] = optionalAverage(arm.OutputTokens, arm.Agg.N) + out[hit1], out[hit10], out[fmt.Sprintf("nDCG@%d", cutoffs.NDCG)] = s.Hit1, s.Hit10, s.NDCG + out["latency"] = arm.Lat.Summary() + out["cost_per_query_usd"] = formatCostPointer(arm.CostUSD, arm.Agg.N) + return out + } + results := make(map[string]any, len(r.Results)) + for mode, shapes := range r.Results { + byShape := make(map[string]any, len(r.Shapes)) + for _, shape := range r.Shapes { + if arm := shapes[shape]; arm != nil { + byShape[shape] = metrics(arm) + } else { + byShape[shape] = map[string]any{"status": "unrun"} + } + } + results[mode] = byShape + } + return map[string]any{ + "shapes": r.Shapes, "top": r.Top, "max_requests": r.MaxRequests, + "model": r.Model, "endpoint": r.Endpoint, "preprocess": r.Preprocess, + "input_usd_per_million": r.InputUSDPerM, + "output_usd_per_million": r.OutputUSDPerM, "complete": r.Complete, + "failure": nullableString(r.Failure), "results": results, + } +} + +func formatCostPointer(value *float64, queries int) any { + if value == nil || queries <= 0 { + return nil + } + return *value / float64(queries) +} + +func optionalAverage(value *int64, queries int) any { + if value == nil || queries <= 0 { + return nil + } + return float64(*value) / float64(queries) +} + +func nullableString(value string) any { + if value == "" { + return nil + } + return value +} + +func readEvalRerankOptions(cmd *cobra.Command) (evalRerankOptions, error) { + opts := evalRerankOptions{Top: evalRerankTop, MaxRequests: evalRerankMaxRequests} + if strings.TrimSpace(evalRerankJev) == "" { + return opts, nil + } + if evalDocKey != "message" { + return opts, errors.New("--rerank-jev requires --doc-key=message") + } + if opts.Top < 2 || opts.Top > rerank.MaxCandidates { + return opts, fmt.Errorf("--rerank-top must be between 2 and %d", rerank.MaxCandidates) + } + if opts.Top > evalLimit { + return opts, fmt.Errorf("--rerank-top (%d) cannot exceed --limit (%d)", opts.Top, evalLimit) + } + if opts.MaxRequests <= 0 { + return opts, errors.New("--rerank-max-requests must be positive") + } + if cmd != nil && !cmd.Flags().Changed("rerank-input-usd-per-million") { + return opts, errors.New("--rerank-input-usd-per-million is required when --rerank-jev is enabled") + } + if cmd != nil && !cmd.Flags().Changed("rerank-output-usd-per-million") { + return opts, errors.New("--rerank-output-usd-per-million is required when --rerank-jev is enabled") + } + if err := validatePrice("--rerank-input-usd-per-million", evalRerankInputUSDPerM); err != nil { + return opts, err + } + if err := validatePrice("--rerank-output-usd-per-million", evalRerankOutputUSDPerM); err != nil { + return opts, err + } + opts.InputUSDPerM = evalRerankInputUSDPerM + opts.OutputUSDPerM = evalRerankOutputUSDPerM + seen := make(map[string]struct{}, 2) + for raw := range strings.SplitSeq(evalRerankJev, ",") { + shape := strings.TrimSpace(raw) + if shape == "" { + continue + } + if shape != "per-candidate" && shape != "batched" { + return opts, fmt.Errorf("invalid --rerank-jev value %q (want per-candidate,batched)", shape) + } + if _, ok := seen[shape]; ok { + continue + } + seen[shape] = struct{}{} + opts.Shapes = append(opts.Shapes, shape) + } + if len(opts.Shapes) == 0 { + return opts, errors.New("--rerank-jev must name per-candidate or batched") + } + key := os.Getenv("TYPESAFE_API_KEY") + if strings.TrimSpace(key) == "" { + return opts, errors.New("TYPESAFE_API_KEY is required when --rerank-jev is enabled") + } + opts.APIKey = key + if state := invocationFromCommand(cmd); state != nil && state.cfg != nil { + vectorConfig := state.cfg.Vector + vectorConfig.ApplyDefaults() + opts.Preprocess = embeddingPreprocessConfig(vectorConfig) + } + return opts, nil +} + +func validatePrice(name string, value float64) error { + if math.IsNaN(value) || math.IsInf(value, 0) || value < 0 { + return fmt.Errorf("%s must be a finite nonnegative number", name) + } + return nil +} + +func validateJevRequestEstimate(topicCount, modeCount int, shapes []string, top, maxRequests int) error { + if topicCount <= 0 || modeCount <= 0 || len(shapes) == 0 { + return nil + } + if top <= 0 || maxRequests <= 0 { + return errors.New("invalid rerank request estimate inputs") + } + requestsPerTopic := 0 + for _, shape := range shapes { + requests := 1 + if shape == "per-candidate" { + requests = top + } else if shape != "batched" { + return fmt.Errorf("unknown Jev request shape %q", shape) + } + if modeCount > maxRequests/requests || modeCount*requests > maxRequests-requestsPerTopic { + return fmt.Errorf("--rerank-max-requests (%d) is below the conservative request estimate for %d judged topics across %d modes", + maxRequests, topicCount, modeCount) + } + requestsPerTopic += modeCount * requests + } + if requestsPerTopic > 0 && topicCount > maxRequests/requestsPerTopic { + return fmt.Errorf("--rerank-max-requests (%d) is below the conservative request estimate for %d judged topics across %d modes", + maxRequests, topicCount, modeCount) + } + return nil +} + +func truncateUTF8Bytes(value string, limit int) string { + if len([]byte(value)) <= limit { + return value + } + value = value[:limit] + for len(value) > 0 && !utf8.ValidString(value) { + value = value[:len(value)-1] + } + return value +} + +func prepareEvalCandidates(ctx context.Context, s *store.Store, keys []string, hits map[string]evalHit, preprocess embed.PreprocessConfig, top int) ([]string, error) { + if len(keys) == 0 { + return nil, nil + } + top = min(top, len(keys)) + texts := make([]string, top) + for i, key := range keys[:top] { + hit, ok := hits[key] + if !ok || hit.MessageID == 0 { + return nil, fmt.Errorf("rerank candidate %d has no message identity", i) + } + message, err := s.GetMessageContext(ctx, hit.MessageID) + if err != nil { + return nil, fmt.Errorf("prepare rerank candidate %d: %w", i, err) + } + body := embed.BodyTextForEmbedding(message.BodyText, message.BodyHTML) + text, _ := embed.Preprocess(message.Subject, body, 0, preprocess) + texts[i] = truncateUTF8Bytes(text, rerank.MaxCandidateBytes) + } + return texts, nil +} + +func rerankEvalKeys(ctx context.Context, scorer evalReranker, query string, keys, texts []string) ([]string, rerank.Result, error) { + if len(keys) < 2 || len(texts) < 2 { + return slices.Clone(keys), rerank.Result{Usage: rerank.Usage{ + InputTokens: new(int64(0)), OutputTokens: new(int64(0)), Complete: true, + }}, nil + } + texts = texts[:min(len(texts), len(keys))] + result, err := scorer.Rerank(ctx, rerank.Request{Query: query, Candidates: texts}) + if err != nil { + return nil, result, err + } + if len(result.Scores) != len(texts) { + return nil, result, fmt.Errorf("%w: returned %d scores for %d candidates", rerank.ErrInvalidResponse, len(result.Scores), len(texts)) + } + order, err := rerank.Order(result.Scores) + if err != nil { + return nil, result, fmt.Errorf("%w: %w", rerank.ErrInvalidResponse, err) + } + out := slices.Clone(keys) + prefix := slices.Clone(keys[:len(texts)]) + for i, index := range order { + out[i] = prefix[index] + } + return out, result, nil +} + +func (r *evalRerankReport) fail(mode, shape, reason string) { + arm := r.arm(mode, shape) + arm.Status, arm.Complete, arm.Error = "failed", false, reason + r.Complete, r.Failure = false, reason +} + +func (r *evalRerankReport) scoreRanking(ctx context.Context, s *store.Store, hits map[string]evalHit, + mode string, topic eval.Topic, ranked []string, rel map[string]struct{}, cutoffs eval.Cutoffs, elapsed time.Duration, +) error { + prepStart := time.Now() + var texts []string + if len(ranked) >= 2 { + var err error + texts, err = prepareEvalCandidates(ctx, s, ranked, hits, r.Preprocess, r.Top) + if err != nil { + for _, shape := range r.Shapes { + r.fail(mode, shape, "candidate preparation failed") + } + return fmt.Errorf("topic %s, mode %s: candidate preparation failed", topic.ID, mode) + } + } + prepElapsed := time.Since(prepStart) + for _, shape := range r.Shapes { + arm := r.arm(mode, shape) + providerStart := time.Now() + reranked, result, err := rerankEvalKeys(ctx, r.scorers[shape], topic.Query, ranked, texts) + arm.addUsage(result.Usage, r.InputUSDPerM, r.OutputUSDPerM) + if err != nil { + failure := rerank.SafeFailure(err) + r.fail(mode, shape, failure) + return fmt.Errorf("topic %s, mode %s, shape %s: %s", topic.ID, mode, shape, failure) + } + latency := elapsed + if len(texts) >= 2 { + latency += prepElapsed + time.Since(providerStart) + } + arm.addQuality(reranked, rel, cutoffs, latency) + } + return nil +} + +func (r *evalRerankReport) reconcile(modes []string, baseline map[string]*eval.Aggregate) { + for _, mode := range modes { + for _, shape := range r.Shapes { + if r.Results[mode][shape] == nil { + arm := r.arm(mode, shape) + arm.Status, arm.Complete = "unrun", false + r.Complete = false + continue + } + arm := r.Results[mode][shape] + if arm.Status != "failed" && (arm.Agg == nil || arm.Agg.N != baseline[mode].N) { + arm.Status, arm.Complete = "incomplete", false + r.Complete = false + } + } + } +} diff --git a/cmd/msgvault/cmd/eval_rerank_test.go b/cmd/msgvault/cmd/eval_rerank_test.go new file mode 100644 index 000000000..e4e72d83d --- /dev/null +++ b/cmd/msgvault/cmd/eval_rerank_test.go @@ -0,0 +1,525 @@ +//go:build fts5 && sqlite_vec + +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "math" + "os" + "path/filepath" + "strings" + "testing" + "time" + "unicode/utf8" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/eval" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/vector" + "go.kenn.io/msgvault/internal/vector/embed" + "go.kenn.io/msgvault/internal/vector/rerank" + "go.kenn.io/msgvault/internal/vector/sqlitevec" +) + +type evalRerankRecorder struct { + requests map[string][]rerank.Request + failAt map[string]int + promoteText string + factoryCalls int +} + +type recordingReranker struct { + shape string + recorder *evalRerankRecorder +} + +func (r *evalRerankRecorder) makeReranker(shape, _ string) (evalReranker, error) { + r.factoryCalls++ + if r.requests == nil { + r.requests = make(map[string][]rerank.Request) + } + return &recordingReranker{shape: shape, recorder: r}, nil +} + +func (r *recordingReranker) Rerank(ctx context.Context, request rerank.Request) (rerank.Result, error) { + request.Candidates = append([]string(nil), request.Candidates...) + r.recorder.requests[r.shape] = append(r.recorder.requests[r.shape], request) + if r.recorder.failAt[r.shape] == len(r.recorder.requests[r.shape]) { + return rerank.Result{}, errors.New("fake provider failure") + } + scores := make([]float64, len(request.Candidates)) + for i := range scores { + if r.recorder.promoteText != "" { + if strings.Contains(strings.ToLower(request.Candidates[i]), r.recorder.promoteText) { + scores[i] = 1 + } + continue + } + scores[i] = float64(len(scores)-i) / float64(len(scores)) + } + input, output := int64(5), int64(2) + requests := 1 + if r.shape == "per-candidate" { + requests = len(request.Candidates) + } + return rerank.Result{Scores: scores, Usage: rerank.Usage{ + Requests: requests, InputTokens: &input, OutputTokens: &output, Complete: true, + }}, nil +} + +func preserveEvalRerankGlobals(t *testing.T) { + t.Helper() + oldQrels, oldTopics, oldModes, oldDocKey := evalQrels, evalTopics, evalModes, evalDocKey + oldLimit, oldJSON := evalLimit, evalJSON + oldJev, oldTop, oldMaxRequests := evalRerankJev, evalRerankTop, evalRerankMaxRequests + oldInput, oldOutput := evalRerankInputUSDPerM, evalRerankOutputUSDPerM + t.Cleanup(func() { + evalQrels, evalTopics, evalModes, evalDocKey = oldQrels, oldTopics, oldModes, oldDocKey + evalLimit, evalJSON = oldLimit, oldJSON + evalRerankJev, evalRerankTop, evalRerankMaxRequests = oldJev, oldTop, oldMaxRequests + evalRerankInputUSDPerM, evalRerankOutputUSDPerM = oldInput, oldOutput + }) +} + +func newEvalRerankTestCommand(t *testing.T, out *bytes.Buffer, inputPrice, outputPrice bool) *cobra.Command { + t.Helper() + cmd := &cobra.Command{} + cmd.Flags().Float64("rerank-input-usd-per-million", 0, "") + cmd.Flags().Float64("rerank-output-usd-per-million", 0, "") + if inputPrice { + require.NoError(t, cmd.Flags().Set("rerank-input-usd-per-million", "1")) + } + if outputPrice { + require.NoError(t, cmd.Flags().Set("rerank-output-usd-per-million", "1")) + } + cmd.SetContext(t.Context()) + cmd.SetOut(out) + cmd.SetErr(&bytes.Buffer{}) + return cmd +} + +func prepareEvalRerankRun(t *testing.T, shapes string, topicCount int) (*cobra.Command, *bytes.Buffer) { + t.Helper() + preserveEvalRerankGlobals(t) + dir := t.TempDir() + seedRankingDivergenceArchiveIn(t, dir) + cfg := config.NewDefaultConfig() + cfg.Data.DataDir = dir + evalModes, evalDocKey, evalLimit, evalJSON = "fts", "message", 10, true + evalRerankJev, evalRerankTop, evalRerankMaxRequests = shapes, 2, 1000 + evalRerankInputUSDPerM, evalRerankOutputUSDPerM = 1, 1 + t.Setenv("TYPESAFE_API_KEY", "test-key") + var topics, qrels strings.Builder + for i := range topicCount { + qid := fmt.Sprintf("q%d", i+1) + fmt.Fprintf(&topics, "%s\trenewal\n", qid) + fmt.Fprintf(&qrels, "%s 0 1\n", qid) + } + evalQrels = writeEvalFile(t, dir, "qrels.txt", qrels.String()) + evalTopics = writeEvalFile(t, dir, "topics.tsv", topics.String()) + out := &bytes.Buffer{} + cmd := newEvalRerankTestCommand(t, out, true, true) + cmd.SetContext(testInvocationContext(cmd.Context(), cfg, invocationOptions{})) + return cmd, out +} + +func TestRunEvalReranksFTSCandidates(t *testing.T) { + assert := assert.New(t) + cmd, out := prepareEvalRerankRun(t, "batched,per-candidate", 2) + recorder := &evalRerankRecorder{} + require.NoError(t, runEvalWithRerankerFactory(cmd, nil, recorder.makeReranker)) + for _, shape := range []string{"batched", "per-candidate"} { + require.Len(t, recorder.requests[shape], 2) + for _, request := range recorder.requests[shape] { + assert.Equal("renewal", request.Query) + require.Len(t, request.Candidates, 2) + candidateText := strings.ToLower(strings.Join(request.Candidates, "\n")) + assert.Contains(candidateText, "lease renewal terms") + assert.Contains(candidateText, "signed and returned") + assert.NotContains(candidateText, "") + assert.NotContains(candidateText, "me@example.com") + } + } + var report struct { + Rerank struct { + Complete bool `json:"complete"` + MaxRequests int `json:"max_requests"` + InputUSDPerMillion float64 `json:"input_usd_per_million"` + OutputUSDPerMillion float64 `json:"output_usd_per_million"` + Results map[string]map[string]struct { + Status string `json:"status"` + Requests int `json:"requests"` + Topics int `json:"topics"` + RequestsPerQuery float64 `json:"requests_per_query"` + InputTokensPerQuery float64 `json:"input_tokens_per_query"` + OutputTokensPerQuery float64 `json:"output_tokens_per_query"` + CostPerQueryUSD float64 `json:"cost_per_query_usd"` + Latency struct { + P95MS *float64 `json:"p95_ms"` + } `json:"latency"` + } `json:"results"` + } `json:"rerank_results"` + } + require.NoError(t, json.Unmarshal(out.Bytes(), &report)) + assert.True(report.Rerank.Complete) + assert.Equal(1000, report.Rerank.MaxRequests) + assert.InDelta(1.0, report.Rerank.InputUSDPerMillion, 1e-9) + assert.InDelta(1.0, report.Rerank.OutputUSDPerMillion, 1e-9) + for shape, requests := range map[string]int{"batched": 2, "per-candidate": 4} { + arm := report.Rerank.Results["fts"][shape] + assert.Equal("complete", arm.Status) + assert.Equal(2, arm.Topics) + assert.Equal(requests, arm.Requests) + assert.InDelta(float64(requests)/2, arm.RequestsPerQuery, 1e-9) + assert.InDelta(5.0, arm.InputTokensPerQuery, 1e-9) + assert.InDelta(2.0, arm.OutputTokensPerQuery, 1e-9) + assert.InDelta(0.000007, arm.CostPerQueryUSD, 1e-12) + assert.NotNil(arm.Latency.P95MS) + } +} + +type fakeReranker struct{} + +func (fakeReranker) Rerank(_ context.Context, _ rerank.Request) (rerank.Result, error) { + return rerank.Result{Scores: []float64{0.1, 0.9}, Usage: rerank.Usage{Complete: true}}, nil +} + +func TestEvalRerankShortlist(t *testing.T) { + assert := assert.New(t) + keys, result, err := rerankEvalKeys(context.Background(), fakeReranker{}, "query", + []string{"first", "second", "tail"}, []string{"one", "two"}) + require.NoError(t, err) + assert.Equal([]string{"second", "first", "tail"}, keys) + assert.Equal([]float64{0.1, 0.9}, result.Scores) + for _, shortlist := range [][]string{nil, {"only"}} { + keys, result, err := rerankEvalKeys(t.Context(), nil, "query", shortlist, shortlist) + require.NoError(t, err) + assert.Equal(shortlist, keys) + assert.Equal(rerank.Usage{InputTokens: new(int64(0)), OutputTokens: new(int64(0)), Complete: true}, result.Usage) + } +} + +func TestRunEvalRerankFailureKeepsCompleteBaseline(t *testing.T) { + cmd, out := prepareEvalRerankRun(t, "batched,per-candidate", 3) + recorder := &evalRerankRecorder{failAt: map[string]int{"batched": 2}} + err := runEvalWithRerankerFactory(cmd, nil, recorder.makeReranker) + require.ErrorContains(t, err, "provider request failed") + require.Len(t, recorder.requests["batched"], 2, "provider work stops after the failed request") + require.Len(t, recorder.requests["per-candidate"], 1) + + var report struct { + TopicsEvaluated int `json:"topics_evaluated"` + Results map[string]struct { + Topics int `json:"topics"` + } `json:"results"` + Rerank struct { + Complete bool `json:"complete"` + Results map[string]map[string]map[string]json.RawMessage `json:"results"` + } `json:"rerank_results"` + } + require.NoError(t, json.Unmarshal(out.Bytes(), &report)) + assert.Equal(t, 3, report.TopicsEvaluated) + assert.Equal(t, 3, report.Results["fts"].Topics) + assert.False(t, report.Rerank.Complete) + arm := report.Rerank.Results["fts"]["batched"] + var status string + require.NoError(t, json.Unmarshal(arm["status"], &status)) + assert.Equal(t, "failed", status) + assert.NotContains(t, arm, "Hit@1") + var requests int + require.NoError(t, json.Unmarshal(arm["requests"], &requests)) + assert.Equal(t, 1, requests) + var inputTokens int64 + require.NoError(t, json.Unmarshal(arm["input_tokens"], &inputTokens)) + assert.Equal(t, int64(5), inputTokens) + var outputTokens int64 + require.NoError(t, json.Unmarshal(arm["output_tokens"], &outputTokens)) + assert.Equal(t, int64(2), outputTokens) + assert.Contains(t, string(arm["cost_usd"]), "null") + var incompleteStatus string + var incompleteTopics int + incomplete := report.Rerank.Results["fts"]["per-candidate"] + require.NoError(t, json.Unmarshal(incomplete["status"], &incompleteStatus)) + require.NoError(t, json.Unmarshal(incomplete["topics"], &incompleteTopics)) + assert.Equal(t, "incomplete", incompleteStatus) + assert.Equal(t, 1, incompleteTopics) + assert.NotContains(t, incomplete, "Hit@1") + assert.NotContains(t, incomplete, "latency") +} + +func TestRunEvalRerankQualityMetricsFollowProviderScoresAcrossModes(t *testing.T) { + cmd, out := prepareEvalRerankRun(t, "batched", 1) + cfg := invocationFromContext(cmd.Context()).cfg + dataDir := cfg.Data.DataDir + evalQrels = writeEvalFile(t, dataDir, "quality-qrels.txt", "q1 0 1\n") + evalModes = "fts,vector,hybrid" + c := evalVectorConfig(t, vector.APIFormatOpenAI, "test-model") + c.Data.DataDir = dataDir + c.Vector.Embeddings.Dimension = 3 + _, endpoint := embedTestServer(t, `{"data":[{"index":0,"embedding":[1,0,0]}]}`) + c.Vector.Embeddings.Endpoint = endpoint + invocationFromContext(cmd.Context()).cfg = c + cfg = c + cmd.SetContext(testInvocationContext(cmd.Context(), cfg, invocationOptions{})) + s, err := store.Open(c.DatabaseDSN()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, s.Close()) }) + require.NoError(t, s.InitSchema()) + seedEmbeddedGeneration(t, dataDir, c.DatabaseDSN(), s, c.Vector, 1, 2) + require.NoError(t, sqlitevec.RegisterExtension()) + backend, err := sqlitevec.Open(context.Background(), sqlitevec.Options{ + Path: filepath.Join(dataDir, "vectors.db"), MainPath: c.DatabaseDSN(), + Dimension: 3, MainDB: s.DB(), + }) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, backend.Close()) }) + generation, err := backend.ActiveGeneration(context.Background()) + require.NoError(t, err) + require.NoError(t, backend.Upsert(context.Background(), generation.ID, []vector.Chunk{ + {MessageID: 1, Vector: []float32{1, 0, 0}, SourceCharLen: 32}, + {MessageID: 2, Vector: []float32{0, 1, 0}, SourceCharLen: 32}, + })) + + recorder := &evalRerankRecorder{promoteText: "weekly digest"} + require.NoError(t, runEvalWithRerankerFactory(cmd, nil, recorder.makeReranker)) + var report struct { + Results map[string]json.RawMessage `json:"results"` + RerankResult struct { + Results map[string]map[string]map[string]json.RawMessage `json:"results"` + } `json:"rerank_results"` + } + require.NoError(t, json.Unmarshal(out.Bytes(), &report)) + for _, mode := range []string{"fts", "vector", "hybrid"} { + var baseline struct { + Hit1 float64 `json:"Hit@1"` + } + require.NoError(t, json.Unmarshal(report.Results[mode], &baseline)) + var reranked struct { + Hit1 float64 `json:"Hit@1"` + Status string `json:"status"` + } + require.NoError(t, json.Unmarshal(report.RerankResult.Results[mode]["batched"]["Hit@1"], &reranked.Hit1)) + require.NoError(t, json.Unmarshal(report.RerankResult.Results[mode]["batched"]["status"], &reranked.Status)) + assert.InDelta(t, 0, baseline.Hit1, 1e-9, mode) + assert.InDelta(t, 1, reranked.Hit1, 1e-9, mode) + assert.Equal(t, "complete", reranked.Status, mode) + } + assert.Len(t, recorder.requests["batched"], 3) +} + +func TestValidateJevRequestEstimate(t *testing.T) { + shapes := []string{"per-candidate", "batched"} + require.NoError(t, validateJevRequestEstimate(10, 3, shapes, 30, 1000)) + require.ErrorContains(t, validateJevRequestEstimate(11, 3, shapes, 30, 1000), "conservative request estimate") +} + +func TestRunEvalPreflightsJevRequestEstimateBeforeOpeningArchive(t *testing.T) { + preserveEvalRerankGlobals(t) + dir := t.TempDir() + cfg := config.NewDefaultConfig() + cfg.Data.DataDir = dir + evalModes, evalDocKey, evalLimit, evalJSON = "fts,vector,hybrid", "message", 100, true + evalRerankJev, evalRerankTop, evalRerankMaxRequests = "per-candidate,batched", 30, 1000 + evalRerankInputUSDPerM, evalRerankOutputUSDPerM = 1, 1 + t.Setenv("TYPESAFE_API_KEY", "test-key") + var topics, qrels strings.Builder + for i := range 11 { + qid := fmt.Sprintf("q%d", i+1) + fmt.Fprintf(&topics, "%s\trenewal\n", qid) + fmt.Fprintf(&qrels, "%s 0 1\n", qid) + } + evalQrels = writeEvalFile(t, dir, "qrels.txt", qrels.String()) + evalTopics = writeEvalFile(t, dir, "topics.tsv", topics.String()) + cmd := newEvalRerankTestCommand(t, &bytes.Buffer{}, true, true) + cmd.SetContext(testInvocationContext(cmd.Context(), cfg, invocationOptions{})) + recorder := &evalRerankRecorder{} + err := runEvalWithRerankerFactory(cmd, nil, recorder.makeReranker) + require.ErrorContains(t, err, "conservative request estimate") + assert.Zero(t, recorder.factoryCalls) + _, statErr := os.Stat(filepath.Join(dir, "msgvault.db")) + assert.True(t, os.IsNotExist(statErr), "the rejected estimate must not open the archive") +} + +func TestEvalRerankCandidateText(t *testing.T) { + assert := assert.New(t) + text := truncateUTF8Bytes(strings.Repeat("界", 1000), rerank.MaxCandidateBytes) + assert.LessOrEqual(len([]byte(text)), 2048) + assert.True(utf8.ValidString(text)) + assert.Equal("abc", truncateUTF8Bytes("abc", 2048)) +} + +func TestEvalRerankOptIn(t *testing.T) { + assert := assert.New(t) + old := evalRerankJev + t.Cleanup(func() { evalRerankJev = old }) + evalRerankJev = "" + options, err := readEvalRerankOptions(nil) + require.NoError(t, err) + assert.Empty(options.Shapes) +} + +func TestReadEvalRerankOptionsRejectsInvalidInputs(t *testing.T) { + cases := []struct { + name string + inputPrice bool + outputPrice bool + key string + wantError string + mutate func() + }{ + {name: "conversation key", inputPrice: true, outputPrice: true, key: "test-key", wantError: "--rerank-jev requires --doc-key=message", mutate: func() { evalDocKey = "conversation" }}, + {name: "top outside provider bound", inputPrice: true, outputPrice: true, key: "test-key", wantError: "--rerank-top must be between", mutate: func() { evalRerankTop = rerank.MaxCandidates + 1 }}, + {name: "top cannot rerank", inputPrice: true, outputPrice: true, key: "test-key", wantError: "--rerank-top must be between", mutate: func() { evalRerankTop = 1 }}, + {name: "top exceeds retrieval depth", inputPrice: true, outputPrice: true, key: "test-key", wantError: "cannot exceed --limit", mutate: func() { evalRerankTop = evalLimit + 1 }}, + {name: "nonpositive request limit", inputPrice: true, outputPrice: true, key: "test-key", wantError: "--rerank-max-requests must be positive", mutate: func() { evalRerankMaxRequests = 0 }}, + {name: "missing input price", outputPrice: true, key: "test-key", wantError: "--rerank-input-usd-per-million is required"}, + {name: "missing output price", inputPrice: true, key: "test-key", wantError: "--rerank-output-usd-per-million is required"}, + {name: "negative input price", inputPrice: true, outputPrice: true, key: "test-key", wantError: "must be a finite nonnegative number", mutate: func() { evalRerankInputUSDPerM = -1 }}, + {name: "nonfinite output price", inputPrice: true, outputPrice: true, key: "test-key", wantError: "must be a finite nonnegative number", mutate: func() { evalRerankOutputUSDPerM = math.Inf(1) }}, + {name: "unknown shape", inputPrice: true, outputPrice: true, key: "test-key", wantError: "invalid --rerank-jev value", mutate: func() { evalRerankJev = "unknown" }}, + {name: "empty shapes", inputPrice: true, outputPrice: true, key: "test-key", wantError: "must name per-candidate or batched", mutate: func() { evalRerankJev = ",," }}, + {name: "missing API key", inputPrice: true, outputPrice: true, key: " ", wantError: "TYPESAFE_API_KEY is required"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + preserveEvalRerankGlobals(t) + cfg := config.NewDefaultConfig() + evalDocKey, evalLimit = "message", 10 + evalRerankJev, evalRerankTop, evalRerankMaxRequests = "batched", 2, 100 + evalRerankInputUSDPerM, evalRerankOutputUSDPerM = 1, 1 + t.Setenv("TYPESAFE_API_KEY", tc.key) + cmd := newEvalRerankTestCommand(t, &bytes.Buffer{}, tc.inputPrice, tc.outputPrice) + cmd.SetContext(testInvocationContext(cmd.Context(), cfg, invocationOptions{})) + if tc.mutate != nil { + tc.mutate() + } + _, err := readEvalRerankOptions(cmd) + require.ErrorContains(t, err, tc.wantError) + }) + } +} + +func TestPrepareEvalCandidatesRequiresMessageIdentity(t *testing.T) { + _, err := prepareEvalCandidates(t.Context(), nil, []string{""}, nil, embed.PreprocessConfig{}, 1) + require.ErrorContains(t, err, "has no message identity") +} + +func TestHitDepthLabel(t *testing.T) { + assert := assert.New(t) + report := evalReport{cutoffs: eval.CutoffsForDepth(5), diag: &runDiagnostics{}} + var output bytes.Buffer + require.NoError(t, report.table(&output)) + assert.Contains(output.String(), "Hit@5") +} + +type evalRerankFailingWriter struct{ err error } + +func (w evalRerankFailingWriter) Write([]byte) (int, error) { return 0, w.err } + +func TestEvalReportTablesReturnWriterErrors(t *testing.T) { + writeErr := errors.New("write failed") + writer := evalRerankFailingWriter{err: writeErr} + + require.ErrorIs(t, (evalReport{}).table(writer), writeErr) + require.ErrorIs(t, newEvalRerankReport(evalRerankOptions{}).table(writer, eval.StandardCutoffs), writeErr) +} + +func TestEvalRerankReport(t *testing.T) { + assert := assert.New(t) + inputPrice, outputPrice := 1.0, 2.0 + report := newEvalRerankReport(evalRerankOptions{ + Shapes: []string{"batched"}, Top: 30, MaxRequests: 1000, InputUSDPerM: inputPrice, OutputUSDPerM: outputPrice, + }) + arm := report.arm("hybrid", "batched") + arm.addUsage(rerank.Usage{Requests: 1}, inputPrice, outputPrice) + arm.addQuality([]string{"relevant", "other"}, map[string]struct{}{"relevant": {}}, eval.CutoffsForDepth(5), time.Millisecond) + var output bytes.Buffer + err := evalReport{modes: []string{"hybrid"}, aggs: map[string]*eval.Aggregate{"hybrid": {}}, + lats: map[string]*eval.LatencyTracker{"hybrid": &eval.LatencyTracker{}}, + catAggs: map[string]map[string]*eval.Aggregate{}, cutoffs: eval.CutoffsForDepth(5), + diag: &runDiagnostics{}, rerank: report}.json(&output) + require.NoError(t, err) + assert.Contains(output.String(), `"usage_complete": false`) + assert.Contains(output.String(), `"input_tokens": 0`) + assert.Contains(output.String(), `"output_tokens": 0`) + assert.Contains(output.String(), `"cost_usd": null`) + assert.Contains(output.String(), `"input_tokens_per_query": 0`) + assert.Contains(output.String(), `"output_tokens_per_query": 0`) + assert.Contains(output.String(), `"cost_per_query_usd": null`) + assert.Contains(output.String(), `"Hit@5"`) +} + +func TestEvalRerankFailure(t *testing.T) { + report := newEvalRerankReport(evalRerankOptions{ + Shapes: []string{"batched"}, Top: 30, InputUSDPerM: 1, OutputUSDPerM: 2, + }) + arm := report.arm("hybrid", "batched") + input, outputTokens := int64(5), int64(2) + arm.addUsage(rerank.Usage{Requests: 1, InputTokens: &input, OutputTokens: &outputTokens}, 1, 2) + arm.addQuality([]string{"relevant", "other"}, map[string]struct{}{"relevant": {}}, eval.CutoffsForDepth(5), time.Millisecond) + arm.Status = "failed" + arm.Complete = false + arm.Error = "provider request failed" + report.Complete = false + report.Failure = "provider request failed" + + base := &eval.Aggregate{} + base.Add(eval.Evaluate([]string{"relevant"}, map[string]struct{}{"relevant": {}}, eval.CutoffsForDepth(5))) + var output bytes.Buffer + err := (evalReport{ + modes: []string{"hybrid"}, aggs: map[string]*eval.Aggregate{"hybrid": base}, + lats: map[string]*eval.LatencyTracker{"hybrid": &eval.LatencyTracker{}}, + catAggs: map[string]map[string]*eval.Aggregate{}, cutoffs: eval.CutoffsForDepth(5), + diag: &runDiagnostics{}, rerank: report, + }).json(&output) + require.NoError(t, err) + assert.Contains(t, output.String(), `"Hit@5"`) + assert.Contains(t, output.String(), `"status": "failed"`) + assert.Contains(t, output.String(), `"failure": "provider request failed"`) + assert.Contains(t, output.String(), `"usage_complete": false`) + assert.Contains(t, output.String(), `"input_tokens": 5`) + assert.Contains(t, output.String(), `"output_tokens": 2`) + assert.Contains(t, output.String(), `"cost_usd": null`) + var document map[string]json.RawMessage + require.NoError(t, json.Unmarshal(output.Bytes(), &document)) + var rerankJSON struct { + Results map[string]map[string]map[string]json.RawMessage `json:"results"` + } + require.NoError(t, json.Unmarshal(document["rerank_results"], &rerankJSON)) + assert.NotContains(t, rerankJSON.Results["hybrid"]["batched"], "Hit@1") + assert.NotContains(t, rerankJSON.Results["hybrid"]["batched"], "Hit@5") + + var table bytes.Buffer + require.NoError(t, report.table(&table, eval.CutoffsForDepth(5))) + var failedRow string + for line := range strings.SplitSeq(table.String(), "\n") { + if strings.Contains(line, "hybrid") && strings.Contains(line, "batched") && strings.Contains(line, "failed") { + failedRow = line + break + } + } + fields := strings.Fields(failedRow) + require.GreaterOrEqual(t, len(fields), 16) + assert.Equal(t, []string{"hybrid", "batched", "failed", "false", "1", "-", "-", "-", "-"}, fields[:9]) + assert.Equal(t, []string{"1", "-", "5", "-", "2", "-", "unknown"}, fields[9:16]) +} + +func TestEvalJSONOutputIsDeterministic(t *testing.T) { + var output bytes.Buffer + require.NoError(t, printJSONTo(&output, map[string]any{ + "z": 1, + "a": map[string]int{"y": 2, "b": 1}, + })) + text := output.String() + assert.Less(t, strings.Index(text, `"a"`), strings.Index(text, `"z"`)) + _, inner, found := strings.Cut(text, `"a"`) + require.True(t, found) + assert.Less(t, strings.Index(inner, `"b"`), strings.Index(inner, `"y"`)) +} diff --git a/cmd/msgvault/cmd/eval_source_scope_test.go b/cmd/msgvault/cmd/eval_source_scope_test.go new file mode 100644 index 000000000..1e40cc284 --- /dev/null +++ b/cmd/msgvault/cmd/eval_source_scope_test.go @@ -0,0 +1,217 @@ +//go:build fts5 && sqlite_vec + +package cmd + +import ( + "path/filepath" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/msgvault/internal/store" +) + +// seedTwoSourceArchiveIn builds an archive with two connected accounts whose +// source-assigned ids overlap, which is the shape a multi-source archiver +// routinely ends up in and no single-account benchmark corpus ever has. +// +// source 1 (gmail) thread-1 / , thread-1 / +// source 2 (whatsapp) thread-1 / +// +// Both the message id and the conversation id are shared across the two +// sources, and every message is live, so retrieval can return either copy. +// Passing shareMessageID=false keeps the message ids disjoint while leaving the +// conversation ids shared, so a test can tell the two doc-keys apart. +func seedTwoSourceArchiveIn(t *testing.T, dataDir string, shareMessageID bool) *store.Store { + t.Helper() + require := require.New(t) + + s, err := store.Open(filepath.Join(dataDir, "msgvault.db")) + require.NoError(err, "open store") + t.Cleanup(func() { require.NoError(s.Close()) }) + require.NoError(s.InitSchema(), "init schema") + + secondMessageID := "" + if !shareMessageID { + secondMessageID = "" + } + _, err = s.DB().Exec(` +INSERT INTO sources (id, source_type, identifier) VALUES + (1, 'gmail', 'me@example.com'), + (2, 'whatsapp', '+15550100'); +INSERT INTO conversations (id, source_id, source_conversation_id, conversation_type) VALUES + (1, 1, 'thread-1', 'email_thread'), + (2, 2, 'thread-1', 'whatsapp_chat'); +INSERT INTO messages + (id, conversation_id, source_id, source_message_id, message_type, subject, sent_at, size_estimate) VALUES + (1, 1, 1, '', 'email', 'Lease renewal terms', '2020-01-01T00:00:00Z', 100), + (2, 1, 1, '', 'email', 'Re: Lease renewal', '2020-01-02T00:00:00Z', 100), + (3, 2, 2, ?, 'whatsapp', 'Lease renewal chat', '2020-01-03T00:00:00Z', 100); +`, secondMessageID) + require.NoError(err, "seed two-source archive") + + for id, body := range map[int64]string{ + 1: "Signed and returned.", + 2: "Counter-signed.", + 3: "Unrelated chat that happens to reuse the id.", + } { + _, err = s.DB().Exec( + `INSERT INTO message_bodies (message_id, body_text) VALUES (?, ?)`, id, body) + require.NoError(err, "seed body %d", id) + } + indexed, err := s.BackfillFTS(nil) + require.NoError(err, "backfill FTS") + require.EqualValues(3, indexed, "every message must be indexed") + return s +} + +// TestRankedFTS_CrossSourceIDsCollapseIntoOneKey characterizes the hazard the +// precondition exists to prevent, so the guard below is not asserting the +// absence of a problem nobody has shown to exist. +// +// Message 1 (gmail) and message 3 (whatsapp) are unrelated documents that +// happen to carry the same source-assigned id. Retrieval finds both, and the +// doc-key extraction reduces them to one key: the ranking hands the scoring +// core two hits' worth of evidence under a single id, so a judgment written +// about the gmail message silently grades the whatsapp one as well, and the +// depth quietly loses a rank. +func TestRankedFTS_CrossSourceIDsCollapseIntoOneKey(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + s := seedTwoSourceArchiveIn(t, t.TempDir(), true) + const topic = "renewal" + + found, _, err := s.SearchMessagesQueryContext(t.Context(), evalTestQuery(t, topic), 0, 10) + require.NoError(err, "production store search") + require.Len(found, 3, "all three messages are live and match the topic") + + ev, _ := newTestEvaluator(t, s, "message") + ev.limit = 10 + ranked, err := ev.rankedFTS(evalTestQuery(t, topic)) + require.NoError(err, "rankedFTS") + + assert.Len(ranked, 2, + "three retrieved messages reduce to two keys: the two sources' ids collided") + assert.Contains(ranked, "", + "and the surviving key names a document in each source at once") +} + +// TestRequireDisjointSourceIDs_RejectsCollidingMessageIDs is the guard for that +// collapse under --doc-key=message. The run must stop with an error that names +// the colliding id, the key, and the column whose uniqueness does not hold. +func TestRequireDisjointSourceIDs_RejectsCollidingMessageIDs(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + s := seedTwoSourceArchiveIn(t, t.TempDir(), true) + registry := newDocKeyRegistry() + + err := requireDisjointSourceIDs(t.Context(), s.DB(), "message", registry["message"]) + require.Error(err, "a colliding message id must stop the run, not be scored") + assert.Contains(err.Error(), "", "the offending id is named") + assert.Contains(err.Error(), "source_message_id", "and so is the column whose uniqueness failed") + assert.NotContains(err.Error(), "", + "an id held by one source only is not a collision") +} + +// TestRequireDisjointSourceIDs_RejectsCollidingConversationIDs pins the same +// guard for the coarser key, whose id lives on conversations rather than +// messages. Here the message ids are disjoint and only the thread ids overlap, +// so the two keys must reach opposite verdicts on the same archive — a check +// that quietly looked at one column for both keys would pass this. +func TestRequireDisjointSourceIDs_RejectsCollidingConversationIDs(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + s := seedTwoSourceArchiveIn(t, t.TempDir(), false) + registry := newDocKeyRegistry() + + require.NoError(requireDisjointSourceIDs(t.Context(), s.DB(), "message", registry["message"]), + "the message ids are disjoint in this archive") + + err := requireDisjointSourceIDs(t.Context(), s.DB(), "conversation", registry["conversation"]) + require.Error(err, "the thread ids are not") + assert.Contains(err.Error(), "thread-1") + assert.Contains(err.Error(), "source_conversation_id") +} + +// TestRequireDisjointSourceIDs_AllowsSeveralSourcesWithDisjointIDs is the other +// half of the decision: the precondition is disjointness, not single-source. +// msgvault archives normally hold several accounts, and refusing to score one +// because it has two sources — when nothing in it can collide — would be a wall +// built for a hazard that is not there. +func TestRequireDisjointSourceIDs_AllowsSeveralSourcesWithDisjointIDs(t *testing.T) { + require := require.New(t) + s := seedTwoSourceArchiveIn(t, t.TempDir(), false) + + var sources int + require.NoError(s.DB().QueryRow( + `SELECT COUNT(DISTINCT source_id) FROM messages`).Scan(&sources)) + require.Equal(2, sources, "the archive really does hold two connected accounts") + + require.NoError(requireDisjointSourceIDs( + t.Context(), s.DB(), "message", newDocKeyRegistry()["message"]), + "two accounts that share no message id are scorable") +} + +// TestRequireDisjointSourceIDs_IgnoresUnretrievableCopies pins the population +// the check runs over. A dedup-hidden or source-deleted copy is not something +// any search here returns, so it cannot reach a ranking and cannot collide with +// anything — counting it would refuse to score an archive that is fine. +func TestRequireDisjointSourceIDs_IgnoresUnretrievableCopies(t *testing.T) { + require := require.New(t) + s := seedTwoSourceArchiveIn(t, t.TempDir(), true) + + // Hide the second source's copy the way dedup does, leaving the row in + // place. + _, err := s.DB().Exec( + `UPDATE messages SET deleted_at = '2023-01-01T00:00:00Z' WHERE id = 3`) + require.NoError(err, "hide the duplicate") + + require.NoError(requireDisjointSourceIDs( + t.Context(), s.DB(), "message", newDocKeyRegistry()["message"]), + "the surviving copy is the only retrievable one, so the id names one document") +} + +// TestRequireDisjointSourceIDs_RejectsAKeyWithNoArchiveColumn keeps the check +// from being skipped by omission. A doc-key added without saying where its ids +// live would otherwise sail past the guard and reintroduce the collapse; it has +// to fail loudly instead, naming itself. +func TestRequireDisjointSourceIDs_RejectsAKeyWithNoArchiveColumn(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + s := seedTwoSourceArchiveIn(t, t.TempDir(), false) + unbacked := docKeySpec{extract: func(evalHit) string { return "x" }} + + err := requireDisjointSourceIDs(t.Context(), s.DB(), "thread", unbacked) + require.Error(err, "an unbacked key must not silently skip the check") + assert.Contains(err.Error(), `"thread"`) +} + +// TestRunEval_StopsOnCrossSourceIDCollisions drives the whole command, so the +// guard is pinned at the call site and not just in its own helper. Without it +// the run completes and reports a perfect MRR for q1: the whatsapp message +// carries the gmail message's id, so its hit is scored against the gmail +// message's judgment. +func TestRunEval_StopsOnCrossSourceIDCollisions(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + dir := t.TempDir() + seedTwoSourceArchiveIn(t, dir, true) + testCtx := configureEvalRun(t, dir, + "q1 0 1\n", + "q1\trenewal\n") + + cmd := &cobra.Command{} + cmd.SetContext(testCtx) + + err := runEval(cmd, nil) + require.Error(err, "the run must stop rather than print a number it cannot justify") + assert.Contains(err.Error(), "more than one connected source") + assert.Contains(err.Error(), "--doc-key=message") +} diff --git a/cmd/msgvault/cmd/eval_stub.go b/cmd/msgvault/cmd/eval_stub.go new file mode 100644 index 000000000..4f0e35803 --- /dev/null +++ b/cmd/msgvault/cmd/eval_stub.go @@ -0,0 +1,25 @@ +//go:build !sqlite_vec + +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +// evalCmd is a stub for builds that lack the sqlite_vec build tag. The eval +// command exercises vector/hybrid retrieval, which needs the sqlite-vec +// extension; binaries from `make build` (which sets `-tags "fts5 sqlite_vec"`) +// use the real implementation in eval.go. +var evalCmd = &cobra.Command{ + Use: "eval", + Short: "Evaluate retrieval quality against relevance judgments (requires sqlite_vec build)", + RunE: func(_ *cobra.Command, _ []string) error { + return fmt.Errorf("eval requires sqlite-vec support; rebuild with `go build -tags \"fts5 sqlite_vec\"`") + }, +} + +func init() { + rootCmd.AddCommand(evalCmd) +} diff --git a/cmd/msgvault/cmd/eval_topics_test.go b/cmd/msgvault/cmd/eval_topics_test.go new file mode 100644 index 000000000..001c195b3 --- /dev/null +++ b/cmd/msgvault/cmd/eval_topics_test.go @@ -0,0 +1,194 @@ +//go:build fts5 && sqlite_vec + +package cmd + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/msgvault/internal/eval" +) + +// evalTopicReport is the slice of the JSON report the topic-handling tests +// read: how many topics were scored, what the scored ones came to, and what the +// run said about the ones it did not score. +type evalTopicReport struct { + TopicsEvaluated int `json:"topics_evaluated"` + Results map[string]struct { + MRR float64 `json:"MRR@10"` + Topics int `json:"topics"` + } `json:"results"` + Diagnostics struct { + SkippedCells []string `json:"skipped_cells"` + UnjudgedTopics []string `json:"unjudged_topics"` + } `json:"diagnostics"` +} + +// runEvalForReport drives the configured eval run and decodes its JSON report. +func runEvalForReport(ctx context.Context, t *testing.T, out *evalTopicReport) { + t.Helper() + cmd := &cobra.Command{} + cmd.SetContext(ctx) + + done := captureStdout(t) + err := runEval(cmd, nil) + text := done() + require.NoError(t, err, "eval run") + require.NoError(t, json.Unmarshal([]byte(text), out), "parse report: %s", text) +} + +// TestRunEval_SkipsATopicThatParsedToAnEmptyQuery is the regression for a topic +// that is non-empty text, parses without error, and still carries no search +// criteria. +// +// `subject:""` is the plain case: the parser drops an empty operator value +// rather than building a `LIKE '%%'` that matches everything, so Query.Err() is +// nil and Query.IsEmpty() is true. Scoring it does not measure retrieval at +// all — the fts path answers an empty query by listing the whole live corpus, +// newest first — so the topic collects whatever the archive's date +// distribution hands it. Here that is a spurious MRR of 0.5 for q2: its judged +// message is the archive's oldest, so the corpus listing puts it second, and +// the run reported a headline 0.75 over "two" topics. +// +// The honest answer is one topic at MRR 1.0, with q2 reported as unscorable. +func TestRunEval_SkipsATopicThatParsedToAnEmptyQuery(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + dir := t.TempDir() + seedRankingDivergenceArchiveIn(t, dir) + testCtx := configureEvalRun(t, dir, + "q1 0 1\n"+ + "q2 0 1\n", + "q1\trenewal\nq2\tsubject:\"\"\n") + + var report evalTopicReport + runEvalForReport(testCtx, t, &report) + scored := report.Results[evalTestMode] + + assert.Equal(1, report.TopicsEvaluated, "the criteria-less topic is not a measurement") + assert.Equal(1, scored.Topics) + assert.InDelta(1.0, scored.MRR, 1e-9, + "only q1 is scored; folding in the full-corpus scan reported 0.75") + + require.Len(report.Diagnostics.SkippedCells, 1, "the skip must be reported, not silent") + assert.Contains(report.Diagnostics.SkippedCells[0], "topic q2") + assert.Contains(report.Diagnostics.SkippedCells[0], "no search criteria") +} + +// TestRunEval_ReportsTopicsWithNoMatchingJudgments pins the coverage half of +// the qrels join. Not scoring an unjudged topic is right — there is nothing to +// score it against — but one judged topic is enough for the command to print a +// headline number, so a qrels file that matches a fraction of the topics file +// reports a mean over a small, self-selected subset and looks like a complete +// run. Here three of four topics go unjudged and the survivor scores a perfect +// 1.0; the run has to say which three it dropped and out of how many. +func TestRunEval_ReportsTopicsWithNoMatchingJudgments(t *testing.T) { + assert := assert.New(t) + + dir := t.TempDir() + seedRankingDivergenceArchiveIn(t, dir) + testCtx := configureEvalRun(t, dir, + "q1 0 1\n", + "q1\trenewal\nq2\trenewal\nq3\trenewal\nq4\trenewal\n") + + var report evalTopicReport + runEvalForReport(testCtx, t, &report) + + assert.Equal(1, report.TopicsEvaluated) + assert.Equal([]string{"q2", "q3", "q4"}, report.Diagnostics.UnjudgedTopics, + "the unscored topics are named, in topics-file order") + assert.Empty(report.Diagnostics.SkippedCells, + "an unjudged topic is not a skip: nothing went wrong with it") + + // The same fact has to reach the table output, which renders notes() and + // never sees the JSON block. + note := findEvalNote(t, report.Diagnostics.UnjudgedTopics, report.TopicsEvaluated, 4) + assert.Contains(note, "3 of 4 topics had no matching qrels entry") + assert.Contains(note, "q2, q3, q4") + assert.Contains(note, "cover 1 of the topics file") +} + +// TestRunEval_UnjudgedCoverageNoteCountsWhatWasActuallyScored is the +// regression for conflating "judged" with "scored". A judged topic that +// parses to no search criteria is skipped, same as an unjudged one, so +// subtracting only the unjudged count from the topics-file total overstates +// what the headline numbers cover. Here q1 scores, q2 is judged but +// criteria-less, q3 is unjudged: TopicsEvaluated is 1, not +// Parsed-len(unjudged) = 3, and the note has to say 1, matching the same +// number the JSON report's own topics_evaluated is built from. +func TestRunEval_UnjudgedCoverageNoteCountsWhatWasActuallyScored(t *testing.T) { + assert := assert.New(t) + + dir := t.TempDir() + seedRankingDivergenceArchiveIn(t, dir) + testCtx := configureEvalRun(t, dir, + "q1 0 1\n"+ + "q2 0 1\n", + "q1\trenewal\nq2\tsubject:\"\"\nq3\trenewal\n") + + var report evalTopicReport + runEvalForReport(testCtx, t, &report) + + assert.Equal(1, report.TopicsEvaluated, "only q1 scores; q2 is judged but criteria-less") + assert.Equal([]string{"q3"}, report.Diagnostics.UnjudgedTopics) + + note := findEvalNote(t, report.Diagnostics.UnjudgedTopics, report.TopicsEvaluated, 3) + assert.Contains(note, "1 of 3 topics had no matching qrels entry") + assert.Contains(note, "cover 1 of the topics file", + "not 2 (Parsed-unjudged): q2 was judged but never scored") +} + +// TestRunEval_CountsEachTopicOnceForARepeatedMode is the end-to-end regression +// for a duplicated --modes value. The aggregates are keyed by mode name, so +// `--modes fts,fts` added every topic's score to the fts aggregate twice: the +// report claimed two topics for a two-topic run over one mode, over a run that +// had also issued every query twice. The count beside the means is the +// denominator a reader compares two runs by, so it has to be the number of +// topics, not the number of times the list mentioned the mode. +func TestRunEval_CountsEachTopicOnceForARepeatedMode(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + dir := t.TempDir() + seedRankingDivergenceArchiveIn(t, dir) + testCtx := configureEvalRun(t, dir, + "q1 0 1\nq2 0 1\n", + "q1\trenewal\nq2\trenewal\n") + // configureEvalRun snapshots and restores every eval flag, so overriding + // one after it is safe. + evalModes = evalTestMode + "," + evalTestMode + + var report evalTopicReport + runEvalForReport(testCtx, t, &report) + + require.Len(report.Results, 1, "a repeated mode is still one mode") + assert.Equal(2, report.TopicsEvaluated) + assert.Equal(2, report.Results[evalTestMode].Topics, + "two topics scored once each; counting the mode twice reported 4") +} + +// findEvalNote re-renders the diagnostics the table output would print and +// returns the line covering the given unjudged topics. scored is the number +// of topics the run actually scored and parsed is the topics file's total, +// both matching runEval's own diag.scored assignment and TopicsLoad.Parsed — +// a caller passing report.TopicsEvaluated and the real topics count keeps +// the synthetic diagnostics consistent with the run it is standing in for. +func findEvalNote(t *testing.T, unjudged []string, scored, parsed int) string { + t.Helper() + diag := &runDiagnostics{UnjudgedTopics: unjudged, scored: scored} + diag.TopicsLoad = eval.LoadStats{Path: "topics.tsv", Lines: parsed, Parsed: parsed} + for _, n := range diag.notes() { + if strings.Contains(n, "no matching qrels entry") { + return n + } + } + t.Fatalf("no coverage note in %v", diag.notes()) + return "" +} diff --git a/cmd/msgvault/cmd/eval_vector_config_test.go b/cmd/msgvault/cmd/eval_vector_config_test.go new file mode 100644 index 000000000..5b9e8e22b --- /dev/null +++ b/cmd/msgvault/cmd/eval_vector_config_test.go @@ -0,0 +1,193 @@ +//go:build fts5 && sqlite_vec + +package cmd + +import ( + "context" + "database/sql" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/vector" + "go.kenn.io/msgvault/internal/vector/hybrid" + "go.kenn.io/msgvault/internal/vector/sqlitevec" +) + +// evalVectorConfig builds a config whose vector section is valid apart from +// the embeddings fields the caller overrides, pointed at a scratch data dir. +func evalVectorConfig(t *testing.T, format vector.EmbeddingAPIFormat, model string) *config.Config { + t.Helper() + c := &config.Config{} + c.Data.DataDir = t.TempDir() + c.Vector.Enabled = true + c.Vector.Embeddings.Endpoint = "http://127.0.0.1:1/v1" + c.Vector.Embeddings.Model = model + c.Vector.Embeddings.APIFormat = format + c.Vector.Embeddings.Dimension = 1024 + c.Vector.ApplyDefaults() + return c +} + +// TestAttachVector_RejectsUnusableEmbeddingConfig pins the eval command's +// fail-fast contract on the resolved vector config. An api_format this binary +// cannot build a client for, and a contextual format paired with a model the +// contextual endpoint does not serve, must both stop the run with an error +// naming the offending value — before any index is opened, so a bad config can +// never be scored as poor retrieval. +func TestAttachVector_RejectsUnusableEmbeddingConfig(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + f, _, _ := setupScopeFixture(t) + + c := evalVectorConfig(t, "voyage", "voyage-context-4") + testCtx := withTestConfig(t, c) + _ = testCtx + + ev := &evaluator{ctx: testCtx} + cleanup, err := ev.attachVector(testCtx, f.Store) + require.Error(err, "an unsupported api_format must not fall back to the OpenAI-compatible client") + assert.Nil(cleanup) + assert.Contains(err.Error(), "api_format") + assert.Contains(err.Error(), `"voyage"`) + + _, statErr := os.Stat(filepath.Join(c.Data.DataDir, "vectors.db")) + assert.True(os.IsNotExist(statErr), "the config check must run before the index is opened") +} + +// seedActiveGeneration activates an empty generation carrying the config's +// own fingerprint, so attachVector gets past the active-generation check and +// on to the part under test. +func seedActiveGeneration(t *testing.T, dataDir, mainPath string, mainDB *sql.DB, vecCfg vector.Config) { + t.Helper() + ctx := context.Background() + require.NoError(t, sqlitevec.RegisterExtension(), "RegisterExtension") + b, err := sqlitevec.Open(ctx, sqlitevec.Options{ + Path: filepath.Join(dataDir, "vectors.db"), + MainPath: mainPath, + Dimension: vecCfg.Embeddings.Dimension, + MainDB: mainDB, + }) + require.NoError(t, err, "open vectors.db") + defer func() { require.NoError(t, b.Close(), "close vectors.db") }() + + gen, err := b.CreateGeneration(ctx, + vecCfg.Embeddings.Model, vecCfg.Embeddings.Dimension, vecCfg.GenerationFingerprint()) + require.NoError(t, err, "CreateGeneration") + require.NoError(t, b.ActivateGeneration(ctx, gen, true), "ActivateGeneration") +} + +// TestAttachVector_EmbedsQueriesThroughConfiguredAPIFormat is the end-to-end +// binding: run the eval command's own vector setup against a config that says +// api_format = "voyage-contextual", then embed a query through the engine it +// wired and watch what goes over the wire. Before this fix the request landed +// on /v1/embeddings as a flat OpenAI-compatible body, so a contextual index was +// scored with query vectors from a different endpoint and a different role. +func TestAttachVector_EmbedsQueriesThroughConfiguredAPIFormat(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + dataDir := t.TempDir() + s := seedRankingDivergenceArchiveIn(t, dataDir) + rec, endpoint := embedTestServer(t, `{"data":[{"index":0,"data":[{"index":0,"embedding":[0.25,0.5,0.75]}]}]}`) + + c := evalVectorConfig(t, vector.APIFormatVoyageContextual, "voyage-context-4") + c.Data.DataDir = dataDir + c.Vector.Embeddings.Endpoint = endpoint + c.Vector.Embeddings.Dimension = 3 + ctx := withTestConfig(t, c) + seedActiveGeneration(t, dataDir, c.DatabaseDSN(), s.DB(), c.Vector) + + ev := &evaluator{ctx: ctx, diag: &runDiagnostics{}} + cleanup, err := ev.attachVector(ctx, s) + require.NoError(err, "attachVector") + defer cleanup() + + vec, err := ev.heng.EmbedQuery(ctx, "lease renewal") + require.NoError(err, "EmbedQuery") + assert.Equal([]float32{0.25, 0.5, 0.75}, vec) + + path, body := rec.seen() + assert.Equal("/v1/contextualizedembeddings", path, "queries must go to the contextual endpoint") + assert.Equal("query", body["input_type"], "queries must carry the query role, not the document role") + assert.Equal("voyage-contextual", ev.prov.APIFormat, "the run reports the format that produced its scores") +} + +// TestAttachVector_RejectsContextualModelMismatch covers the other resolved +// config the eval tool must refuse: api_format = "voyage-contextual" with a +// non-contextual model. +func TestAttachVector_RejectsContextualModelMismatch(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + f, _, _ := setupScopeFixture(t) + + testCtx := withTestConfig(t, evalVectorConfig(t, vector.APIFormatVoyageContextual, "voyage-large-4")) + _ = testCtx + + ev := &evaluator{ctx: testCtx} + _, err := ev.attachVector(testCtx, f.Store) + require.Error(err) + assert.Contains(err.Error(), "voyage-large-4") +} + +func TestAttachVector_HonorsSQLiteAcceleratorMode(t *testing.T) { + req := require.New(t) + dataDir := t.TempDir() + s := seedRankingDivergenceArchiveIn(t, dataDir) + _, endpoint := embedTestServer(t, `{"data":[{"index":0,"embedding":[1,0,0,0]}]}`) + c := evalVectorConfig(t, vector.APIFormatOpenAI, "test-model") + c.Data.DataDir = dataDir + c.Vector.Embeddings.Endpoint = endpoint + c.Vector.Embeddings.Dimension = 4 + testCtx := withTestConfig(t, c) + _ = testCtx + ctx := testCtx + seedActiveGeneration(t, dataDir, c.DatabaseDSN(), s.DB(), c.Vector) + + vectorPath := filepath.Join(dataDir, "vectors.db") + backend, err := sqlitevec.Open(ctx, sqlitevec.Options{Path: vectorPath, Dimension: 4}) + req.NoError(err) + t.Cleanup(func() { _ = backend.Close() }) + gen, err := backend.ActiveGeneration(ctx) + req.NoError(err) + // Training needs 512 vectors; distinct chunks keep the archive fixture small. + chunks := make([]vector.Chunk, 512) + for i := range chunks { + chunks[i] = vector.Chunk{ + MessageID: 1, ChunkIndex: i, Vector: []float32{float32(i % 17), 1, 2, 3}, + } + } + req.NoError(backend.Upsert(ctx, gen.ID, chunks)) + plan, err := backend.PrepareAccelerator(ctx, gen.ID, sqlitevec.OptimizeOptions{Threads: 1}) + req.NoError(err) + req.True(plan.Applicable, plan.Reason) + req.NoError(sqlitevec.RunAcceleratorWorker(ctx, vectorPath, gen.ID, 1)) + _, err = backend.PublishAccelerator(ctx, gen.ID) + req.NoError(err) + req.NoError(backend.Close()) + + for _, tc := range []struct { + mode, wantPath string + }{ + {"auto", "vec1_ivf_opq"}, + {"exact", "exact"}, + } { + t.Run(tc.mode, func(t *testing.T) { + require := require.New(t) + c.Vector.Search.SQLiteAccelerator = tc.mode + ev := &evaluator{ctx: ctx, diag: &runDiagnostics{}} + cleanup, err := ev.attachVector(ctx, s) + require.NoError(err) + defer cleanup() + hits, meta, err := ev.heng.Search(ctx, hybrid.SearchRequest{ + Mode: hybrid.ModeVector, FreeText: "lease renewal", Limit: 1, + }) + require.NoError(err) + require.Len(hits, 1) + assert.Equal(t, tc.wantPath, meta.Accelerator) + }) + } +} diff --git a/cmd/msgvault/cmd/eval_vector_corpus_test.go b/cmd/msgvault/cmd/eval_vector_corpus_test.go new file mode 100644 index 000000000..28cf0a232 --- /dev/null +++ b/cmd/msgvault/cmd/eval_vector_corpus_test.go @@ -0,0 +1,230 @@ +//go:build fts5 && sqlite_vec + +package cmd + +import ( + "context" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/vector" + "go.kenn.io/msgvault/internal/vector/sqlitevec" +) + +// seedEmbeddedGeneration embeds exactly msgIDs, activates the result, and +// stamps embed_gen on those rows in the main DB the way a real embed run +// would — needed because EmbeddedMessageCount requires the stamp, not just a +// vectors.db row, to count a message as embedded. +func seedEmbeddedGeneration( + t *testing.T, dataDir, mainPath string, mainDB *store.Store, vecCfg vector.Config, msgIDs ...int64, +) { + t.Helper() + ctx := context.Background() + require.NoError(t, sqlitevec.RegisterExtension(), "RegisterExtension") + b, err := sqlitevec.Open(ctx, sqlitevec.Options{ + Path: filepath.Join(dataDir, "vectors.db"), + MainPath: mainPath, + Dimension: vecCfg.Embeddings.Dimension, + MainDB: mainDB.DB(), + }) + require.NoError(t, err, "open vectors.db") + defer func() { require.NoError(t, b.Close(), "close vectors.db") }() + + gen, err := b.CreateGeneration(ctx, + vecCfg.Embeddings.Model, vecCfg.Embeddings.Dimension, vecCfg.GenerationFingerprint()) + require.NoError(t, err, "CreateGeneration") + + dim := vecCfg.Embeddings.Dimension + chunks := make([]vector.Chunk, 0, len(msgIDs)) + for _, id := range msgIDs { + v := make([]float32, dim) + for i := range v { + v[i] = float32(id) / 10 + } + chunks = append(chunks, vector.Chunk{MessageID: id, Vector: v, SourceCharLen: 32}) + } + require.NoError(t, b.Upsert(ctx, gen, chunks), "Upsert") + require.NoError(t, b.ActivateGeneration(ctx, gen, true), "ActivateGeneration") + + for _, id := range msgIDs { + _, err = mainDB.DB().Exec(`UPDATE messages SET embed_gen = ? WHERE id = ?`, int64(gen), id) + require.NoError(t, err, "stamp embed_gen on message %d", id) + } +} + +// TestAttachVector_ReportsScopedCorpusSeparatelyFromTheArchive is the +// regression for provenance that described the archive when a run's vector +// generation only ever searches part of it. An account-scoped +// [vector.embed.scope] means retrieval can return source 1's two messages +// and never source 2's, so a report that prints the whole three-message, +// two-conversation archive as "the corpus" overstates what vector mode's +// recall is actually measured against. +func TestAttachVector_ReportsScopedCorpusSeparatelyFromTheArchive(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + dataDir := t.TempDir() + s := seedTwoSourceArchiveIn(t, dataDir, false) + + c := evalVectorConfig(t, vector.APIFormatOpenAI, "test-model") + c.Data.DataDir = dataDir + c.Vector.Embeddings.Dimension = 3 + c.Vector.Embed.Scope.SourceIDs = []int64{1} + ctx := withTestConfig(t, c) + + seedEmbeddedGeneration(t, dataDir, c.DatabaseDSN(), s, c.Vector, 1, 2) + + ev := &evaluator{ctx: ctx, diag: &runDiagnostics{}} + cleanup, err := ev.attachVector(ctx, s) + require.NoError(err, "attachVector") + defer cleanup() + ev.collectCorpusStats(s.DB()) + + assert.EqualValues(3, ev.prov.Messages, "the archive holds three live messages across both sources") + assert.EqualValues(2, ev.prov.Conversations, "and two conversations") + assert.EqualValues(2, ev.prov.VectorMessages, + "only source 1's two messages are embedded in the active generation") + assert.EqualValues(1, ev.prov.VectorConversations, + "and only source 1's one conversation") +} + +// TestAttachVector_CorpusScopeFilterExcludesOutOfScopeStampedMessages pins +// the scope-filter clause in collectVectorCorpusStats specifically, as +// opposed to the embed_gen stamp doing all the work by coincidence. Message +// 3 (source 2) is embedded and stamped into the very same generation as +// messages 1 and 2 — as if a stray embed run, or a since-narrowed scope, +// left an out-of-scope message carrying this generation's stamp — while +// [vector.embed.scope] declares only source 1. Without the source_id filter +// this would count all three messages and both conversations; with it, the +// out-of-scope stamp must not move the reported corpus at all. +func TestAttachVector_CorpusScopeFilterExcludesOutOfScopeStampedMessages(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + dataDir := t.TempDir() + s := seedTwoSourceArchiveIn(t, dataDir, false) + + c := evalVectorConfig(t, vector.APIFormatOpenAI, "test-model") + c.Data.DataDir = dataDir + c.Vector.Embeddings.Dimension = 3 + c.Vector.Embed.Scope.SourceIDs = []int64{1} + ctx := withTestConfig(t, c) + + seedEmbeddedGeneration(t, dataDir, c.DatabaseDSN(), s, c.Vector, 1, 2, 3) + + ev := &evaluator{ctx: ctx, diag: &runDiagnostics{}} + cleanup, err := ev.attachVector(ctx, s) + require.NoError(err, "attachVector") + defer cleanup() + + assert.EqualValues(2, ev.prov.VectorMessages, + "source 2's stamped-but-out-of-scope message must not count") + assert.EqualValues(1, ev.prov.VectorConversations, + "nor must its conversation") +} + +// TestAttachVector_MatchesArchiveCorpusWhenScopeCoversIt pins the other +// half: a generation with no [vector.embed.scope] restriction embeds every +// live message in the archive, so VectorMessages/VectorConversations equal +// Messages/Conversations exactly and the table print (eval.go's table +// method) has nothing narrower to add. +func TestAttachVector_MatchesArchiveCorpusWhenScopeCoversIt(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + dataDir := t.TempDir() + s := seedRankingDivergenceArchiveIn(t, dataDir) + + c := evalVectorConfig(t, vector.APIFormatOpenAI, "test-model") + c.Data.DataDir = dataDir + c.Vector.Embeddings.Dimension = 3 + ctx := withTestConfig(t, c) + + // seedRankingDivergenceArchiveIn's two live messages (1 and 2) sit in + // separate conversations; message 3 is deleted from its source and + // excluded from both the archive count and the embedded set. + seedEmbeddedGeneration(t, dataDir, c.DatabaseDSN(), s, c.Vector, 1, 2) + + ev := &evaluator{ctx: ctx, diag: &runDiagnostics{}} + cleanup, err := ev.attachVector(ctx, s) + require.NoError(err, "attachVector") + defer cleanup() + ev.collectCorpusStats(s.DB()) + + require.EqualValues(2, ev.prov.Messages) + require.EqualValues(2, ev.prov.Conversations) + assert.Equal(ev.prov.Messages, ev.prov.VectorMessages, + "an unscoped generation embeds the whole archive") + assert.Equal(ev.prov.Conversations, ev.prov.VectorConversations) +} + +// TestAttachVector_CorpusScopeNormalizesMessageTypeCase is the regression for +// reading vecCfg.Embed.Scope.MessageTypes directly instead of through +// BuildScope(): the archive stores message_type lowercase ("email"), but +// [vector.embed.scope] is user-typed TOML with no case convention enforced. +// A raw, unnormalized "EMAIL" would match nothing and silently zero the +// vector corpus even though every message is actually in scope. +func TestAttachVector_CorpusScopeNormalizesMessageTypeCase(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + dataDir := t.TempDir() + s := seedRankingDivergenceArchiveIn(t, dataDir) + + c := evalVectorConfig(t, vector.APIFormatOpenAI, "test-model") + c.Data.DataDir = dataDir + c.Vector.Embeddings.Dimension = 3 + c.Vector.Embed.Scope.MessageTypes = []string{"EMAIL"} + ctx := withTestConfig(t, c) + + seedEmbeddedGeneration(t, dataDir, c.DatabaseDSN(), s, c.Vector, 1, 2) + + ev := &evaluator{ctx: ctx, diag: &runDiagnostics{}} + cleanup, err := ev.attachVector(ctx, s) + require.NoError(err, "attachVector") + defer cleanup() + + assert.EqualValues(2, ev.prov.VectorMessages, + "an uppercase configured message type must still match the archive's lowercase rows") + assert.EqualValues(2, ev.prov.VectorConversations) +} + +// TestAttachVector_CorpusIncludesMessagesWithStaleEmbedGenStamp is the +// regression for requiring messages.embed_gen = gen in the corpus query. +// Content changes reset a message's embed_gen to mark it for re-embedding, +// but Backend.Search reads vectors.db purely by generation_id: the old +// vector stays searchable, embed_gen or not, until a re-embed run actually +// replaces it. A count that required the stamp would report a smaller +// corpus than what the run's own search can retrieve — message 1's stale +// stamp here must not remove it. +func TestAttachVector_CorpusIncludesMessagesWithStaleEmbedGenStamp(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + dataDir := t.TempDir() + s := seedRankingDivergenceArchiveIn(t, dataDir) + + c := evalVectorConfig(t, vector.APIFormatOpenAI, "test-model") + c.Data.DataDir = dataDir + c.Vector.Embeddings.Dimension = 3 + ctx := withTestConfig(t, c) + + seedEmbeddedGeneration(t, dataDir, c.DatabaseDSN(), s, c.Vector, 1, 2) + + // Simulate a content change on message 1 after it was embedded: the + // backfill machinery resets embed_gen to mark it for re-embedding, but + // its vector row in vectors.db is untouched until that re-embed runs. + _, err := s.DB().Exec(`UPDATE messages SET embed_gen = NULL WHERE id = 1`) + require.NoError(err, "reset embed_gen to simulate a pending re-embed") + + ev := &evaluator{ctx: ctx, diag: &runDiagnostics{}} + cleanup, err := ev.attachVector(ctx, s) + require.NoError(err, "attachVector") + defer cleanup() + + assert.EqualValues(2, ev.prov.VectorMessages, + "message 1's stale embed_gen must not drop it from a corpus its own stale vector is still searchable in") + assert.EqualValues(2, ev.prov.VectorConversations) +} diff --git a/cmd/msgvault/cmd/export_attachment_test.go b/cmd/msgvault/cmd/export_attachment_test.go index 143b3deb9..6a32b51d3 100644 --- a/cmd/msgvault/cmd/export_attachment_test.go +++ b/cmd/msgvault/cmd/export_attachment_test.go @@ -1,7 +1,6 @@ package cmd import ( - "context" "crypto/sha256" "encoding/base64" "encoding/json" @@ -116,6 +115,9 @@ func TestExportAttachmentBinaryDownloadPreservesExistingFileOnCloseError(t *test } func TestExportAttachmentUsesLocalDaemonHTTPAndPreservesFileOutput(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) dataDir := t.TempDir() @@ -142,7 +144,10 @@ func TestExportAttachmentUsesLocalDaemonHTTPAndPreservesFileOutput(t *testing.T) Data: config.DataConfig{DataDir: dataDir}, Remote: config.RemoteConfig{URL: "http://configured-daemonclient.invalid"}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true t.Chdir(dataDir) exportAttachmentOutput = "attachment.bin" exportAttachmentJSON = false @@ -150,7 +155,8 @@ func TestExportAttachmentUsesLocalDaemonHTTPAndPreservesFileOutput(t *testing.T) doneErr := captureStderr(t) cmd := &cobra.Command{Use: "export-attachment"} - cmd.SetContext(context.Background()) + cmd.SetContext(testCtx) + cmd.SetContext(testCtx) err := runExportAttachment(cmd, []string{contentHash}) stderr := doneErr() @@ -166,6 +172,9 @@ func TestExportAttachmentUsesLocalDaemonHTTPAndPreservesFileOutput(t *testing.T) } func TestExportAttachmentUsesLocalDaemonHTTPAndPreservesJSONOutput(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) dataDir := t.TempDir() @@ -192,14 +201,18 @@ func TestExportAttachmentUsesLocalDaemonHTTPAndPreservesJSONOutput(t *testing.T) Data: config.DataConfig{DataDir: dataDir}, Remote: config.RemoteConfig{URL: "http://configured-daemonclient.invalid"}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true exportAttachmentOutput = "" exportAttachmentJSON = true exportAttachmentBase64 = false done := captureStdout(t) cmd := &cobra.Command{Use: "export-attachment"} - cmd.SetContext(context.Background()) + cmd.SetContext(testCtx) + cmd.SetContext(testCtx) err := runExportAttachment(cmd, []string{contentHash}) out := done() @@ -218,6 +231,9 @@ func TestExportAttachmentUsesLocalDaemonHTTPAndPreservesJSONOutput(t *testing.T) } func TestExportAttachmentUsesLocalDaemonHTTPAndPreservesBase64Output(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) dataDir := t.TempDir() @@ -244,14 +260,18 @@ func TestExportAttachmentUsesLocalDaemonHTTPAndPreservesBase64Output(t *testing. Data: config.DataConfig{DataDir: dataDir}, Remote: config.RemoteConfig{URL: "http://configured-daemonclient.invalid"}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true exportAttachmentOutput = "" exportAttachmentJSON = false exportAttachmentBase64 = true done := captureStdout(t) cmd := &cobra.Command{Use: "export-attachment"} - cmd.SetContext(context.Background()) + cmd.SetContext(testCtx) + cmd.SetContext(testCtx) err := runExportAttachment(cmd, []string{contentHash}) out := done() diff --git a/cmd/msgvault/cmd/export_attachments_test.go b/cmd/msgvault/cmd/export_attachments_test.go index 087c3ab8b..12b4fcd33 100644 --- a/cmd/msgvault/cmd/export_attachments_test.go +++ b/cmd/msgvault/cmd/export_attachments_test.go @@ -31,7 +31,15 @@ func TestExportAttachmentsCmd_Registration(t *testing.T) { assert.Equal("o", f.Shorthand, "output shorthand") } -func setupExportAttachmentsHTTPTest(t *testing.T) ([]byte, []byte, *atomic.Int32, *atomic.Int32) { +type exportAttachmentsHTTPTestFixture struct { + reportData []byte + photoData []byte + messageRequests *atomic.Int32 + attachmentRequests *atomic.Int32 + ctx context.Context +} + +func setupExportAttachmentsHTTPTest(t *testing.T) exportAttachmentsHTTPTestFixture { t.Helper() dataDir := t.TempDir() reportData := []byte("PDF content here") @@ -46,24 +54,24 @@ func setupExportAttachmentsHTTPTest(t *testing.T) ([]byte, []byte, *atomic.Int32 photoData, ) writeStatsHTTPDaemonRuntime(t, dataDir, server) - configureExportAttachmentsDaemonTest(t, dataDir) - return reportData, photoData, messageRequests, attachmentRequests + ctx := configureExportAttachmentsDaemonTest(t, dataDir) + return exportAttachmentsHTTPTestFixture{ + reportData: reportData, + photoData: photoData, + messageRequests: messageRequests, + attachmentRequests: attachmentRequests, + ctx: ctx, + } } -func configureExportAttachmentsDaemonTest(t *testing.T, dataDir string) { +func configureExportAttachmentsDaemonTest(t *testing.T, dataDir string) context.Context { t.Helper() - oldCfg := cfg - oldUseLocal := useLocal - cfg = &config.Config{ + cfg := &config.Config{ HomeDir: dataDir, Data: config.DataConfig{DataDir: dataDir}, Remote: config.RemoteConfig{URL: "http://configured-daemonclient.invalid"}, } - useLocal = true - t.Cleanup(func() { - cfg = oldCfg - useLocal = oldUseLocal - }) + return testInvocationContext(t.Context(), cfg, invocationOptions{useLocal: true}) } func TestResolveExportAttachmentsOutputDir_CreatesMissingDir(t *testing.T) { @@ -103,14 +111,14 @@ func TestResolveExportAttachmentsOutputDir_RejectsFilePath(t *testing.T) { func TestExportAttachments_FullFlow(t *testing.T) { require := require.New(t) assert := assert.New(t) - setupExportAttachmentsHTTPTest(t) + fixture := setupExportAttachmentsHTTPTest(t) outputDir := t.TempDir() exportAttachmentsOutput = outputDir defer func() { exportAttachmentsOutput = "" }() c := exportAttachmentsCmd - c.SetContext(context.Background()) + c.SetContext(fixture.ctx) require.NoError(runExportAttachments(c, []string{"1"}), "runExportAttachments") // Verify both files were exported @@ -129,7 +137,7 @@ func TestExportAttachmentsUsesLocalDaemonHTTPAndPreservesDirectoryOutput(t *test require := require.New(t) assert := assert.New(t) outputDir := t.TempDir() - reportData, photoData, messageRequests, attachmentRequests := setupExportAttachmentsHTTPTest(t) + fixture := setupExportAttachmentsHTTPTest(t) oldOutput := exportAttachmentsOutput defer func() { @@ -139,7 +147,7 @@ func TestExportAttachmentsUsesLocalDaemonHTTPAndPreservesDirectoryOutput(t *test doneErr := captureStderr(t) cmd := exportAttachmentsCmd - cmd.SetContext(context.Background()) + cmd.SetContext(fixture.ctx) err := runExportAttachments(cmd, []string{"gmail_abc123"}) stderr := doneErr() @@ -149,10 +157,10 @@ func TestExportAttachmentsUsesLocalDaemonHTTPAndPreservesDirectoryOutput(t *test require.NoError(err, "read report") photoOut, err := os.ReadFile(filepath.Join(outputDir, "photo.jpg")) require.NoError(err, "read photo") - assert.Equal(reportData, reportOut, "report data") - assert.Equal(photoData, photoOut, "photo data") - assert.Equal(1, int(messageRequests.Load()), "message endpoint calls") - assert.Equal(2, int(attachmentRequests.Load()), "attachment endpoint calls") + assert.Equal(fixture.reportData, reportOut, "report data") + assert.Equal(fixture.photoData, photoOut, "photo data") + assert.Equal(1, int(fixture.messageRequests.Load()), "message endpoint calls") + assert.Equal(2, int(fixture.attachmentRequests.Load()), "attachment endpoint calls") assert.Contains(stderr, " report.pdf (", "report stderr") assert.Contains(stderr, " photo.jpg (", "photo stderr") assert.Contains(stderr, "Exported 2 attachment(s)", "summary") @@ -160,7 +168,7 @@ func TestExportAttachmentsUsesLocalDaemonHTTPAndPreservesDirectoryOutput(t *test } func TestExportAttachments_GmailIDFallback(t *testing.T) { - setupExportAttachmentsHTTPTest(t) + fixture := setupExportAttachmentsHTTPTest(t) outputDir := t.TempDir() exportAttachmentsOutput = outputDir @@ -168,7 +176,7 @@ func TestExportAttachments_GmailIDFallback(t *testing.T) { // Use Gmail source ID instead of numeric ID cmd := exportAttachmentsCmd - cmd.SetContext(context.Background()) + cmd.SetContext(fixture.ctx) require.NoError(t, runExportAttachments(cmd, []string{"gmail_abc123"}), "runExportAttachments with Gmail ID") entries, _ := os.ReadDir(outputDir) @@ -176,10 +184,10 @@ func TestExportAttachments_GmailIDFallback(t *testing.T) { } func TestExportAttachments_MessageNotFound(t *testing.T) { - setupExportAttachmentsHTTPTest(t) + fixture := setupExportAttachmentsHTTPTest(t) cmd := exportAttachmentsCmd - cmd.SetContext(context.Background()) + cmd.SetContext(fixture.ctx) err := runExportAttachments(cmd, []string{"99999"}) require.Error(t, err, "expected error for nonexistent message") assert.ErrorContains(t, err, "message not found") @@ -188,7 +196,7 @@ func TestExportAttachments_MessageNotFound(t *testing.T) { func TestExportAttachments_OutputDirAutoCreated(t *testing.T) { require := require.New(t) assert := assert.New(t) - setupExportAttachmentsHTTPTest(t) + fixture := setupExportAttachmentsHTTPTest(t) // Point to a non-existent nested directory; it should be created like the // sibling exporters create the file/path they are asked to write to. @@ -197,7 +205,7 @@ func TestExportAttachments_OutputDirAutoCreated(t *testing.T) { defer func() { exportAttachmentsOutput = "" }() cmd := exportAttachmentsCmd - cmd.SetContext(context.Background()) + cmd.SetContext(fixture.ctx) require.NoError(runExportAttachments(cmd, []string{"1"}), "runExportAttachments") entries, err := os.ReadDir(outputDir) @@ -206,7 +214,7 @@ func TestExportAttachments_OutputDirAutoCreated(t *testing.T) { } func TestExportAttachments_NotADirectory(t *testing.T) { - setupExportAttachmentsHTTPTest(t) + fixture := setupExportAttachmentsHTTPTest(t) // Point to a file, not a directory tmpFile := filepath.Join(t.TempDir(), "afile.txt") @@ -215,7 +223,7 @@ func TestExportAttachments_NotADirectory(t *testing.T) { defer func() { exportAttachmentsOutput = "" }() cmd := exportAttachmentsCmd - cmd.SetContext(context.Background()) + cmd.SetContext(fixture.ctx) err := runExportAttachments(cmd, []string{"1"}) require.Error(t, err, "expected error for file as output dir") assert.ErrorContains(t, err, "not a directory") diff --git a/cmd/msgvault/cmd/export_discord.go b/cmd/msgvault/cmd/export_discord.go index bf62158a6..42845030a 100644 --- a/cmd/msgvault/cmd/export_discord.go +++ b/cmd/msgvault/cmd/export_discord.go @@ -54,6 +54,9 @@ func newExportDiscordLocalCmd(deps discordCommandDeps) *cobra.Command { Short: "Export bounded Discord history from the local archive", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + if invocationFromContext(cmd.Context()) != nil && deps.bind != nil { + deps = deps.bind(cmd.Context()) + } return runExportDiscord(cmd, deps, args[0], opts) }, } diff --git a/cmd/msgvault/cmd/export_eml.go b/cmd/msgvault/cmd/export_eml.go index 2a4881db2..6ea8d63c3 100644 --- a/cmd/msgvault/cmd/export_eml.go +++ b/cmd/msgvault/cmd/export_eml.go @@ -4,10 +4,15 @@ import ( "errors" "fmt" "path/filepath" + "strconv" "strings" + "time" "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/daemonclient" + "go.kenn.io/msgvault/internal/export" "go.kenn.io/msgvault/internal/fileutil" + "go.kenn.io/msgvault/internal/query" "go.kenn.io/msgvault/internal/store" ) @@ -17,7 +22,9 @@ const ( ) var ( - exportEMLOutput string + exportEMLOutput string + exportEMLThread bool + exportEMLAccount string ) var exportEMLCmd = &cobra.Command{ @@ -28,27 +35,30 @@ var exportEMLCmd = &cobra.Command{ This command retrieves the raw MIME data stored during sync and writes it to a file. The .eml format is compatible with most email clients. +With --thread, it writes every message in the conversation that has stored +MIME into a directory, numbered oldest first, and reports messages it had to +skip and when the account last synced. + Examples: msgvault export-eml 12345 msgvault export-eml 12345 --output message.eml - msgvault export-eml 18f0abc123def -o important.eml`, + msgvault export-eml 18f0abc123def -o important.eml + msgvault export-eml 18f0abc123def --thread -o thread/`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { id, err := resolveMessageIDArg(args[0]) if err != nil { return err } + if exportEMLThread { + return runExportEMLThread(cmd, id, exportEMLAccount, exportEMLOutput) + } return runExportEML(cmd, id, exportEMLOutput) }, } func sanitizeEMLFilename(sourceMessageID string) string { - safe := strings.Map(func(r rune) rune { - if r == '/' || r == '\\' || r == '\x00' { - return '_' - } - return r - }, sourceMessageID) + safe := export.SanitizeFilename(strings.ReplaceAll(sourceMessageID, "\x00", "_")) // Ensure the result is a plain filename with no directory // components, guarding against IMAP mailbox names with // path separators or traversal sequences. @@ -102,7 +112,90 @@ func writeExportedEML(cmd *cobra.Command, sourceMessageID, outputPath string, ra return nil } +// runExportEMLThread writes each archived message of messageRef's +// conversation that has stored MIME to outputDir as -.eml, +// numbered oldest first. Numeric references try an internal message ID first, +// then a provider message ID if no internal message exists. +func runExportEMLThread(cmd *cobra.Command, messageRef, account, outputDir string) error { + if outputDir == stdoutSentinel { + return errors.New("--thread writes one file per message; pass a directory with -o") + } + if outputDir == "" { + outputDir = "." + } + s, _, err := OpenHTTPStore(cmd.Context()) + if err != nil { + return fmt.Errorf("open store: %w", err) + } + defer func() { _ = s.Close() }() + engine := daemonclient.NewEngineAdapter(s) + + ref := query.MessageRef{Account: account} + if id, parseErr := strconv.ParseInt(messageRef, 10, 64); parseErr == nil && id > 0 { + ref.ID = id + } else { + ref.SourceMessageID = messageRef + } + + header, err := engine.ListThread(cmd.Context(), query.ThreadQuery{MessageRef: ref, All: true}) + if ref.ID != 0 && errors.Is(err, store.ErrMessageNotFound) { + ref.ID, ref.SourceMessageID = 0, messageRef + header, err = engine.ListThread(cmd.Context(), query.ThreadQuery{MessageRef: ref, All: true}) + } + if errors.Is(err, store.ErrMessageNotFound) { + return fmt.Errorf("message not found: %s", messageRef) + } + if errors.Is(err, query.ErrAmbiguousReference) { + return fmt.Errorf("list thread: %w; pass --account", err) + } + if err != nil { + return fmt.Errorf("list thread: %w", err) + } + messages := header.Messages + + if err := fileutil.SecureMkdirAll(outputDir, 0o700); err != nil { + return fmt.Errorf("create output directory: %w", err) + } + width := len(strconv.Itoa(len(messages))) + exported := 0 + for i, msg := range messages { + if !msg.HasRaw { + cmd.Printf("Skipped %s: no original MIME stored\n", msg.SourceMessageID) + continue + } + original, err := engine.ReadOriginalMessage(cmd.Context(), query.MessageRef{ID: msg.ID}, 0) + if errors.Is(err, store.ErrMessageNotFound) { + cmd.Printf("Skipped %s: message no longer available\n", msg.SourceMessageID) + continue + } + if errors.Is(err, query.ErrOriginalMIMEUnavailable) { + cmd.Printf("Skipped %s: no original MIME stored\n", msg.SourceMessageID) + continue + } + if err != nil { + return fmt.Errorf("export message %d: %w", msg.ID, err) + } + name := fmt.Sprintf("%0*d-%s", width, i+1, sanitizeEMLFilename(msg.SourceMessageID)) + if err := fileutil.SecureWriteFile(filepath.Join(outputDir, name), original.MIME, emlFileMode); err != nil { + return fmt.Errorf("write file: %w", err) + } + exported++ + } + + cmd.Printf("Exported %d of %d messages in thread %s to %s\n", + exported, len(messages), header.SourceConversationID, outputDir) + if header.LastSyncAt != nil { + cmd.Printf("%s last synced %s; newer replies may not be archived yet\n", + header.Account, header.LastSyncAt.Format(time.RFC3339)) + } else { + cmd.Printf("%s has never completed a sync; newer replies may not be archived yet\n", header.Account) + } + return nil +} + func init() { rootCmd.AddCommand(exportEMLCmd) - exportEMLCmd.Flags().StringVarP(&exportEMLOutput, "output", "o", "", "Output file path (default: .eml, use - for stdout)") + exportEMLCmd.Flags().StringVarP(&exportEMLOutput, "output", "o", "", "Output file path (default: .eml, use - for stdout); with --thread, the output directory (default: current directory)") + exportEMLCmd.Flags().BoolVar(&exportEMLThread, "thread", false, "Export every message in the conversation that has stored MIME") + exportEMLCmd.Flags().StringVar(&exportEMLAccount, "account", "", "With --thread, the account (email address) that holds a provider message ID found in several accounts") } diff --git a/cmd/msgvault/cmd/export_eml_test.go b/cmd/msgvault/cmd/export_eml_test.go index 9daaeeb61..2132bf1af 100644 --- a/cmd/msgvault/cmd/export_eml_test.go +++ b/cmd/msgvault/cmd/export_eml_test.go @@ -2,23 +2,35 @@ package cmd import ( "bytes" - "context" + "database/sql" + "fmt" + "log/slog" "net/http" "net/http/httptest" "os" "path/filepath" "strconv" + "strings" + "sync" "sync/atomic" "testing" + "time" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.kenn.io/kit/daemon" + "go.kenn.io/msgvault/internal/api" "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/query" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil" ) func TestExportEMLUsesLocalDaemonHTTPAndPreservesFileOutput(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) dataDir := t.TempDir() @@ -37,12 +49,16 @@ func TestExportEMLUsesLocalDaemonHTTPAndPreservesFileOutput(t *testing.T) { HomeDir: dataDir, Data: config.DataConfig{DataDir: dataDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true outputPath := filepath.Join(dataDir, "message.eml") var out bytes.Buffer cmd := &cobra.Command{Use: "export-eml"} - cmd.SetContext(context.Background()) + cmd.SetContext(testCtx) + cmd.SetContext(testCtx) cmd.SetOut(&out) err := runExportEML(cmd, "gmail-raw", outputPath) @@ -57,6 +73,9 @@ func TestExportEMLUsesLocalDaemonHTTPAndPreservesFileOutput(t *testing.T) { } func TestExportEMLHTTPNotFoundPreservesCLIError(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) dataDir := t.TempDir() @@ -74,11 +93,15 @@ func TestExportEMLHTTPNotFoundPreservesCLIError(t *testing.T) { HomeDir: dataDir, Data: config.DataConfig{DataDir: dataDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true var out bytes.Buffer cmd := &cobra.Command{Use: "export-eml"} - cmd.SetContext(context.Background()) + cmd.SetContext(testCtx) + cmd.SetContext(testCtx) cmd.SetOut(&out) err := runExportEML(cmd, "missing", filepath.Join(dataDir, "missing.eml")) @@ -99,14 +122,14 @@ func TestWriteExportedEMLDefaultsToSourceMessageIDFilename(t *testing.T) { cmd := &cobra.Command{Use: "export-eml"} cmd.SetOut(&out) - err := writeExportedEML(cmd, "gmail-raw", "", raw) + err := writeExportedEML(cmd, "INBOX|\x00/42", "", raw) require.NoError(err) - outputPath := filepath.Join(outputDir, "gmail-raw.eml") + outputPath := filepath.Join(outputDir, "INBOX___42.eml") got, err := os.ReadFile(outputPath) require.NoError(err) assert.Equal(raw, got) - assert.Contains(out.String(), "Exported message to: gmail-raw.eml") + assert.Contains(out.String(), "Exported message to: INBOX___42.eml") } func TestWriteExportedEMLWritesRawBytesToStdout(t *testing.T) { @@ -162,3 +185,221 @@ func emlHTTPNotFoundDaemon(t *testing.T) *httptest.Server { t.Cleanup(server.Close) return server } + +// TestExportEMLThreadWritesEveryOriginalInOrder runs against the real API +// server with an IMAP-style source: no provider thread IDs, so the +// conversation key is the root Message-ID msgvault derived from References. +func TestExportEMLThreadWritesEveryOriginalInOrder(t *testing.T) { + must := require.New(t) + checks := assert.New(t) + st := testutil.NewTestStore(t) + + src, err := st.GetOrCreateSource("imap", "owner@example.com") + must.NoError(err) + convID, err := st.EnsureConversation(src.ID, "", "Quarterly report") + must.NoError(err) + base := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + persist := func(sourceMessageID string, sentAt time.Time, raw []byte) { + _, err := st.PersistMessage(&store.MessagePersistData{ + Message: &store.Message{ + SourceID: src.ID, ConversationID: convID, SourceMessageID: sourceMessageID, + MessageType: "email", SentAt: sql.NullTime{Time: sentAt, Valid: true}, + }, + RawMIME: raw, + }) + must.NoError(err) + } + reply := []byte("Message-ID: \r\nReferences: \r\n\r\nreply \xe9\n") + root := []byte("Message-ID: \r\n\r\nroot\r\n") + persist("INBOX|2", base.Add(time.Hour), reply) + persist("INBOX|1", base, root) + persist("INBOX|3", base.Add(2*time.Hour), nil) + + cmd, out, dataDir := emlArchiveCommand(t, st, nil) + outDir := filepath.Join(dataDir, "thread") + must.NoError(runExportEMLThread(cmd, "INBOX|2", "", outDir)) + + entries, err := os.ReadDir(outDir) + must.NoError(err) + names := make([]string, 0, len(entries)) + for _, entry := range entries { + names = append(names, entry.Name()) + } + checks.Equal([]string{"1-INBOX_1.eml", "2-INBOX_2.eml"}, names) + got, err := os.ReadFile(filepath.Join(outDir, "1-INBOX_1.eml")) + must.NoError(err) + checks.Equal(root, got) + got, err = os.ReadFile(filepath.Join(outDir, "2-INBOX_2.eml")) + must.NoError(err) + checks.Equal(reply, got) + checks.Contains(out.String(), "Skipped INBOX|3: no original MIME stored") + checks.Contains(out.String(), "Exported 2 of 3 messages") + checks.Contains(out.String(), "owner@example.com has never completed a sync") + + err = runExportEMLThread(cmd, "INBOX|2", "", "-") + checks.ErrorContains(err, "--thread writes one file per message") +} + +// emlArchiveCommand uses the real daemon handlers and a private runtime record. +// beforeRequest lets tests coordinate actual store mutations between reads. +func emlArchiveCommand(t *testing.T, st *store.Store, beforeRequest func(*http.Request)) (*cobra.Command, *bytes.Buffer, string) { + t.Helper() + dataDir := t.TempDir() + engine := query.NewEngine(st.DB(), st.IsPostgreSQL()) + t.Cleanup(func() { _ = engine.Close() }) + router := api.NewServerWithOptions(api.ServerOptions{ + Config: &config.Config{}, Store: st, Engine: engine, Logger: slog.New(slog.DiscardHandler), + }).Router() + mux := http.NewServeMux() + mux.Handle("/api/ping", daemon.NewPingHandler(daemon.PingHandlerOptions{Service: daemonService, Version: Version})) + mux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if beforeRequest != nil { + beforeRequest(r) + } + router.ServeHTTP(w, r) + })) + server := httptest.NewServer(mux) + t.Cleanup(server.Close) + writeStatsHTTPDaemonRuntime(t, dataDir, server) + cfg := &config.Config{HomeDir: dataDir, Data: config.DataConfig{DataDir: dataDir}} + cmd := &cobra.Command{Use: "export-eml"} + cmd.SetContext(testInvocationContext(t.Context(), cfg, invocationOptions{useLocal: true})) + out := &bytes.Buffer{} + cmd.SetOut(out) + return cmd, out, dataDir +} + +func TestExportEMLThreadResolvesNumericReferencesAndAccount(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := testutil.NewTestStore(t) + src, err := st.GetOrCreateSource("gmail", "owner@example.com") + require.NoError(err) + conv, err := st.EnsureConversation(src.ID, "wanted-thread", "Wanted") + require.NoError(err) + persist := func(source, conversation int64, provider, body string) int64 { + id, err := st.PersistMessage(&store.MessagePersistData{Message: &store.Message{ + SourceID: source, ConversationID: conversation, SourceMessageID: provider, MessageType: "email", + }, RawMIME: []byte(body)}) + require.NoError(err) + return id + } + wantedID := persist(src.ID, conv, "999999", "wanted MIME") + other, err := st.GetOrCreateSource("gmail", "other@example.com") + require.NoError(err) + otherConv, err := st.EnsureConversation(other.ID, "other-thread", "Other") + require.NoError(err) + persist(other.ID, otherConv, strconv.FormatInt(wantedID, 10), "unrelated MIME") + cmd, out, dataDir := emlArchiveCommand(t, st, nil) + for _, ref := range []string{"999999", strconv.FormatInt(wantedID, 10)} { + outDir := filepath.Join(dataDir, ref) + require.NoError(runExportEMLThread(cmd, ref, "", outDir)) + got, err := os.ReadFile(filepath.Join(outDir, "1-999999.eml")) + require.NoError(err) + assert.Equal("wanted MIME", string(got)) + } + persist(other.ID, otherConv, "999999", "other MIME") + err = runExportEMLThread(cmd, "999999", "", filepath.Join(dataDir, "ambiguous")) + require.ErrorIs(err, query.ErrAmbiguousReference) + assert.Equal(1, strings.Count(err.Error(), "--account")) + assert.NotContains(err.Error(), "; pass account") + out.Reset() + require.NoError(runExportEMLThread(cmd, "999999", "owner@example.com", filepath.Join(dataDir, "scoped"))) + assert.Contains(out.String(), "thread wanted-thread") +} + +func TestExportEMLThreadHandlesOriginalReadFailures(t *testing.T) { + for _, missing := range []string{"message", "MIME", "corrupt MIME"} { + t.Run(missing, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := testutil.NewTestStore(t) + src, err := st.GetOrCreateSource("gmail", "owner@example.com") + require.NoError(err) + conv, err := st.EnsureConversation(src.ID, "partial-thread", "Partial") + require.NoError(err) + var ids []int64 + for _, name := range []string{"first", "middle", "last"} { + id, err := st.PersistMessage(&store.MessagePersistData{Message: &store.Message{ + SourceID: src.ID, ConversationID: conv, SourceMessageID: name, MessageType: "email", + }, RawMIME: []byte("original MIME")}) + require.NoError(err) + ids = append(ids, id) + } + cmd, out, dataDir := emlArchiveCommand(t, st, func(r *http.Request) { + if r.URL.Path != "/api/v1/cli/message/original" || r.URL.Query().Get("id") != strconv.FormatInt(ids[1], 10) { + return + } + switch missing { + case "message": + _, err := st.MergeDuplicates(ids[0], []int64{ids[1]}, "test-dedup") + if !assert.NoError(err) { //nolint:testifylint // HTTP callback cannot call FailNow on the test goroutine. + return + } + case "MIME": + _, err := st.DB().Exec(st.Rebind("DELETE FROM message_raw WHERE message_id = ?"), ids[1]) + if !assert.NoError(err) { //nolint:testifylint // HTTP callback cannot call FailNow on the test goroutine. + return + } + case "corrupt MIME": + _, err := st.DB().Exec(st.Rebind("UPDATE message_raw SET raw_data = ? WHERE message_id = ?"), []byte("invalid zlib data"), ids[1]) + assert.NoError(err) + } + }) + outDir := filepath.Join(dataDir, "thread") + err = runExportEMLThread(cmd, "first", "", outDir) + if missing == "corrupt MIME" { + require.ErrorContains(err, "export message") + return + } + require.NoError(err) + entries, err := os.ReadDir(outDir) + require.NoError(err) + var names []string + for _, entry := range entries { + names = append(names, entry.Name()) + } + assert.Equal([]string{"1-first.eml", "3-last.eml"}, names) + assert.Contains(out.String(), "Skipped middle:") + assert.Contains(out.String(), "Exported 2 of 3 messages") + assert.Contains(out.String(), "owner@example.com has never completed a sync") + }) + } +} + +func TestExportEMLThreadKeepsInitialMembershipDuringInsert(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := testutil.NewTestStore(t) + src, err := st.GetOrCreateSource("gmail", "owner@example.com") + require.NoError(err) + conv, err := st.EnsureConversation(src.ID, "large-thread", "Large") + require.NoError(err) + base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + persist := func(name string, sentAt time.Time) error { + _, err := st.PersistMessage(&store.MessagePersistData{Message: &store.Message{ + SourceID: src.ID, ConversationID: conv, SourceMessageID: name, MessageType: "email", + SentAt: sql.NullTime{Time: sentAt, Valid: true}, + }, RawMIME: []byte(name)}) + return err + } + for i := 1; i <= 501; i++ { + require.NoError(persist(fmt.Sprintf("original-%03d", i), base.Add(time.Duration(i)*time.Minute))) + } + var inserted sync.Once + cmd, out, dataDir := emlArchiveCommand(t, st, func(r *http.Request) { + if r.URL.Path == "/api/v1/cli/message/original" || + (r.URL.Path == "/api/v1/cli/message/thread" && r.URL.Query().Get("offset") != "") { + inserted.Do(func() { assert.NoError(persist("new-older-message", base)) }) + } + }) + outDir := filepath.Join(dataDir, "thread") + require.NoError(runExportEMLThread(cmd, "original-001", "", outDir)) + entries, err := os.ReadDir(outDir) + require.NoError(err) + require.Len(entries, 501) + for i, entry := range entries { + assert.Equal(fmt.Sprintf("%03d-original-%03d.eml", i+1, i+1), entry.Name()) + } + assert.Contains(out.String(), "Exported 501 of 501 messages") +} diff --git a/cmd/msgvault/cmd/export_messages.go b/cmd/msgvault/cmd/export_messages.go index 739dcf230..abf2b25a1 100644 --- a/cmd/msgvault/cmd/export_messages.go +++ b/cmd/msgvault/cmd/export_messages.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "encoding/json/jsontext" "encoding/json/v2" "errors" @@ -18,7 +19,7 @@ import ( const messageExportSchema = "msgvault-message-export/1" type exportMessagesDeps struct { - openStore func() (*store.Store, func(), error) + openStore func(context.Context) (*store.Store, func(), error) } type exportMessagesOptions struct { @@ -154,7 +155,9 @@ func (s exportMessagesJSONLSink) Message(message store.MessageExportMessage) err } func defaultExportMessagesDeps() exportMessagesDeps { - return exportMessagesDeps{openStore: openWritableStoreAndInitForIngest} + return exportMessagesDeps{openStore: func(ctx context.Context) (*store.Store, func(), error) { + return openWritableStoreAndInitForIngestInvocation(invocationFromContext(ctx)) + }} } func newExportMessagesCmd(deps exportMessagesDeps) *cobra.Command { @@ -216,7 +219,7 @@ func runExportMessages( if deps.openStore == nil { return errors.New("open message archive is not configured") } - st, cleanup, err := deps.openStore() + st, cleanup, err := deps.openStore(cmd.Context()) if err != nil { return err } diff --git a/cmd/msgvault/cmd/export_messages_test.go b/cmd/msgvault/cmd/export_messages_test.go index f458dbbaf..b5a447410 100644 --- a/cmd/msgvault/cmd/export_messages_test.go +++ b/cmd/msgvault/cmd/export_messages_test.go @@ -3,6 +3,7 @@ package cmd import ( "bufio" "bytes" + "context" "encoding/json" "errors" "io" @@ -32,7 +33,7 @@ func TestExportMessagesCommandWritesContractOrderAndCounts(t *testing.T) { ) cmd := newExportMessagesLocalCmd(exportMessagesDeps{ - openStore: func() (*store.Store, func(), error) { + openStore: func(context.Context) (*store.Store, func(), error) { return st, func() {}, nil }, }) @@ -92,7 +93,7 @@ func TestExportMessagesCommandNormalizesFilters(t *testing.T) { require.NoError(err) cmd := newExportMessagesLocalCmd(exportMessagesDeps{ - openStore: func() (*store.Store, func(), error) { + openStore: func(context.Context) (*store.Store, func(), error) { return st, func() {}, nil }, }) @@ -173,7 +174,7 @@ func TestExportMessagesCommandRejectsPreflightErrorsWithoutOutput(t *testing.T) require := require.New(t) st := testutil.NewTestStore(t) cmd := newExportMessagesLocalCmd(exportMessagesDeps{ - openStore: func() (*store.Store, func(), error) { + openStore: func(context.Context) (*store.Store, func(), error) { return st, func() {}, nil }, }) @@ -215,7 +216,7 @@ func TestExportMessagesCommandOmitsCompletionAfterStreamFailure(t *testing.T) { ) cmd := newExportMessagesLocalCmd(exportMessagesDeps{ - openStore: func() (*store.Store, func(), error) { + openStore: func(context.Context) (*store.Store, func(), error) { return st, func() {}, nil }, }) @@ -248,10 +249,12 @@ func TestExportMessagesCommandRoutesThroughDaemon(t *testing.T) { "{\"type\":\"stdout\",\"data\":\"{\\\"record_type\\\":\\\"complete\\\"}\\n\"}\n"+ "{\"type\":\"complete\"}\n", ) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx t.Setenv(daemonCLISubprocessEnv, "") cmd := newExportMessagesCmd(exportMessagesDeps{}) + cmd.SetContext(testCtx) var output bytes.Buffer cmd.SetOut(&output) cmd.SetErr(io.Discard) @@ -369,7 +372,7 @@ func TestExportMessagesCommandPersonScopeUsesBoundParticipants(t *testing.T) { t.Run(selector, func(t *testing.T) { assertions := assert.New(t) requirements := require.New(t) - cmd := newExportMessagesLocalCmd(exportMessagesDeps{openStore: func() (*store.Store, func(), error) { return st, func() {}, nil }}) + cmd := newExportMessagesLocalCmd(exportMessagesDeps{openStore: func(context.Context) (*store.Store, func(), error) { return st, func() {}, nil }}) var output bytes.Buffer cmd.SetOut(&output) cmd.SetErr(io.Discard) diff --git a/cmd/msgvault/cmd/export_token.go b/cmd/msgvault/cmd/export_token.go index 0b99411f8..1567554a2 100644 --- a/cmd/msgvault/cmd/export_token.go +++ b/cmd/msgvault/cmd/export_token.go @@ -14,6 +14,7 @@ import ( "time" "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/config" ) var ( @@ -196,12 +197,31 @@ func (e *tokenExporter) addAccount(baseURL, apiKey, email string) { } } -func runExportToken(_ *cobra.Command, args []string) error { +func runExportToken(cmd *cobra.Command, args []string) error { + return runExportTokenWithClient(cmd, args, &http.Client{Timeout: 30 * time.Second}) +} + +func runExportTokenWithClient(cmd *cobra.Command, args []string, client *http.Client) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg email := args[0] // Resolution order: flag > env var > config file remoteURL := resolveParam(exportTokenTo, "MSGVAULT_REMOTE_URL", cfg.Remote.URL) - apiKey := resolveParam(exportTokenAPIKey, "MSGVAULT_REMOTE_API_KEY", cfg.Remote.APIKey) + apiKey := exportTokenAPIKey + persistInlineKey := apiKey != "" + if !persistInlineKey { + if cmd.Flags().Changed("api-key") { + return errors.New("--api-key must not be empty") + } + if err := cfg.ResolveRemoteKey(); err != nil { + return err + } + apiKey = cfg.Remote.AuthenticationKey() + } if remoteURL == "" { return errors.New("remote URL required: use --to flag, MSGVAULT_REMOTE_URL env var, or [remote] url in config.toml") @@ -211,31 +231,43 @@ func runExportToken(_ *cobra.Command, args []string) error { } exporter := &tokenExporter{ - httpClient: &http.Client{Timeout: 30 * time.Second}, + httpClient: client, tokensDir: cfg.TokensDir(), stdout: os.Stdout, stderr: os.Stderr, } - allowInsecure := exportAllowInsecure || cfg.Remote.AllowInsecure + allowInsecure := cfg.Remote.AllowInsecure + if cmd.Flags().Changed("allow-insecure") { + allowInsecure = exportAllowInsecure + } result, err := exporter.export(email, remoteURL, apiKey, allowInsecure) if err != nil { return err } - // Save remote config for future use - if cfg.Remote.URL != result.remoteURL || - cfg.Remote.APIKey != result.apiKey || - (result.allowInsecure && !cfg.Remote.AllowInsecure) { - cfg.Remote.URL = result.remoteURL - cfg.Remote.APIKey = result.apiKey - if result.allowInsecure { - cfg.Remote.AllowInsecure = true + // Persist explicit choices even when they equal runtime overrides. Editing + // the file preserves unrelated configuration and leaves environment-only + // values and mounted credentials out of saved configuration. + var edits []config.Edit + if exportTokenTo != "" { + edits = append(edits, config.Edit{Key: "remote.url", Value: result.remoteURL}) + } + if persistInlineKey { + edits = append(edits, config.Edit{Key: "remote.api_key", Value: result.apiKey}) + } + if cmd.Flags().Changed("allow-insecure") { + edits = append(edits, config.Edit{Key: "remote.allow_insecure", Value: result.allowInsecure}) + } + if len(edits) > 0 { + snapshot, err := config.ReadConfigFile(cfg.ConfigFilePath()) + if err == nil { + _, err = config.EditConfigFilePrivate(cfg.ConfigFilePath(), snapshot.ETag, edits) } - if err := cfg.Save(); err != nil { + if err != nil { fmt.Fprintf(os.Stderr, "Note: Could not save remote config: %v\n", err) } else { - fmt.Printf("Remote server saved to %s (future exports won't need --to/--api-key)\n", + fmt.Printf("Remote settings saved to %s\n", cfg.ConfigFilePath()) } } diff --git a/cmd/msgvault/cmd/export_token_test.go b/cmd/msgvault/cmd/export_token_test.go index 256108460..208b91701 100644 --- a/cmd/msgvault/cmd/export_token_test.go +++ b/cmd/msgvault/cmd/export_token_test.go @@ -339,40 +339,6 @@ func TestExport_AccountPostFailureIsNonFatal(t *testing.T) { assert.Contains(t, stderr.String(), "Warning", "stderr should warn about account POST failure") } -func TestExport_AllowInsecureFromConfig(t *testing.T) { - // Regression: when config has allow_insecure=true for an HTTP URL, - // export should succeed even without the --allow-insecure flag. - // This simulates the resolution in runExportToken: - // allowInsecure := exportAllowInsecure || cfg.Remote.AllowInsecure - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if strings.HasPrefix(r.URL.Path, "/api/v1/auth/token/") { - w.WriteHeader(http.StatusCreated) - return - } - w.WriteHeader(http.StatusCreated) - })) - defer srv.Close() - - tokensDir := t.TempDir() - writeTestToken(t, tokensDir, `{"token":"data"}`) - - e := &tokenExporter{ - httpClient: srv.Client(), - tokensDir: tokensDir, - stdout: io.Discard, - stderr: io.Discard, - } - - // Simulate: CLI flag is false, but config had allow_insecure=true - cliFlag := false - configAllowInsecure := true - allowInsecure := cliFlag || configAllowInsecure - - result, err := e.export("user@gmail.com", srv.URL, "key", allowInsecure) - require.NoError(t, err, "export should succeed with config allow_insecure=true") - assert.True(t, result.allowInsecure, "result.allowInsecure should be true") -} - func TestExport_InvalidScheme(t *testing.T) { e := &tokenExporter{ httpClient: http.DefaultClient, diff --git a/cmd/msgvault/cmd/forward_draft.go b/cmd/msgvault/cmd/forward_draft.go new file mode 100644 index 000000000..f45cfe4f1 --- /dev/null +++ b/cmd/msgvault/cmd/forward_draft.go @@ -0,0 +1,25 @@ +package cmd + +import "github.com/spf13/cobra" + +func init() { + rootCmd.AddCommand(newDraftForwardCommand()) +} + +func newDraftForwardCommand() *cobra.Command { + command := &cobra.Command{ + Use: "draft-forward ", + Short: "Create an IMAP draft forwarding an archived message", + Args: cobra.ExactArgs(1), + RunE: runDaemonCLICommandHTTPFromCobra, + } + command.Flags().String(draftFromFlag, "", "confirmed destination source identity for the draft") + command.Flags().StringArray("to", nil, "recipient address, repeatable") + command.Flags().StringArray("cc", nil, "Cc recipient address, repeatable") + command.Flags().StringArray("bcc", nil, "Bcc recipient address, repeatable") + command.Flags().String("account", "", "destination source account or display name") + command.Flags().Int64("source-id", 0, "exact destination source ID") + command.Flags().String("body", "", "forwarding note") + command.Flags().Bool("json", false, "emit one JSON result") + return command +} diff --git a/cmd/msgvault/cmd/forward_draft_test.go b/cmd/msgvault/cmd/forward_draft_test.go new file mode 100644 index 000000000..ecf5a74bd --- /dev/null +++ b/cmd/msgvault/cmd/forward_draft_test.go @@ -0,0 +1,78 @@ +package cmd + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/msgvault/internal/attachmentpolicy" + imaplib "go.kenn.io/msgvault/internal/imap" + "go.kenn.io/msgvault/internal/store" +) + +func TestDraftForwardArgs(t *testing.T) { + assertions := assert.New(t) + requirements := require.New(t) + intent, err := parseDraftForwardArgs([]string{ + "draft-forward", "42", "--source-id", "7", "--from", "sender@example.test", + "--to", "to@example.test", "--cc=cc@example.test", "--bcc", "bcc@example.test", + }) + requirements.NoError(err) + assertions.Equal(int64(42), intent.MessageID) + assertions.Equal(int64(7), intent.SourceID) + assertions.Equal([]string{"to@example.test"}, intent.To) + assertions.Equal([]string{"cc@example.test"}, intent.Cc) + assertions.Equal([]string{"bcc@example.test"}, intent.Bcc) +} + +func TestDraftForwardArgsRequireExplicitDestinationAndRecipient(t *testing.T) { + assertions := assert.New(t) + for _, args := range [][]string{ + {"draft-forward", "42", "--to", "to@example.test"}, + {"draft-forward", "42", "--source-id", "7"}, + {"draft-forward", "42", "--source-id", "7", "--account", "account@example.test", "--to", "to@example.test"}, + } { + _, err := parseDraftForwardArgs(args) + assertions.Error(err, args) + } +} + +func TestDraftForwardArgsAllowEmptyNote(t *testing.T) { + intent, err := parseDraftForwardArgs([]string{ + "draft-forward", "42", "--account=account@example.test", "--to=to@example.test", "--body=", + }) + require.NoError(t, err) + assert.Empty(t, intent.Body) +} + +func TestPrepareIMAPDraftAttachmentWritesKeepsOccurrences(t *testing.T) { + assertions := assert.New(t) + requirements := require.New(t) + content := []byte("same bytes") + digest := sha256.Sum256(content) + hash := hex.EncodeToString(digest[:]) + draft, err := imaplib.BuildForward(imaplib.ForwardOptions{ + From: "sender@example.test", To: []string{"recipient@example.test"}, Subject: "Original", + Attachments: []imaplib.ForwardAttachment{ + {Filename: "one.txt", ContentType: "text/plain", Content: content}, + {Filename: "two.txt", ContentType: "text/plain", Content: content}, + }, + }, time.Now(), "forward@example.test") + requirements.NoError(err) + refs := []store.AttachmentRef{ + {Filename: "one.txt", ContentHash: hash, State: attachmentpolicy.StateStored}, + {Filename: "two.txt", ContentHash: hash, State: attachmentpolicy.StateStored}, + } + writes, err := prepareIMAPDraftAttachmentWrites(context.Background(), draft.Parsed, refs) + requirements.NoError(err) + requirements.Len(writes, 2) + assertions.NotEmpty(writes[0].SourcePartKey) + assertions.NotEqual(writes[0].SourcePartKey, writes[1].SourcePartKey) + assertions.Equal(hash, writes[0].ContentHash) + assertions.Equal(hash, writes[1].ContentHash) +} diff --git a/cmd/msgvault/cmd/gc.go b/cmd/msgvault/cmd/gc.go index 2f62c4767..5b2da63a7 100644 --- a/cmd/msgvault/cmd/gc.go +++ b/cmd/msgvault/cmd/gc.go @@ -1,6 +1,7 @@ package cmd import ( + "errors" "fmt" "os" "path/filepath" @@ -59,8 +60,13 @@ explicitly opt out of the backup. This command never deletes remote messages.`, } func runGCLocal(cmd *cobra.Command, options gcOptions) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg ctx := cmd.Context() - st, cleanup, err := openWritableStoreAndInit() + st, cleanup, err := openWritableStoreAndInitForInvocation(state) if err != nil { return err } @@ -146,6 +152,11 @@ func runGCLocal(cmd *cobra.Command, options gcOptions) error { func purgeGCMessages( cmd *cobra.Command, st *store.Store, plan store.GCPlan, noBackup bool, ) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg ctx := cmd.Context() out := cmd.OutOrStdout() if !noBackup { diff --git a/cmd/msgvault/cmd/gc_test.go b/cmd/msgvault/cmd/gc_test.go index 9df107144..e26f7134f 100644 --- a/cmd/msgvault/cmd/gc_test.go +++ b/cmd/msgvault/cmd/gc_test.go @@ -2,6 +2,7 @@ package cmd import ( "bytes" + "context" "database/sql" "net/url" "os" @@ -24,11 +25,13 @@ func TestGCAlwaysProxiesThroughDaemonCLIRunner(t *testing.T) { `{"type":"stdout","data":"Deleted 2 source-deleted message(s).\n"}`, `{"type":"complete"}`, ) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx t.Setenv(daemonCLISubprocessEnv, "") var stdout bytes.Buffer cmd := newGCCmd() + cmd.SetContext(testCtx) cmd.SetArgs([]string{"--yes", "--no-backup"}) cmd.SetOut(&stdout) @@ -44,11 +47,13 @@ func TestGCConfirmsBeforeProxyingToDaemon(t *testing.T) { }, `{"type":"complete"}`, ) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx t.Setenv(daemonCLISubprocessEnv, "") var stdout bytes.Buffer cmd := newGCCmd() + cmd.SetContext(testCtx) cmd.SetArgs([]string{"--no-backup"}) cmd.SetIn(strings.NewReader("y\n")) cmd.SetOut(&stdout) @@ -60,11 +65,13 @@ func TestGCConfirmsBeforeProxyingToDaemon(t *testing.T) { func TestGCCancelledConfirmationDoesNotProxy(t *testing.T) { server, requests := newDaemonCLIRunnerTestServer(t, nil, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx t.Setenv(daemonCLISubprocessEnv, "") var stdout bytes.Buffer cmd := newGCCmd() + cmd.SetContext(testCtx) cmd.SetIn(strings.NewReader("n\n")) cmd.SetOut(&stdout) @@ -75,10 +82,10 @@ func TestGCCancelledConfirmationDoesNotProxy(t *testing.T) { func TestRunGCLocalCancellationWritesNothing(t *testing.T) { assert := assert.New(t) - deletedID, _, _ := seedGCCommandArchive(t) + deletedID, _, _, testCtx, cfg := seedGCCommandArchive(t) var output bytes.Buffer cmd := &cobra.Command{} - cmd.SetContext(t.Context()) + cmd.SetContext(testCtx) cmd.SetIn(strings.NewReader("n\n")) cmd.SetOut(&output) @@ -93,15 +100,15 @@ func TestRunGCLocalCancellationWritesNothing(t *testing.T) { func TestRunGCLocalBacksUpBeforeDeleteAndCompacts(t *testing.T) { assert := assert.New(t) require := require.New(t) - deletedID, activeID, dedupID := seedGCCommandArchive(t) + deletedID, activeID, dedupID, testCtx, cfg := seedGCCommandArchive(t) var output bytes.Buffer cmd := &cobra.Command{} - cmd.SetContext(t.Context()) + cmd.SetContext(testCtx) cmd.SetOut(&output) - orphanBlob := seedGCLooseBlob(t, deletedID, strings.Repeat("0a", 32)) - sharedBlob := seedGCLooseBlob(t, deletedID, strings.Repeat("0b", 32)) - seedGCAttachmentRow(t, activeID, strings.Repeat("0b", 32)) + orphanBlob := seedGCLooseBlob(t, cfg, deletedID, strings.Repeat("0a", 32)) + sharedBlob := seedGCLooseBlob(t, cfg, deletedID, strings.Repeat("0b", 32)) + seedGCAttachmentRow(t, cfg, activeID, strings.Repeat("0b", 32)) require.NoError(runGCLocal(cmd, gcOptions{yes: true})) assert.Contains(output.String(), "Source-deleted messages to purge: 1") @@ -136,9 +143,9 @@ func TestRunGCLocalBacksUpBeforeDeleteAndCompacts(t *testing.T) { func TestRunGCLocalSweepsOrphanBlobsWhenNothingToPurge(t *testing.T) { assert := assert.New(t) require := require.New(t) - _, activeID, _ := seedGCCommandArchive(t) + _, activeID, _, testCtx, cfg := seedGCCommandArchive(t) firstRun := &cobra.Command{} - firstRun.SetContext(t.Context()) + firstRun.SetContext(testCtx) firstRun.SetOut(&bytes.Buffer{}) require.NoError(runGCLocal(firstRun, gcOptions{yes: true, noBackup: true})) @@ -148,11 +155,11 @@ func TestRunGCLocalSweepsOrphanBlobsWhenNothingToPurge(t *testing.T) { orphanBlob := filepath.Join(cfg.AttachmentsDir(), orphanHash[:2], orphanHash) require.NoError(os.MkdirAll(filepath.Dir(orphanBlob), 0o755), "create blob dir") require.NoError(os.WriteFile(orphanBlob, []byte("orphan"), 0o600), "write orphan blob") - sharedBlob := seedGCLooseBlob(t, activeID, strings.Repeat("0d", 32)) + sharedBlob := seedGCLooseBlob(t, cfg, activeID, strings.Repeat("0d", 32)) var output bytes.Buffer rerun := &cobra.Command{} - rerun.SetContext(t.Context()) + rerun.SetContext(testCtx) rerun.SetOut(&output) require.NoError(runGCLocal(rerun, gcOptions{yes: true})) @@ -173,7 +180,7 @@ func TestRunGCLocalSweepsOrphanBlobsWhenNothingToPurge(t *testing.T) { func TestRunGCLocalBacksUpFileURIDatabaseBesideArchive(t *testing.T) { assert := assert.New(t) require := require.New(t) - deletedID, _, _ := seedGCCommandArchive(t) + deletedID, _, _, testCtx, cfg := seedGCCommandArchive(t) dbPath := cfg.DatabaseDSN() cfg.Data.DatabaseURL = (&url.URL{ Scheme: "file", @@ -182,7 +189,7 @@ func TestRunGCLocalBacksUpFileURIDatabaseBesideArchive(t *testing.T) { var output bytes.Buffer cmd := &cobra.Command{} - cmd.SetContext(t.Context()) + cmd.SetContext(testCtx) cmd.SetOut(&output) require.NoError(runGCLocal(cmd, gcOptions{yes: true})) @@ -195,7 +202,7 @@ func TestRunGCLocalBacksUpFileURIDatabaseBesideArchive(t *testing.T) { assert.NotContains(filepath.Base(backups[0]), "?") } -func seedGCCommandArchive(t *testing.T) (deletedID, activeID, dedupID int64) { +func seedGCCommandArchive(t *testing.T) (deletedID, activeID, dedupID int64, testCtx context.Context, cfg *config.Config) { t.Helper() require := require.New(t) dataDir := t.TempDir() @@ -205,6 +212,7 @@ func seedGCCommandArchive(t *testing.T) (deletedID, activeID, dedupID int64) { Data: config.DataConfig{DataDir: dataDir}, } t.Cleanup(func() { cfg = savedCfg }) + testCtx = testInvocationContext(t.Context(), cfg, invocationOptions{}) st, err := store.OpenForTest(cfg.DatabaseDSN()) require.NoError(err, "OpenForTest") @@ -238,14 +246,14 @@ func seedGCCommandArchive(t *testing.T) (deletedID, activeID, dedupID int64) { sql.NullString{String: strings.Repeat("deleted payload ", 100_000), Valid: true}, sql.NullString{}), "store deleted payload") require.NoError(st.Close(), "close seed store") - return deletedID, activeID, dedupID + return deletedID, activeID, dedupID, testCtx, cfg } // seedGCLooseBlob writes a loose content-addressed blob file and attaches it // to messageID, returning the blob's filesystem path. -func seedGCLooseBlob(t *testing.T, messageID int64, hash string) string { +func seedGCLooseBlob(t *testing.T, cfg *config.Config, messageID int64, hash string) string { t.Helper() - seedGCAttachmentRow(t, messageID, hash) + seedGCAttachmentRow(t, cfg, messageID, hash) fullPath := filepath.Join( cfg.AttachmentsDir(), hash[:2], hash, ) @@ -255,7 +263,7 @@ func seedGCLooseBlob(t *testing.T, messageID int64, hash string) string { return fullPath } -func seedGCAttachmentRow(t *testing.T, messageID int64, hash string) { +func seedGCAttachmentRow(t *testing.T, cfg *config.Config, messageID int64, hash string) { t.Helper() st, err := store.OpenForTest(cfg.DatabaseDSN()) require.NoError(t, err, "open store for attachment seed") diff --git a/cmd/msgvault/cmd/granola.go b/cmd/msgvault/cmd/granola.go index b8794cc6e..179f71617 100644 --- a/cmd/msgvault/cmd/granola.go +++ b/cmd/msgvault/cmd/granola.go @@ -4,10 +4,7 @@ import ( "context" "errors" "fmt" - "os" - "os/signal" "strings" - "syscall" "time" "github.com/spf13/cobra" @@ -24,7 +21,7 @@ var ( var ( newGranolaClient = granola.NewClient - rebuildGranolaCacheAfterWrite = rebuildCacheAfterWrite + rebuildGranolaCacheAfterWrite = rebuildCacheAfterManualSync rebuildGranolaCacheAfterScheduledSync = rebuildCacheAfterScheduledSync ) @@ -40,7 +37,10 @@ const granolaConfigHint = `Add to your config.toml: // resolveGranolaSource picks the [[granola]] entry for an optional CLI // argument: an explicit identifier must match a configured entry; with no // argument there must be exactly one entry. -func resolveGranolaSource(args []string) (*config.GranolaSource, error) { +func resolveGranolaSource(args []string, cfg *config.Config) (*config.GranolaSource, error) { + if cfg == nil { + return nil, errors.New("configuration is unavailable") + } if len(cfg.Granola) == 0 { return nil, errors.New("no [[granola]] sources configured\n\n" + granolaConfigHint) } @@ -76,11 +76,16 @@ Examples: msgvault add-granola you@example.com`, Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg if !isDaemonCLISubprocess() { return runDaemonCLICommandHTTPFromCobra(cmd, args) } - src, err := resolveGranolaSource(args) + src, err := resolveGranolaSource(args, cfg) if err != nil { return err } @@ -98,7 +103,7 @@ Examples: return fmt.Errorf("validate Granola API key: %w", err) } - s, cleanup, err := openWritableStoreAndInitForIngest() + s, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) if err != nil { return err } @@ -109,7 +114,7 @@ Examples: ); err != nil { return err } - if err := runPostSourceCreateMigrations(s); err != nil { + if err := runPostSourceCreateMigrationsForInvocation(s, state); err != nil { return fmt.Errorf("post-source-create migrations: %w", err) } @@ -139,13 +144,18 @@ Examples: msgvault sync-granola --full --after 2024-01-01`, Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg if !isDaemonCLISubprocess() { return runDaemonCLICommandHTTPFromCobra(cmd, args) } var sources []config.GranolaSource if len(args) > 0 || len(cfg.Granola) == 1 { - src, err := resolveGranolaSource(args) + src, err := resolveGranolaSource(args, cfg) if err != nil { return err } @@ -183,26 +193,15 @@ Examples: }) } - s, cleanup, err := openWritableStoreAndInitForIngest() + s, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) if err != nil { return err } defer cleanup() dbPath := cfg.DatabaseDSN() - ctx, cancel := context.WithCancel(cmd.Context()) - defer cancel() - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) - defer signal.Stop(sigChan) - go func() { - select { - case <-sigChan: - _, _ = fmt.Fprintln(cmd.ErrOrStderr(), "\nInterrupted. Finishing current note...") - cancel() - case <-ctx.Done(): - } - }() + ctx, stop := withInterruptCancel(cmd, "\nInterrupted. Finishing current note...") + defer stop() pendingCacheWrites := &granola.ImportSummary{} for _, validated := range validatedSources { @@ -225,11 +224,11 @@ Examples: if ctx.Err() != nil { _, _ = fmt.Fprintln(cmd.OutOrStdout(), "\nInterrupted — re-run sync-granola to resume.") return finishGranolaImport(src.Identifier, pendingCacheWrites, ctx.Err(), func() error { - return rebuildGranolaCacheAfterWrite(dbPath) + return rebuildGranolaCacheAfterWrite(dbPath, state) }) } if finishErr := finishGranolaImport(src.Identifier, pendingCacheWrites, err, func() error { - return rebuildGranolaCacheAfterWrite(dbPath) + return rebuildGranolaCacheAfterWrite(dbPath, state) }); finishErr != nil { return finishErr } @@ -244,7 +243,7 @@ Examples: } } - return rebuildGranolaCacheAfterWrite(dbPath) + return rebuildGranolaCacheAfterWrite(dbPath, state) }, } @@ -311,5 +310,5 @@ func init() { syncGranolaCmd.Flags().StringVar(&syncGranolaAfter, "after", "", "full-sync only notes created after this date (YYYY-MM-DD; implies --full)") syncGranolaCmd.Flags().BoolVar(&syncGranolaFull, "full", false, "ignore stored cursor and re-fetch every note (repairs existing rows in place)") rootCmd.AddCommand(addGranolaCmd) - rootCmd.AddCommand(syncGranolaCmd) + rootCmd.AddCommand(addManualSyncCacheFlags(syncGranolaCmd)) } diff --git a/cmd/msgvault/cmd/granola_test.go b/cmd/msgvault/cmd/granola_test.go index bc412b107..bcd4c77bd 100644 --- a/cmd/msgvault/cmd/granola_test.go +++ b/cmd/msgvault/cmd/granola_test.go @@ -129,12 +129,12 @@ func TestManualGranolaPartialImportRefreshesCacheBeforeReturningError(t *testing AccountEmail: "user-a@example.com", APIKey: "grn_test", }} - withStoreResolverConfig(t, testCfg) + testCtx := withStoreResolverConfig(t, testCfg) savedRefresh := rebuildGranolaCacheAfterWrite refreshes := 0 refreshSawWrite := false - rebuildGranolaCacheAfterWrite = func(dbPath string) error { + rebuildGranolaCacheAfterWrite = func(dbPath string, _ *invocation) error { refreshes++ st, openErr := store.Open(dbPath) require.NoError(openErr) @@ -152,7 +152,8 @@ func TestManualGranolaPartialImportRefreshesCacheBeforeReturningError(t *testing syncGranolaLimit, syncGranolaAfter, syncGranolaFull = oldLimit, oldAfter, oldFull }) cmd := &cobra.Command{Use: "sync-granola"} - cmd.SetContext(context.Background()) + cmd.SetContext(testCtx) + cmd.SetContext(testCtx) cmd.SetOut(&bytes.Buffer{}) cmd.SetErr(&bytes.Buffer{}) @@ -174,11 +175,12 @@ func TestManualGranolaCancellationReturnsError(t *testing.T) { testCfg.Granola = []config.GranolaSource{{ Identifier: "work", AccountEmail: "user-a@example.com", APIKey: "grn_test", }} - withStoreResolverConfig(t, testCfg) + testCtx := withStoreResolverConfig(t, testCfg) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(testCtx) cancel() cmd := &cobra.Command{Use: "sync-granola"} + cmd.SetContext(testCtx) cmd.SetContext(ctx) cmd.SetOut(&bytes.Buffer{}) cmd.SetErr(&bytes.Buffer{}) @@ -275,12 +277,12 @@ func TestManualGranolaLaterFailureRefreshesEarlierSourceWrites(t *testing.T) { {Identifier: "first", AccountEmail: "user-a@example.com", APIKey: "grn_clean"}, {Identifier: "second", AccountEmail: "user-b@example.com", APIKey: "grn_partial"}, } - withStoreResolverConfig(t, testCfg) + testCtx := withStoreResolverConfig(t, testCfg) savedRefresh := rebuildGranolaCacheAfterWrite refreshes := 0 refreshSawWrite := false - rebuildGranolaCacheAfterWrite = func(dbPath string) error { + rebuildGranolaCacheAfterWrite = func(dbPath string, _ *invocation) error { refreshes++ st, openErr := store.Open(dbPath) require.NoError(openErr) @@ -298,7 +300,8 @@ func TestManualGranolaLaterFailureRefreshesEarlierSourceWrites(t *testing.T) { syncGranolaLimit, syncGranolaAfter, syncGranolaFull = oldLimit, oldAfter, oldFull }) cmd := &cobra.Command{Use: "sync-granola"} - cmd.SetContext(context.Background()) + cmd.SetContext(testCtx) + cmd.SetContext(testCtx) cmd.SetOut(&bytes.Buffer{}) cmd.SetErr(&bytes.Buffer{}) @@ -328,10 +331,11 @@ func TestManualGranolaPrevalidatesAllSourcesBeforeImport(t *testing.T) { {Identifier: "first", AccountEmail: "user-a@example.com", APIKey: "grn_clean"}, {Identifier: "second", AccountEmail: "user-b@example.com"}, } - withStoreResolverConfig(t, testCfg) + testCtx := withStoreResolverConfig(t, testCfg) cmd := &cobra.Command{Use: "sync-granola"} - cmd.SetContext(context.Background()) + cmd.SetContext(testCtx) + cmd.SetContext(testCtx) cmd.SetOut(&bytes.Buffer{}) cmd.SetErr(&bytes.Buffer{}) diff --git a/cmd/msgvault/cmd/identity.go b/cmd/msgvault/cmd/identity.go index f66eb5123..e5183a891 100644 --- a/cmd/msgvault/cmd/identity.go +++ b/cmd/msgvault/cmd/identity.go @@ -598,7 +598,8 @@ func renderIdentityRemoveResult(w io.Writer, result daemonclient.CLIIdentityRemo func renderIdentityNoIdentityWarning(w io.Writer, account string) { _, _ = fmt.Fprintf(w, "Warning: %s now has no confirmed identity. "+ "Dedup sent-copy detection for this account will rely on is_from_me "+ - "and SENT label signals only.\n", account) + "and SENT label signals only. Future syncs will not restore a default; "+ + "use the source's add command with --no-default-identity=false to re-enable it.\n", account) } func identityShowArgs(cmd *cobra.Command, args []string) error { @@ -676,6 +677,7 @@ func init() { identityCmd.AddCommand(identityRemoveCmd) identityCmd.AddCommand(identityDiscoverCmd) identityCmd.AddCommand(identityImportCmd) + identityCmd.AddCommand(newIdentityMatchesCommand()) identityListCmd.Flags().StringVar(&identityListAccount, "account", "", "Restrict to a single account") @@ -703,7 +705,7 @@ func init() { identityDiscoverCmd.Flags().BoolVar(&identityDiscoverApply, "apply", false, "Confirm strong identity evidence after the preview scan completes") identityDiscoverCmd.Flags().BoolVar(&identityDiscoverProvider, - "provider", false, "Include configured provider alias inventory") + "provider", false, "Include authenticated Gmail profile or configured provider alias inventory") identityDiscoverCmd.Flags().StringArrayVar(&identityDiscoverConfirm, "confirm", nil, "Explicitly confirm a weak candidate (repeatable; requires --apply)") identityDiscoverCmd.Flags().BoolVar(&identityDiscoverJSON, diff --git a/cmd/msgvault/cmd/identity_matches.go b/cmd/msgvault/cmd/identity_matches.go new file mode 100644 index 000000000..399ddc2ec --- /dev/null +++ b/cmd/msgvault/cmd/identity_matches.go @@ -0,0 +1,253 @@ +package cmd + +import ( + "encoding/json/jsontext" + "encoding/json/v2" + "errors" + "fmt" + "io" + "os" + "strconv" + "strings" + "text/tabwriter" + + "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/textutil" + "go.kenn.io/msgvault/pkg/client/generated" +) + +func newIdentityMatchesCommand() *cobra.Command { + command := &cobra.Command{Use: "matches", Short: "Review suggested identity matches", Args: cobra.NoArgs} + command.AddCommand(newIdentityMatchesListCommand()) + command.AddCommand(newIdentityMatchesShowCommand()) + command.AddCommand(newIdentityMatchesDecisionCommand("accept")) + command.AddCommand(newIdentityMatchesDecisionCommand("reject")) + return command +} + +func newIdentityMatchesListCommand() *cobra.Command { + var state string + var limit, offset int64 + var jsonOutput bool + command := &cobra.Command{ + Use: "list", Short: "List reviewable identity matches", Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + if state != "all" && state != "candidate" && state != "accepted" && + state != "rejected" && state != "conflict" { + return usageErr(cmd, errors.New("--state must be candidate, accepted, rejected, conflict, or all")) + } + if limit < 1 || limit > 500 || offset < 0 { + return usageErr(cmd, errors.New("--limit must be 1–500 and --offset must be nonnegative")) + } + client, _, err := OpenHTTPStore(cmd.Context()) + if err != nil { + return err + } + defer func() { _ = client.Close() }() + response, err := client.ListIdentityMatches(cmd.Context(), state, limit, offset) + if err != nil { + return err + } + if jsonOutput { + return writeIdentityMatchesJSON(cmd.OutOrStdout(), response) + } + return writeIdentityMatchesTable(cmd.OutOrStdout(), response.Candidates) + }, + } + command.Flags().StringVar(&state, "state", "candidate", "Candidate state or all") + command.Flags().Int64Var(&limit, "limit", 100, "Maximum matches (1–500)") + command.Flags().Int64Var(&offset, "offset", 0, "Zero-based page offset") + command.Flags().BoolVar(&jsonOutput, "json", false, "Print the complete API response") + return command +} + +func newIdentityMatchesShowCommand() *cobra.Command { + var jsonOutput bool + command := &cobra.Command{ + Use: "show ", Short: "Inspect one identity match and its review token", Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + id, err := positiveIdentityMatchID(args[0]) + if err != nil { + return usageErr(cmd, err) + } + client, _, err := OpenHTTPStore(cmd.Context()) + if err != nil { + return err + } + defer func() { _ = client.Close() }() + response, err := client.GetIdentityMatch(cmd.Context(), id) + if err != nil { + return err + } + if jsonOutput { + return writeIdentityMatchesJSON(cmd.OutOrStdout(), response) + } + return writeIdentityMatchDetail(cmd.OutOrStdout(), response) + }, + } + command.Flags().BoolVar(&jsonOutput, "json", false, "Print the complete API response") + return command +} + +func newIdentityMatchesDecisionCommand(action string) *cobra.Command { + var token, notesFile string + var notesStdin, jsonOutput bool + command := &cobra.Command{ + Use: action + " ", Short: strings.ToUpper(action[:1]) + action[1:] + " an identity match", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + id, err := positiveIdentityMatchID(args[0]) + if err != nil { + return usageErr(cmd, err) + } + if strings.TrimSpace(token) == "" { + return usageErr(cmd, errors.New("--review-token is required; inspect the match first")) + } + if cmd.Flags().Changed("notes-file") && notesStdin { + return usageErr(cmd, errors.New("--notes-file and --notes-stdin are mutually exclusive")) + } + notes, err := identityMatchesNotes(cmd, notesFile, notesStdin) + if err != nil { + return err + } + client, _, err := OpenHTTPStore(cmd.Context()) + if err != nil { + return err + } + defer func() { _ = client.Close() }() + if action == "accept" { + response, err := client.AcceptIdentityMatch(cmd.Context(), id, token, notes) + if err != nil { + return err + } + if jsonOutput { + return writeIdentityMatchesJSON(cmd.OutOrStdout(), response) + } + return writeIdentityMatchDecision(cmd.OutOrStdout(), &response.Candidate, + response.IdentityRevision, string(response.CacheState)) + } + response, err := client.RejectIdentityMatch(cmd.Context(), id, token, notes) + if err != nil { + return err + } + if jsonOutput { + return writeIdentityMatchesJSON(cmd.OutOrStdout(), response) + } + return writeIdentityMatchDecision(cmd.OutOrStdout(), &response.Candidate, + response.IdentityRevision, string(response.CacheState)) + }, + } + command.Flags().StringVar(&token, "review-token", "", "Token from the reviewed match") + command.Flags().StringVar(¬esFile, "notes-file", "", "Read decision notes from a file") + command.Flags().BoolVar(¬esStdin, "notes-stdin", false, "Read decision notes from standard input") + command.Flags().BoolVar(&jsonOutput, "json", false, "Print the complete API response") + return command +} + +func positiveIdentityMatchID(raw string) (int64, error) { + id, err := strconv.ParseInt(raw, 10, 64) + if err != nil || id <= 0 { + return 0, errors.New("identity match ID must be a positive integer") + } + return id, nil +} + +func identityMatchesNotes(cmd *cobra.Command, file string, stdin bool) (*string, error) { + if !stdin && !cmd.Flags().Changed("notes-file") { + return nil, nil //nolint:nilnil // No notes were requested; omit the optional API field. + } + reader := cmd.InOrStdin() + if !stdin { + opened, err := os.Open(file) + if err != nil { + return nil, fmt.Errorf("open notes file: %w", err) + } + defer func() { _ = opened.Close() }() + reader = opened + } + data, err := io.ReadAll(io.LimitReader(reader, 1<<20+1)) + if err != nil { + return nil, fmt.Errorf("read identity match notes: %w", err) + } + if len(data) > 1<<20 { + return nil, errors.New("identity match notes exceed 1 MiB") + } + notes := strings.TrimSpace(string(data)) + return ¬es, nil +} + +func writeIdentityMatchesJSON(w io.Writer, value any) error { + return json.MarshalEncode(jsontext.NewEncoder(w), value, json.Deterministic(true)) +} + +func writeIdentityMatchesTable(w io.Writer, candidates []generated.IdentityMatchCandidate) error { + tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0) + if _, err := fmt.Fprintln(tw, "ID\tSTATE\tENDPOINTS\tBASIS\tEVIDENCE\tREVIEW TOKEN\tBLOCKER"); err != nil { + return fmt.Errorf("write identity match header: %w", err) + } + for _, candidate := range candidates { + if _, err := fmt.Fprintf(tw, "%d\t%s\t%s:%d → %s:%d\t%s\t%d\t%s\t%s\n", + candidate.ID, candidate.State, candidate.LeftKind, candidate.LeftID, + candidate.RightKind, candidate.RightID, candidate.Basis, + len(candidate.Evidence), textutil.SanitizeTerminal(stringOrEmpty(candidate.ReviewToken)), + textutil.SanitizeTerminal(stringOrEmpty(candidate.Blocker))); err != nil { + return fmt.Errorf("write identity match row: %w", err) + } + } + if err := tw.Flush(); err != nil { + return fmt.Errorf("flush identity match table: %w", err) + } + return nil +} + +func writeIdentityMatchDetail(w io.Writer, c *generated.IdentityMatchCandidate) error { + if _, err := fmt.Fprintf(w, "Match %d: %s:%d → %s:%d\nState: %s\nBasis: %s\nEvidence: %d\nReview token: %s\n", + c.ID, c.LeftKind, c.LeftID, c.RightKind, c.RightID, c.State, c.Basis, + len(c.Evidence), textutil.SanitizeTerminal(stringOrEmpty(c.ReviewToken))); err != nil { + return fmt.Errorf("write identity match detail: %w", err) + } + for _, evidence := range c.Evidence { + if _, err := fmt.Fprintf(w, " - %s (%s; %d archive source(s))", + textutil.SanitizeTerminal(evidence.EvidenceKind), evidence.Source, + len(evidence.SourceSupport)); err != nil { + return fmt.Errorf("write identity match evidence: %w", err) + } + if evidence.Detail != nil && *evidence.Detail != "" { + if _, err := fmt.Fprintf(w, ": %s", textutil.SanitizeTerminal(*evidence.Detail)); err != nil { + return fmt.Errorf("write identity match evidence detail: %w", err) + } + } + if _, err := fmt.Fprintln(w); err != nil { + return fmt.Errorf("finish identity match evidence: %w", err) + } + } + if c.Blocker != nil && *c.Blocker != "" { + _, err := fmt.Fprintf(w, "Blocker: %s\n", textutil.SanitizeTerminal(*c.Blocker)) + if err != nil { + return fmt.Errorf("write identity match blocker: %w", err) + } + } + return nil +} + +func writeIdentityMatchDecision(w io.Writer, c *generated.IdentityMatchCandidate, revision int64, cacheState string) error { + next := "Inspect the resulting person and CardDAV publication." + if c.ApplicationPending { + next = "Read this match again after link recovery before checking the person." + } else if c.State == "rejected" { + next = "No identity link was created." + } + _, err := fmt.Fprintf(w, "Match %d: %s\nIdentity revision: %d\nCache: %s\nApplication pending: %t\nNext: %s\n", + c.ID, c.State, revision, cacheState, c.ApplicationPending, next) + if err != nil { + return fmt.Errorf("write identity match decision: %w", err) + } + return nil +} + +func stringOrEmpty(value *string) string { + if value == nil { + return "" + } + return *value +} diff --git a/cmd/msgvault/cmd/identity_matches_test.go b/cmd/msgvault/cmd/identity_matches_test.go new file mode 100644 index 000000000..76e72a8c1 --- /dev/null +++ b/cmd/msgvault/cmd/identity_matches_test.go @@ -0,0 +1,141 @@ +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/daemonclient" +) + +const identityMatchesCLICandidate = `{"id":17,"left_kind":"participant","left_id":7,"right_kind":"participant","right_id":8,"basis":"email","source":"archive_observation","state":"candidate","review_token":"review-token-17","actionable":true,"application_pending":false,"evidence":[{"id":4,"candidate_id":17,"evidence_kind":"email","source":"archive_observation","created_at":"2026-01-01T00:00:00Z"}],"created_at":"2026-01-01T00:00:00Z","updated_at":"2026-01-01T00:00:00Z"}` + +func runIdentityMatchesCLI(ctx context.Context, t *testing.T, stdin string, args ...string) (string, error) { + t.Helper() + root := &cobra.Command{Use: "msgvault", SilenceErrors: true, SilenceUsage: true} + localFlag := rootCmd.PersistentFlags().Lookup("local") + savedChanged := localFlag.Changed + t.Cleanup(func() { localFlag.Changed = savedChanged }) + root.PersistentFlags().AddFlag(localFlag) + identity := &cobra.Command{Use: "identity"} + identity.AddCommand(newIdentityMatchesCommand()) + root.AddCommand(identity) + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(io.Discard) + root.SetIn(strings.NewReader(stdin)) + root.SetArgs(append([]string{"identity", "matches"}, args...)) + err := root.ExecuteContext(ctx) + return output.String(), err +} + +func TestIdentityMatchesCLIReviewUsesDaemonRoutes(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + type recorded struct { + method, path, query, body string + } + requests := make(chan recorded, 8) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + requests <- recorded{r.Method, r.URL.Path, r.URL.RawQuery, string(body)} + w.Header().Set("Content-Type", "application/json") + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v1/identity/match-candidates": + _, _ = fmt.Fprintf(w, `{"candidates":[%s],"limit":2,"offset":3}`, identityMatchesCLICandidate) + case r.Method == http.MethodGet && r.URL.Path == "/api/v1/identity/match-candidates/17": + _, _ = io.WriteString(w, identityMatchesCLICandidate) + case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/review/accept"): + if strings.Contains(string(body), `"stale-token"`) { + w.WriteHeader(http.StatusConflict) + _, _ = io.WriteString(w, `{"error":"identity_match_review_stale","message":"The match changed"}`) + return + } + _, _ = fmt.Fprintf(w, `{"candidate":%s,"identity_revision":9,"cache_state":"ready"}`, + strings.Replace(identityMatchesCLICandidate, `"state":"candidate"`, `"state":"accepted"`, 1)) + case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/review/reject"): + _, _ = fmt.Fprintf(w, `{"candidate":%s,"identity_revision":9,"cache_state":"ready"}`, + strings.Replace(identityMatchesCLICandidate, `"state":"candidate"`, `"state":"rejected"`, 1)) + default: + http.Error(w, "unexpected route", http.StatusNotFound) + } + })) + t.Cleanup(server.Close) + ctx := withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) + + out, err := runIdentityMatchesCLI(ctx, t, "", "list", "--state", "candidate", "--limit", "2", "--offset", "3", "--json") + require.NoError(err) + assert.Contains(out, `"review_token":"review-token-17"`) + list := <-requests + assert.Equal("/api/v1/identity/match-candidates", list.path) + assert.Contains(list.query, "offset=3") + assert.Contains(list.query, "limit=2") + + out, err = runIdentityMatchesCLI(ctx, t, "", "show", "17", "--json") + require.NoError(err) + assert.Contains(out, `"evidence"`) + assert.Equal("/api/v1/identity/match-candidates/17", (<-requests).path) + out, err = runIdentityMatchesCLI(ctx, t, "", "show", "17") + require.NoError(err) + assert.Contains(out, "Review token: review-token-17") + assert.Contains(out, "- email") + assert.Equal("/api/v1/identity/match-candidates/17", (<-requests).path) + + out, err = runIdentityMatchesCLI(ctx, t, "", "accept", "17", "--review-token", "review-token-17", "--json") + require.NoError(err) + assert.Contains(out, `"identity_revision":9`) + accepted := <-requests + assert.Equal("/api/v1/identity/match-candidates/17/review/accept", accepted.path) + var acceptedBody map[string]any + require.NoError(json.Unmarshal([]byte(accepted.body), &acceptedBody)) + assert.Equal("review-token-17", acceptedBody["review_token"]) + + _, err = runIdentityMatchesCLI(ctx, t, "Different people", "reject", "17", + "--review-token", "review-token-17", "--notes-stdin") + require.NoError(err) + rejected := <-requests + assert.Equal("/api/v1/identity/match-candidates/17/review/reject", rejected.path) + var rejectedBody map[string]any + require.NoError(json.Unmarshal([]byte(rejected.body), &rejectedBody)) + assert.Equal("Different people", rejectedBody["notes"]) + + notesPath := filepath.Join(t.TempDir(), "review-notes.txt") + require.NoError(os.WriteFile(notesPath, []byte("Confirmed from another source\n"), 0o600)) + out, err = runIdentityMatchesCLI(ctx, t, "", "accept", "17", + "--review-token", "review-token-17", "--notes-file", notesPath) + require.NoError(err) + assert.Contains(out, "Next: Inspect the resulting person and CardDAV publication.") + fromFile := <-requests + var fileBody map[string]any + require.NoError(json.Unmarshal([]byte(fromFile.body), &fileBody)) + assert.Equal("Confirmed from another source", fileBody["notes"]) + + _, err = runIdentityMatchesCLI(ctx, t, "", "accept", "17", + "--review-token", "stale-token") + require.Error(err) + var apiErr *daemonclient.APIError + require.ErrorAs(err, &apiErr) + assert.Equal(http.StatusConflict, apiErr.Status) + assert.Equal("identity_match_review_stale", apiErr.Code) + assert.Equal("/api/v1/identity/match-candidates/17/review/accept", (<-requests).path) + + _, err = runIdentityMatchesCLI(ctx, t, "", "accept", "17") + require.Error(err) + require.ErrorContains(err, "--review-token") + + _, err = runIdentityMatchesCLI(ctx, t, "", "show", "0") + require.Error(err) + require.ErrorContains(err, "positive") +} diff --git a/cmd/msgvault/cmd/identity_test.go b/cmd/msgvault/cmd/identity_test.go index c48ed35e8..ebc57921f 100644 --- a/cmd/msgvault/cmd/identity_test.go +++ b/cmd/msgvault/cmd/identity_test.go @@ -158,6 +158,9 @@ func TestRenderIdentityDiscoverProgressWrapsWriterError(t *testing.T) { } func TestIdentityDiscoverProviderSourceIDApplyConfirmJSONUsesHTTPAndSuppressesProgress(t *testing.T) { + cfg := testConfigValue() + useLocal := false + assertions := assert.New(t) requirements := require.New(t) dataDir := t.TempDir() @@ -210,10 +213,14 @@ func TestIdentityDiscoverProviderSourceIDApplyConfirmJSONUsesHTTPAndSuppressesPr Data: config.DataConfig{DataDir: dataDir}, Remote: config.RemoteConfig{URL: "http://configured-daemonclient.invalid"}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true var stdout, stderr bytes.Buffer root := newTestRootCmd() + root.SetContext(testCtx) root.SetOut(&stdout) root.SetErr(&stderr) root.AddCommand(identityCmd) @@ -397,6 +404,9 @@ func TestIdentityImportJSONFileApplyUsesSourceIDAndStableOutput(t *testing.T) { // identity subcommand tests. Returns (store, root, stdout buffer, stderr buffer). func newIdentityCLITest(t *testing.T) (*store.Store, *cobra.Command, *bytes.Buffer, *bytes.Buffer) { t.Helper() + cfg := testConfigValue() + logger := testLoggerValue() + useLocal := false tmpDir := t.TempDir() dbPath := filepath.Join(tmpDir, "msgvault.db") @@ -484,12 +494,16 @@ func newIdentityCLITest(t *testing.T) (*store.Store, *cobra.Command, *bytes.Buff Data: config.DataConfig{DataDir: tmpDir}, Remote: config.RemoteConfig{URL: "http://configured-daemonclient.invalid"}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true logger = slog.New(slog.DiscardHandler) startStoreAPIDaemon(t, tmpDir, s, nil) var stdout, stderr bytes.Buffer root := newTestRootCmd() + root.SetContext(testCtx) root.SetOut(&stdout) root.SetErr(&stderr) root.AddCommand(identityCmd) @@ -498,6 +512,9 @@ func newIdentityCLITest(t *testing.T) (*store.Store, *cobra.Command, *bytes.Buff } func TestIdentityListUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) dataDir := t.TempDir() @@ -518,12 +535,16 @@ func TestIdentityListUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { Data: config.DataConfig{DataDir: dataDir}, Remote: config.RemoteConfig{URL: "http://configured-daemonclient.invalid"}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true identityListJSON = false var stdout bytes.Buffer var stderr bytes.Buffer cmd := &cobra.Command{Use: "list", RunE: runIdentityList} + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetErr(&stderr) @@ -540,6 +561,9 @@ func TestIdentityListUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { } func TestIdentityShowUsesLocalDaemonHTTPAndPreservesHint(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) dataDir := t.TempDir() @@ -560,7 +584,10 @@ func TestIdentityShowUsesLocalDaemonHTTPAndPreservesHint(t *testing.T) { Data: config.DataConfig{DataDir: dataDir}, Remote: config.RemoteConfig{URL: "http://configured-daemonclient.invalid"}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true identityShowJSON = false var stdout bytes.Buffer @@ -570,6 +597,7 @@ func TestIdentityShowUsesLocalDaemonHTTPAndPreservesHint(t *testing.T) { Args: identityShowCmd.Args, RunE: runIdentityShow, } + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetErr(&stderr) cmd.SetArgs([]string{"empty@example.com"}) @@ -584,6 +612,9 @@ func TestIdentityShowUsesLocalDaemonHTTPAndPreservesHint(t *testing.T) { } func TestIdentityAddUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) dataDir := t.TempDir() @@ -604,7 +635,10 @@ func TestIdentityAddUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { Data: config.DataConfig{DataDir: dataDir}, Remote: config.RemoteConfig{URL: "http://configured-daemonclient.invalid"}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true identityAddSignal = "manual" var stdout bytes.Buffer @@ -614,6 +648,7 @@ func TestIdentityAddUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { Args: identityAddCmd.Args, RunE: runIdentityAdd, } + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetErr(&stderr) cmd.SetArgs([]string{"alice@example.com", "extra@example.com"}) @@ -629,6 +664,9 @@ func TestIdentityAddUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { } func TestIdentityRemoveUsesLocalDaemonHTTPAndPreservesWarning(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) dataDir := t.TempDir() @@ -647,7 +685,10 @@ func TestIdentityRemoveUsesLocalDaemonHTTPAndPreservesWarning(t *testing.T) { Data: config.DataConfig{DataDir: dataDir}, Remote: config.RemoteConfig{URL: "http://configured-daemonclient.invalid"}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true var stdout bytes.Buffer var stderr bytes.Buffer @@ -656,6 +697,7 @@ func TestIdentityRemoveUsesLocalDaemonHTTPAndPreservesWarning(t *testing.T) { Args: identityRemoveCmd.Args, RunE: runIdentityRemove, } + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetErr(&stderr) cmd.SetArgs([]string{"alice@example.com", "alice@example.com"}) diff --git a/cmd/msgvault/cmd/imap_folder_state_sync_test.go b/cmd/msgvault/cmd/imap_folder_state_sync_test.go index c3470f90b..19bfea95e 100644 --- a/cmd/msgvault/cmd/imap_folder_state_sync_test.go +++ b/cmd/msgvault/cmd/imap_folder_state_sync_test.go @@ -294,7 +294,7 @@ func TestIMAPFolderStateOptions_RoundTripSkipsUnchangedFolders(t *testing.T) { beforeMemberships := imapMembershipRowCount(t, st, src.ID) require.NotZero(beforeMemberships) - opts := imapFolderStateOptions(st, src, false) + opts := imapFolderStateOptions(st, src, false, testConfigValue(), testDiscardLogger()) require.NotEmpty(opts, "saved states must produce a client option") second := listedIMAPClient(t, addr, opts...) @@ -332,7 +332,7 @@ func TestIMAPFolderStateOptions_ForceRescanRetainsStatesAndEnumerates(t *testing context.Background(), st, src, first, completedIMAPSyncSummary(t, st, src), 0)) require.NoError(first.Close()) - opts := imapFolderStateOptions(st, src, true) + opts := imapFolderStateOptions(st, src, true, testConfigValue(), testDiscardLogger()) second := listedIMAPClient(t, addr, opts...) ctx, cancel := context.WithTimeout( @@ -619,7 +619,7 @@ func TestSaveIMAPFolderStates_RepublishGoneUIDRecoversByMessageCount(t *testing. // skipped, and reading it again restores both the membership and the // message. hideUID(false) - third, thirdLabels := syncedIMAPClient(t, addr, imapFolderStateOptions(st, src, false)...) + third, thirdLabels := syncedIMAPClient(t, addr, imapFolderStateOptions(st, src, false, testConfigValue(), testDiscardLogger())...) // The recovery is the count check, not another republish. The saved // baseline holds one UID and the server reports two, so the mailbox is // read again and the one missing UID is diffed back in. diff --git a/cmd/msgvault/cmd/imap_legacy_message_id_test.go b/cmd/msgvault/cmd/imap_legacy_message_id_test.go new file mode 100644 index 000000000..fe9732979 --- /dev/null +++ b/cmd/msgvault/cmd/imap_legacy_message_id_test.go @@ -0,0 +1,80 @@ +package cmd + +import ( + "fmt" + "testing" + + imapapi "github.com/emersion/go-imap/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/gmail" + imaplib "go.kenn.io/msgvault/internal/imap" + msgsync "go.kenn.io/msgvault/internal/sync" + "go.kenn.io/msgvault/internal/testutil" +) + +func TestIMAPLegacyMessageIDFullSyncPersistsMemberships(t *testing.T) { + for _, header := range []string{"123456789", "<[legacy-token==@example.test]>"} { + t.Run(header, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + addr, user := testutil.StartIMAPMemServerWithSpecialUse(t, + map[string]int{"All Mail": 0, "INBOX": 0}, + map[string][]imapapi.MailboxAttr{"All Mail": {imapapi.MailboxAttrAll}}, + ) + raw := []byte(fmt.Sprintf("From: sender@example.test\r\nTo: recipient@example.test\r\n"+ + "Date: Mon, 1 Jan 2024 00:00:00 +0000\r\nMessage-ID: %s\r\n"+ + "Subject: Synthetic legacy identifier\r\n\r\nSynthetic body.\r\n", header)) + for _, mailbox := range []string{"All Mail", "INBOX"} { + testutil.AppendIMAPRawMessage(t, user, mailbox, raw) + } + st := testutil.NewTestStore(t) + source, err := st.GetOrCreateSource(sourceTypeIMAP, "imap://legacy@example.test") + require.NoError(err) + initial := newScriptedRFC7162Client(t, addr, imaplib.WithFolderFilter([]string{"INBOX"}, nil)) + initialOptions := msgsync.DefaultOptions() + initialOptions.SourceType = sourceTypeIMAP + initialSummary, err := newMessageSyncer(initial, st, initialOptions, + testInvocationWithConfig(testConfigValue())).Full(t.Context(), source.Identifier) + require.NoError(err) + require.Zero(initialSummary.Errors) + require.NoError(initial.Close()) + + // After a filtered import, both a full scan and a repeated refresh must + // publish the same two mailbox memberships for a single archive row. + for attempt := range 2 { + client := newScriptedRFC7162Client(t, addr, imapFolderStateOptions(st, source, true, testConfigValue(), testDiscardLogger())...) + options := msgsync.DefaultOptions() + options.SourceType = sourceTypeIMAP + options.NoResume = true + summary, err := newMessageSyncer(client, st, options, + testInvocationWithConfig(testConfigValue())). + FullWithFinalizer(t.Context(), source, func(summary *gmail.SyncSummary) error { + return saveIMAPFolderStates(t.Context(), st, source, client, summary, 0) + }) + require.NoError(err, "sync attempt %d", attempt+1) + require.Zero(summary.Errors) + assert.Equal(2, imapMembershipRowCount(t, st, source.ID)) + + var count int + require.NoError(st.DB().QueryRow(st.Rebind( + `SELECT COUNT(*) FROM messages WHERE source_id = ?`), source.ID).Scan(&count)) + assert.Equal(1, count) + var allMailID, inboxID int64 + for mailbox, target := range map[string]*int64{"All Mail": &allMailID, "INBOX": &inboxID} { + require.NoError(st.DB().QueryRow(st.Rebind( + `SELECT message_id FROM imap_message_memberships WHERE source_id = ? AND mailbox = ? AND uid = 1`, + ), source.ID, mailbox).Scan(target)) + } + assert.Equal(allMailID, inboxID) + storedRaw, err := st.GetMessageRaw(allMailID) + require.NoError(err) + assert.Equal(raw, storedRaw) + states, err := loadIMAPFolderStates(st, source.ID) + require.NoError(err) + assert.Len(states, 2) + require.NoError(client.Close()) + } + }) + } +} diff --git a/cmd/msgvault/cmd/imap_qresync_integration_test.go b/cmd/msgvault/cmd/imap_qresync_integration_test.go index 4987b9854..a62ab1da5 100644 --- a/cmd/msgvault/cmd/imap_qresync_integration_test.go +++ b/cmd/msgvault/cmd/imap_qresync_integration_test.go @@ -18,6 +18,7 @@ import ( imapapi "github.com/emersion/go-imap/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/imap" "go.kenn.io/msgvault/internal/store" msgsync "go.kenn.io/msgvault/internal/sync" @@ -25,13 +26,15 @@ import ( ) type scriptedRFC7162Message struct { - UID imapapi.UID - MessageID string - Subject string - Body string - MissingRaw bool - Flags []imapapi.Flag - ModSeq uint64 + UID imapapi.UID + MessageID string + // MessageIDHeader overrides the header value for damaged legacy IDs. + MessageIDHeader string + Subject string + Body string + MissingRaw bool + Flags []imapapi.Flag + ModSeq uint64 // Raw overrides the synthesized message bytes verbatim (already CRLF // encoded) so tests can vary recipients, attachments, and HTML. Raw string @@ -402,8 +405,8 @@ func writeScriptedRFC7162Fetch( } if headerOnly { body := "\r\n" - if message.MessageID != "" { - body = fmt.Sprintf("Message-ID: <%s>\r\n\r\n", message.MessageID) + if header := scriptedRFC7162MessageIDHeader(message); header != "" { + body = header + "\r\n" } _, _ = fmt.Fprintf(w, "* %d FETCH (UID %d FLAGS (%s)%s BODY[HEADER.FIELDS (MESSAGE-ID)] {%d}\r\n%s)\r\n", @@ -429,10 +432,7 @@ func scriptedRFC7162RawMessage(message scriptedRFC7162Message) string { if subject == "" { subject = "Synthetic message" } - messageIDHeader := "" - if message.MessageID != "" { - messageIDHeader = fmt.Sprintf("Message-ID: <%s>\r\n", message.MessageID) - } + messageIDHeader := scriptedRFC7162MessageIDHeader(message) body := message.Body if body == "" { body = "Synthetic body." @@ -442,6 +442,16 @@ func scriptedRFC7162RawMessage(message scriptedRFC7162Message) string { messageIDHeader, subject, body) } +func scriptedRFC7162MessageIDHeader(message scriptedRFC7162Message) string { + if message.MessageIDHeader != "" { + return "Message-ID: " + message.MessageIDHeader + "\r\n" + } + if message.MessageID != "" { + return fmt.Sprintf("Message-ID: <%s>\r\n", message.MessageID) + } + return "" +} + func newScriptedRFC7162Client( t *testing.T, addr string, @@ -464,17 +474,19 @@ func runScriptedRFC7162Sync( st *store.Store, identifier string, addr string, + contexts ...context.Context, ) (*imap.Client, *store.Source, error) { t.Helper() + testCtx := scriptedSyncTestContext(t, contexts...) source, err := st.GetOrCreateSource(sourceTypeIMAP, identifier) require.NoError(t, err) - client := newScriptedRFC7162Client(t, addr, imapFolderStateOptions(st, source, false)...) + client := newScriptedRFC7162Client(t, addr, imapFolderStateOptionsForTest(testCtx, st, source, false)...) options := msgsync.DefaultOptions() options.SourceType = sourceTypeIMAP options.NoResume = true - summary, err := newMessageSyncer(client, st, options). + summary, err := newMessageSyncer(client, st, options, invocationFromContext(testCtx)). WithLogger(slog.New(slog.DiscardHandler)). - Full(t.Context(), identifier) + Full(testCtx, identifier) if err != nil { return client, source, err } @@ -483,7 +495,31 @@ func runScriptedRFC7162Sync( "scripted IMAP sync completed with %d errors", summary.Errors) } return client, source, saveIMAPFolderStates( - context.Background(), st, source, client, summary, options.Limit) + testCtx, st, source, client, summary, options.Limit) +} + +func scriptedSyncTestContext(t *testing.T, contexts ...context.Context) context.Context { + t.Helper() + if len(contexts) > 0 && contexts[0] != nil { + if invocationFromContext(contexts[0]) != nil { + return contexts[0] + } + return testInvocationContext(contexts[0], config.NewDefaultConfig(), invocationOptions{}) + } + return testInvocationContext(t.Context(), config.NewDefaultConfig(), invocationOptions{}) +} + +func imapFolderStateOptionsForTest( + testCtx context.Context, + st *store.Store, + source *store.Source, + forceRescan bool, +) []imap.Option { + var cfg *config.Config + if state := invocationFromContext(testCtx); state != nil { + cfg = state.cfg + } + return imapFolderStateOptions(st, source, forceRescan, cfg, testDiscardLogger()) } func requireScriptedRFC7162Sync( @@ -491,9 +527,10 @@ func requireScriptedRFC7162Sync( st *store.Store, identifier string, addr string, + contexts ...context.Context, ) (*imap.Client, *store.Source) { t.Helper() - client, source, err := runScriptedRFC7162Sync(t, st, identifier, addr) + client, source, err := runScriptedRFC7162Sync(t, st, identifier, addr, contexts...) require.NoError(t, err) return client, source } @@ -645,6 +682,85 @@ func queryScriptedRFC7162MessageLabels( return labels } +func TestIMAPQresyncEndToEndLegacyMessageIDBaseline(t *testing.T) { + for _, test := range []struct { + name, header, stored string + }{ + {"missing closing bracket", "", ` type="multipart/alternative"`}, + } { + t.Run(test.name, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + message := newScriptedRFC7162Message(1, "") + message.MessageIDHeader = test.header + message.Subject = "Archived subject" + message.Body = "Archived body" + archivedRaw := []byte(scriptedRFC7162RawMessage(message)) + duplicate := message + duplicate.UID = 2 + baseline := scriptedRFC7162Snapshot{ + Capabilities: scriptedRFC7162Capabilities(), + Mailboxes: []scriptedRFC7162Mailbox{ + {Name: "Archive", UIDValidity: 77, UIDNext: 3, HighestModSeq: 10, Messages: []scriptedRFC7162Message{message, duplicate}}, + scriptedRFC7162Inbox(77, 2, 10, message), + }, + } + addr, server := startScriptedRFC7162Server(t, baseline) + st := testutil.NewTestStore(t) + const identifier = "imap://legacy-baseline@example.test" + source, err := st.GetOrCreateSource(sourceTypeIMAP, identifier) + requirements.NoError(err) + conversation, err := st.EnsureConversation(source.ID, "legacy", "Archived subject") + requirements.NoError(err) + messageID, err := st.UpsertMessage(&store.Message{ + SourceID: source.ID, ConversationID: conversation, SourceMessageID: "Archive|1", + RFC822MessageID: sql.NullString{String: test.stored, Valid: true}, MessageType: "email", + }) + requirements.NoError(err) + requirements.NoError(st.UpsertMessageBody(messageID, sql.NullString{String: "Archived body", Valid: true}, sql.NullString{})) + requirements.NoError(st.UpsertMessageRaw(messageID, archivedRaw)) + states, err := st.GetIMAPFolderStates(source.ID) + requirements.NoError(err) + requirements.Empty(states) + + first, _ := requireScriptedRFC7162Sync(t, st, identifier, addr) + requirements.NoError(first.Close()) + known, err := st.GetIMAPKnownUIDs(source.ID) + requirements.NoError(err) + assertions.Equal(map[string][]uint32{"Archive": {1, 2}, "INBOX": {1}}, known) + states, err = st.GetIMAPFolderStates(source.ID) + requirements.NoError(err) + assertions.ElementsMatch([]store.IMAPFolderState{ + {Mailbox: "Archive", UIDValidity: 77, UIDNext: 3, HighestModSeq: 10}, + {Mailbox: "INBOX", UIDValidity: 77, UIDNext: 2, HighestModSeq: 10}, + }, states) + assertions.Contains(server.commandsFor(1), "UID SEARCH") + + second, _ := requireScriptedRFC7162Sync(t, st, identifier, addr) + requirements.NoError(second.Close()) + assertions.Contains(server.commandsFor(2), "CHANGEDSINCE 10 VANISHED") + assertions.NotContains(server.commandsFor(2), "UID SEARCH") + var gotID int64 + var stored string + requirements.NoError(st.DB().QueryRow(st.Rebind(`SELECT id, rfc822_message_id FROM messages WHERE source_id = ?`), source.ID).Scan(&gotID, &stored)) + assertions.Equal(messageID, gotID) + assertions.Equal(test.stored, stored) + var count, memberships int + requirements.NoError(st.DB().QueryRow(st.Rebind(`SELECT COUNT(*) FROM messages WHERE source_id = ?`), source.ID).Scan(&count)) + assertions.Equal(1, count) + requirements.NoError(st.DB().QueryRow(st.Rebind(`SELECT COUNT(*) FROM imap_message_memberships WHERE source_id = ? AND message_id = ?`), source.ID, messageID).Scan(&memberships)) + assertions.Equal(3, memberships) + body, err := st.GetMessageBodyText(messageID) + requirements.NoError(err) + assertions.Equal("Archived body", body) + raw, err := st.GetMessageRaw(messageID) + requirements.NoError(err) + assertions.Equal(archivedRaw, raw) + }) + } +} + func TestIMAPQresyncEndToEndAppend(t *testing.T) { requirements := require.New(t) assertions := assert.New(t) @@ -791,11 +907,12 @@ func TestIMAPQresyncEndToEndRetiresChangedMailboxTopology(t *testing.T) { Mailboxes: []scriptedRFC7162Mailbox{}, }) second := newScriptedRFC7162Client( - t, addr, imapFolderStateOptions(st, source, false)...) + t, addr, imapFolderStateOptions(st, source, false, testConfigValue(), testDiscardLogger())...) options := msgsync.DefaultOptions() options.SourceType = sourceTypeIMAP options.NoResume = true - summary, err := newMessageSyncer(second, st, options). + summary, err := newMessageSyncer(second, st, options, + testInvocationWithConfig(testConfigValue())). WithLogger(slog.New(slog.DiscardHandler)). Full(t.Context(), identifier) requirements.NoError(err) @@ -890,12 +1007,13 @@ func TestIMAPQresyncEndToEndLimitedRunPreservesOverlappingMailboxLabels(t *testi }, }) limitedClient := newScriptedRFC7162Client( - t, addr, imapFolderStateOptions(st, source, false)...) + t, addr, imapFolderStateOptions(st, source, false, testConfigValue(), testDiscardLogger())...) options := msgsync.DefaultOptions() options.SourceType = sourceTypeIMAP options.NoResume = true options.Limit = 1 - summary, err := newMessageSyncer(limitedClient, st, options). + summary, err := newMessageSyncer(limitedClient, st, options, + testInvocationWithConfig(testConfigValue())). WithLogger(slog.New(slog.DiscardHandler)).Full(t.Context(), identifier) requirements.NoError(err) requirements.NoError(saveIMAPFolderStates( @@ -1006,11 +1124,12 @@ func TestIMAPQresyncEndToEndReplaysAfterFailedApplication(t *testing.T) { Capabilities: scriptedRFC7162Capabilities(), Mailboxes: []scriptedRFC7162Mailbox{changed}, }) failedClient := newScriptedRFC7162Client( - t, addr, imapFolderStateOptions(st, source, false)...) + t, addr, imapFolderStateOptions(st, source, false, testConfigValue(), testDiscardLogger())...) options := msgsync.DefaultOptions() options.SourceType = sourceTypeIMAP options.NoResume = true - summary, err := newMessageSyncer(failedClient, st, options). + summary, err := newMessageSyncer(failedClient, st, options, + testInvocationWithConfig(testConfigValue())). WithLogger(slog.New(slog.DiscardHandler)).Full(t.Context(), identifier) requirements.NoError(err) _, err = st.DB().Exec(st.Rebind(` @@ -1060,11 +1179,12 @@ func TestIMAPQresyncEndToEndFailedMoveApplyPreservesBaselineLabels(t *testing.T) }, }) failedClient := newScriptedRFC7162Client( - t, addr, imapFolderStateOptions(st, source, false)...) + t, addr, imapFolderStateOptions(st, source, false, testConfigValue(), testDiscardLogger())...) options := msgsync.DefaultOptions() options.SourceType = sourceTypeIMAP options.NoResume = true - summary, err := newMessageSyncer(failedClient, st, options). + summary, err := newMessageSyncer(failedClient, st, options, + testInvocationWithConfig(testConfigValue())). WithLogger(slog.New(slog.DiscardHandler)).Full(t.Context(), identifier) requirements.NoError(err) installScriptedRFC7162ApplyFailureTrigger(t, st) diff --git a/cmd/msgvault/cmd/imap_relocation_integration_test.go b/cmd/msgvault/cmd/imap_relocation_integration_test.go index c299d0c42..0421d1615 100644 --- a/cmd/msgvault/cmd/imap_relocation_integration_test.go +++ b/cmd/msgvault/cmd/imap_relocation_integration_test.go @@ -79,14 +79,19 @@ func newRelocationFixture(t *testing.T) relocationFixture { func (f relocationFixture) sync(t *testing.T, extra ...imap.Option) (*imap.Client, error) { t.Helper() - opts := append(imapFolderStateOptions(f.st, f.source, false), extra...) + testCtx := scriptedSyncTestContext(t) + var cfg *config.Config + if state := invocationFromContext(testCtx); state != nil { + cfg = state.cfg + } + opts := append(imapFolderStateOptions(f.st, f.source, false, cfg, testDiscardLogger()), extra...) client := newScriptedRFC7162Client(t, f.addr, opts...) options := msgsync.DefaultOptions() options.SourceType = sourceTypeIMAP options.NoResume = true - summary, err := newMessageSyncer(client, f.st, options).WithLogger(slog.New(slog.DiscardHandler)).FullWithFinalizer( - t.Context(), f.source, func(summary *gmail.SyncSummary) error { - return saveIMAPFolderStates(t.Context(), f.st, f.source, client, summary, options.Limit) + summary, err := newMessageSyncer(client, f.st, options, invocationFromContext(testCtx)).WithLogger(slog.New(slog.DiscardHandler)).FullWithFinalizer( + testCtx, f.source, func(summary *gmail.SyncSummary) error { + return saveIMAPFolderStates(testCtx, f.st, f.source, client, summary, options.Limit) }) if err == nil && summary.Errors != 0 { err = fmt.Errorf("relocation sync completed with %d errors", summary.Errors) @@ -521,11 +526,15 @@ func TestIMAPRelocationSkipsSameCompositeForcedCandidate(t *testing.T) { // external senders cannot produce; every other adoption keeps the canonical // snapshot and only rekeys the location, matching pre-relocation behavior. func TestIMAPRelocationForgedSurvivorPreservesSnapshot(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Sync.ArchiveRemoteImages = true victim := newScriptedRFC7162Message(1, "forged-survivor@example.test", imapapi.FlagSeen) victim.Body = "victimoriginalword" @@ -822,20 +831,22 @@ func forgerWithUID(message scriptedRFC7162Message, uid imapapi.UID) scriptedRFC7 func runScriptedRelocationSync( t *testing.T, st *store.Store, identifier, addr string, mod func(*msgsync.Options), + contexts ...context.Context, ) (*imap.Client, *store.Source, error) { t.Helper() + testCtx := scriptedSyncTestContext(t, contexts...) source, err := st.GetOrCreateSource(sourceTypeIMAP, identifier) require.NoError(t, err) - client := newScriptedRFC7162Client(t, addr, imapFolderStateOptions(st, source, false)...) + client := newScriptedRFC7162Client(t, addr, imapFolderStateOptionsForTest(testCtx, st, source, false)...) options := msgsync.DefaultOptions() options.SourceType = sourceTypeIMAP options.NoResume = true if mod != nil { mod(options) } - summary, err := newMessageSyncer(client, st, options). + summary, err := newMessageSyncer(client, st, options, invocationFromContext(testCtx)). WithLogger(slog.New(slog.DiscardHandler)). - Full(t.Context(), identifier) + Full(testCtx, identifier) if err != nil { return client, source, err } @@ -844,7 +855,7 @@ func runScriptedRelocationSync( "scripted IMAP sync completed with %d errors", summary.Errors) } return client, source, saveIMAPFolderStates( - context.Background(), st, source, client, summary, options.Limit) + testCtx, st, source, client, summary, options.Limit) } func scriptedOutgoingRaw( @@ -976,12 +987,16 @@ func TestIMAPRelocationDraftsSentAllMailTopology(t *testing.T) { // trusted_imap_sent_mailboxes configuration restores the edited-copy // refresh through the same production wiring. func TestIMAPRelocationConfiguredTrustedOutgoingMailbox(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) const identifier = "imap://configured-trust@example.test" cfg = config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Sync.TrustedIMAPSentMailboxes = map[string][]string{ identifier: {"Gesendete Elemente"}, } @@ -997,7 +1012,7 @@ func TestIMAPRelocationConfiguredTrustedOutgoingMailbox(t *testing.T) { }} addr, server := startScriptedRFC7162Server(t, baseline) st := testutil.NewTestStore(t) - first, source := requireScriptedRFC7162Sync(t, st, identifier, addr) + first, source := requireScriptedRFC7162Sync(t, st, identifier, addr, testCtx) require.NoError(first.Close()) id, err := st.GetMessageIDByRFC822ID(source.ID, "<"+draft.MessageID+">") require.NoError(err) @@ -1012,7 +1027,7 @@ func TestIMAPRelocationConfiguredTrustedOutgoingMailbox(t *testing.T) { edited.Mailboxes[1].ChangedUIDs = []imapapi.UID{1} edited.Mailboxes[1].Messages = []scriptedRFC7162Message{sent} server.setSnapshot(edited) - second, _, err := runScriptedRFC7162Sync(t, st, identifier, addr) + second, _, err := runScriptedRFC7162Sync(t, st, identifier, addr, testCtx) require.NoError(err) require.NoError(second.Close()) message, err := st.GetMessage(id) @@ -1096,15 +1111,16 @@ func runScriptedRelocationResync( t *testing.T, st *store.Store, identifier, addr string, ) (*imap.Client, error) { t.Helper() + testCtx := scriptedSyncTestContext(t) source, err := st.GetOrCreateSource(sourceTypeIMAP, identifier) require.NoError(t, err) - client := newScriptedRFC7162Client(t, addr, imapFolderStateOptions(st, source, true)...) + client := newScriptedRFC7162Client(t, addr, imapFolderStateOptionsForTest(testCtx, st, source, true)...) options := msgsync.DefaultOptions() options.SourceType = sourceTypeIMAP options.NoResume = true - summary, err := newMessageSyncer(client, st, options). + summary, err := newMessageSyncer(client, st, options, invocationFromContext(testCtx)). WithLogger(slog.New(slog.DiscardHandler)). - Full(t.Context(), identifier) + Full(testCtx, identifier) if err != nil { return client, err } @@ -1113,7 +1129,7 @@ func runScriptedRelocationResync( "scripted IMAP resync completed with %d errors", summary.Errors) } return client, saveIMAPFolderStates( - context.Background(), st, source, client, summary, options.Limit) + testCtx, st, source, client, summary, options.Limit) } // Force-full rescans and realistic mailbox rebuilds — including an All Mail @@ -1264,6 +1280,8 @@ func TestIMAPRelocationRescanOrderingKeepsEditedSent(t *testing.T) { // conflicting advertised role or the INBOX name: those placements are denied // exactly as when they are advertised without configuration. func TestIMAPRelocationConfiguredConflictDenied(t *testing.T) { + cfg := testConfigValue() + for _, mode := range []string{"sent-plus-all", "all-only", "inbox"} { t.Run(mode, func(t *testing.T) { assert := assert.New(t) @@ -1271,6 +1289,7 @@ func TestIMAPRelocationConfiguredConflictDenied(t *testing.T) { savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) draft := newScriptedRFC7162Message(1, "configured-conflict@example.test", imapapi.FlagDraft) draft.Body = "draftoriginalword" @@ -1296,7 +1315,7 @@ func TestIMAPRelocationConfiguredConflictDenied(t *testing.T) { }} addr, server := startScriptedRFC7162Server(t, baseline) st := testutil.NewTestStore(t) - first, source := requireScriptedRFC7162Sync(t, st, identifier, addr) + first, source := requireScriptedRFC7162Sync(t, st, identifier, addr, testCtx) require.NoError(first.Close()) id, err := st.GetMessageIDByRFC822ID(source.ID, "<"+draft.MessageID+">") require.NoError(err) @@ -1311,7 +1330,7 @@ func TestIMAPRelocationConfiguredConflictDenied(t *testing.T) { edited.Mailboxes[1].ChangedUIDs = []imapapi.UID{1} edited.Mailboxes[1].Messages = []scriptedRFC7162Message{sent} server.setSnapshot(edited) - second, _, err := runScriptedRFC7162Sync(t, st, identifier, addr) + second, _, err := runScriptedRFC7162Sync(t, st, identifier, addr, testCtx) require.NoError(err) require.NoError(second.Close()) @@ -1335,20 +1354,21 @@ func runScriptedRelocationSyncSummary( t *testing.T, st *store.Store, identifier, addr string, ) (*gmail.SyncSummary, error) { t.Helper() + testCtx := scriptedSyncTestContext(t) source, err := st.GetOrCreateSource(sourceTypeIMAP, identifier) require.NoError(t, err) - client := newScriptedRFC7162Client(t, addr, imapFolderStateOptions(st, source, false)...) + client := newScriptedRFC7162Client(t, addr, imapFolderStateOptionsForTest(testCtx, st, source, false)...) options := msgsync.DefaultOptions() options.SourceType = sourceTypeIMAP options.NoResume = true - summary, err := newMessageSyncer(client, st, options). + summary, err := newMessageSyncer(client, st, options, invocationFromContext(testCtx)). WithLogger(slog.New(slog.DiscardHandler)). - Full(t.Context(), identifier) + Full(testCtx, identifier) if err != nil { return summary, err } return summary, saveIMAPFolderStates( - context.Background(), st, source, client, summary, options.Limit) + testCtx, st, source, client, summary, options.Limit) } // A persistently failing relocation target must not starve unrelated mail: @@ -1618,11 +1638,12 @@ func TestIMAPRelocationDeferredTargetInterruptedRunRestart(t *testing.T) { source2, err := st.GetSourceByIdentifier(identifier) require.NoError(err) interrupted := &cancelOnSecondListIMAP{Client: newScriptedRFC7162Client( - t, addr, imapFolderStateOptions(st, source2, false)...)} + t, addr, imapFolderStateOptions(st, source2, false, testConfigValue(), testDiscardLogger())...)} options := msgsync.DefaultOptions() options.SourceType = sourceTypeIMAP options.NoResume = true - _, err = newMessageSyncer(interrupted, st, options). + _, err = newMessageSyncer(interrupted, st, options, + testInvocationWithConfig(testConfigValue())). WithLogger(slog.New(slog.DiscardHandler)). Full(t.Context(), identifier) require.ErrorIs(err, context.Canceled) @@ -1639,8 +1660,9 @@ func TestIMAPRelocationDeferredTargetInterruptedRunRestart(t *testing.T) { // is deferred, and the guarded row is untouched. resumableOptions := msgsync.DefaultOptions() resumableOptions.SourceType = sourceTypeIMAP - resumable := newScriptedRFC7162Client(t, addr, imapFolderStateOptions(st, source2, false)...) - summary, err := newMessageSyncer(resumable, st, resumableOptions). + resumable := newScriptedRFC7162Client(t, addr, imapFolderStateOptions(st, source2, false, testConfigValue(), testDiscardLogger())...) + summary, err := newMessageSyncer(resumable, st, resumableOptions, + testInvocationWithConfig(testConfigValue())). WithLogger(slog.New(slog.DiscardHandler)). Full(t.Context(), identifier) require.NoError(err, "the retry attempt completes with per-item errors") @@ -1909,11 +1931,12 @@ func TestIMAPRelocationUntrustedAdoptionKeepsOldKeyOnLabelFailure(t *testing.T) src, srcErr := st.GetSourceByIdentifier(identifier) require.NoError(srcErr) client := &immediateLabelScriptedIMAP{Client: newScriptedRFC7162Client( - t, addr, imapFolderStateOptions(st, src, false)...)} + t, addr, imapFolderStateOptions(st, src, false, testConfigValue(), testDiscardLogger())...)} options := msgsync.DefaultOptions() options.SourceType = sourceTypeIMAP options.NoResume = true - summary, err := newMessageSyncer(client, st, options). + summary, err := newMessageSyncer(client, st, options, + testInvocationWithConfig(testConfigValue())). WithLogger(slog.New(slog.DiscardHandler)). Full(t.Context(), identifier) if closeErr := client.Close(); err == nil { @@ -2041,20 +2064,22 @@ func TestIMAPRelocationFullEnumerationRefreshesEditedSent(t *testing.T) { func runScriptedSourceSync( t *testing.T, st *store.Store, identifier, addr string, mod func(*msgsync.Options), + contexts ...context.Context, ) (*imap.Client, *store.Source, error) { t.Helper() + testCtx := scriptedSyncTestContext(t, contexts...) source, err := st.GetOrCreateSource(sourceTypeIMAP, identifier) require.NoError(t, err) - client := newScriptedRFC7162Client(t, addr, imapFolderStateOptions(st, source, false)...) + client := newScriptedRFC7162Client(t, addr, imapFolderStateOptionsForTest(testCtx, st, source, false)...) options := msgsync.DefaultOptions() options.SourceType = sourceTypeIMAP options.NoResume = true if mod != nil { mod(options) } - summary, err := newMessageSyncer(client, st, options). + summary, err := newMessageSyncer(client, st, options, invocationFromContext(testCtx)). WithLogger(slog.New(slog.DiscardHandler)). - Full(t.Context(), identifier) + Full(testCtx, identifier) if err != nil { return client, source, err } @@ -2063,7 +2088,7 @@ func runScriptedSourceSync( "scripted IMAP sync completed with %d errors", summary.Errors) } return client, source, saveIMAPFolderStates( - context.Background(), st, source, client, summary, options.Limit) + testCtx, st, source, client, summary, options.Limit) } // Sent-folder trust is scoped to the exact IMAP source identifier: two @@ -2074,11 +2099,15 @@ func runScriptedSourceSync( // snapshot, its raw MIME, participants, attachments, or search content, and // rejected bytes never reach remote-image processing. func TestIMAPRelocationSentTrustIsSourceScoped(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Sync.ArchiveRemoteImages = true const identifierA = "imap://scoped-a@example.test" const identifierB = "imap://scoped-b@example.test" @@ -2101,7 +2130,7 @@ func TestIMAPRelocationSentTrustIsSourceScoped(t *testing.T) { addrA, serverA := startScriptedRFC7162Server(t, baselineA) stA := testutil.NewTestStore(t) withAttachments := func(o *msgsync.Options) { o.AttachmentsDir = t.TempDir() } - firstA, sourceA, err := runScriptedSourceSync(t, stA, identifierA, addrA, withAttachments) + firstA, sourceA, err := runScriptedSourceSync(t, stA, identifierA, addrA, withAttachments, testCtx) require.NoError(err) require.NoError(firstA.Close()) idA, err := stA.GetMessageIDByRFC822ID(sourceA.ID, "<"+draftA.MessageID+">") @@ -2119,7 +2148,7 @@ func TestIMAPRelocationSentTrustIsSourceScoped(t *testing.T) { editedA.Mailboxes[2].UIDNext = 2 editedA.Mailboxes[2].Messages = []scriptedRFC7162Message{sentA} serverA.setSnapshot(editedA) - secondA, _, err := runScriptedSourceSync(t, stA, identifierA, addrA, withAttachments) + secondA, _, err := runScriptedSourceSync(t, stA, identifierA, addrA, withAttachments, testCtx) require.NoError(err) require.NoError(secondA.Close()) messageA, err := stA.GetMessage(idA) @@ -2150,7 +2179,7 @@ func TestIMAPRelocationSentTrustIsSourceScoped(t *testing.T) { }} addrB, serverB := startScriptedRFC7162Server(t, baselineB) stB := testutil.NewTestStore(t) - firstB, sourceB, err := runScriptedSourceSync(t, stB, identifierB, addrB, withAttachments) + firstB, sourceB, err := runScriptedSourceSync(t, stB, identifierB, addrB, withAttachments, testCtx) require.NoError(err) require.NoError(firstB.Close()) idB, err := stB.GetMessageIDByRFC822ID(sourceB.ID, "<"+victimB.MessageID+">") @@ -2170,7 +2199,7 @@ func TestIMAPRelocationSentTrustIsSourceScoped(t *testing.T) { replacedB.Mailboxes[1].ChangedUIDs = []imapapi.UID{2} replacedB.Mailboxes[1].Messages = []scriptedRFC7162Message{forgerB} serverB.setSnapshot(replacedB) - secondB, _, err := runScriptedSourceSync(t, stB, identifierB, addrB, withAttachments) + secondB, _, err := runScriptedSourceSync(t, stB, identifierB, addrB, withAttachments, testCtx) require.NoError(err) require.NoError(secondB.Close()) @@ -2202,11 +2231,15 @@ func TestIMAPRelocationSentTrustIsSourceScoped(t *testing.T) { // account's Sent folder and grant its duplicate copies the trusted dedup // bypass that would let a stale draft downgrade a fresh snapshot. func TestIMAPRelocationConfiguredDraftsRoleNotSent(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx const identifier = "imap://configured-drafts@example.test" cfg.Sync.TrustedIMAPSentMailboxes = map[string][]string{ identifier: {"My Drafts"}, @@ -2259,6 +2292,8 @@ func TestIMAPRelocationConfiguredDraftsRoleNotSent(t *testing.T) { // placements never gain that precedence, and a later stale Drafts copy must // not downgrade the refreshed snapshot. func TestIMAPRelocationSentOutranksStaleDraftsCanonical(t *testing.T) { + cfg := testConfigValue() + for _, mode := range []string{"qresync", "full enumeration", "configured localized sent", "removed sent mailbox", "removed sent mailbox full enumeration"} { t.Run(mode, func(t *testing.T) { assert := assert.New(t) @@ -2266,6 +2301,7 @@ func TestIMAPRelocationSentOutranksStaleDraftsCanonical(t *testing.T) { savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) draft := newScriptedRFC7162Message(1, "sent-precedence@example.test", imapapi.FlagDraft) draft.Body = "draftoriginalword" sent := newScriptedRFC7162Message(1, draft.MessageID, imapapi.FlagSeen) @@ -2291,7 +2327,7 @@ func TestIMAPRelocationSentOutranksStaleDraftsCanonical(t *testing.T) { addr, server := startScriptedRFC7162Server(t, baseline) st := testutil.NewTestStore(t) const identifier = "imap://sent-precedence@example.test" - first, source := requireScriptedRFC7162Sync(t, st, identifier, addr) + first, source := requireScriptedRFC7162Sync(t, st, identifier, addr, testCtx) require.NoError(first.Close()) id, err := st.GetMessageIDByRFC822ID(source.ID, "<"+draft.MessageID+">") require.NoError(err) @@ -2309,7 +2345,7 @@ func TestIMAPRelocationSentOutranksStaleDraftsCanonical(t *testing.T) { edited.Mailboxes[1].ChangedUIDs = []imapapi.UID{1} edited.Mailboxes[1].Messages = []scriptedRFC7162Message{sent} server.setSnapshot(edited) - second, _, err := runScriptedRFC7162Sync(t, st, identifier, addr) + second, _, err := runScriptedRFC7162Sync(t, st, identifier, addr, testCtx) require.NoError(err) require.NoError(second.Close()) @@ -2337,7 +2373,7 @@ func TestIMAPRelocationSentOutranksStaleDraftsCanonical(t *testing.T) { // A repeated run must not let the surviving stale Drafts copy // downgrade the refreshed snapshot. server.setSnapshot(edited) - third, _, err := runScriptedRFC7162Sync(t, st, identifier, addr) + third, _, err := runScriptedRFC7162Sync(t, st, identifier, addr, testCtx) require.NoError(err) require.NoError(third.Close()) message, err = st.GetMessage(id) @@ -2361,7 +2397,7 @@ func TestIMAPRelocationSentOutranksStaleDraftsCanonical(t *testing.T) { } for range 2 { server.setSnapshot(edited) - client, _, err := runScriptedRFC7162Sync(t, st, identifier, addr) + client, _, err := runScriptedRFC7162Sync(t, st, identifier, addr, testCtx) require.NoError(err) require.NoError(client.Close()) message, err = st.GetMessage(id) @@ -2385,11 +2421,15 @@ func TestIMAPRelocationSentOutranksStaleDraftsCanonical(t *testing.T) { // yields to a genuine Sent copy, while never itself gaining the Sent // placement's dedup bypass. func TestIMAPRelocationConfiguredDraftsCanonicalStillYieldsToSent(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx const identifier = "imap://configured-drafts-canonical@example.test" cfg.Sync.TrustedIMAPSentMailboxes = map[string][]string{ identifier: {"My Drafts"}, @@ -2477,6 +2517,8 @@ func seedLegacyDraftsCanonicalWithSentMembership( // it must neither authorize snapshot replacement nor gain the Sent placement // precedence or dedup bypass, including when the account lists it explicitly. func TestIMAPRelocationDualSentDraftsRoleDenied(t *testing.T) { + cfg := testConfigValue() + for _, mode := range []string{"advertised", "advertised and configured"} { t.Run(mode, func(t *testing.T) { assert := assert.New(t) diff --git a/cmd/msgvault/cmd/import.go b/cmd/msgvault/cmd/import.go index 89aa6aef2..527a466a1 100644 --- a/cmd/msgvault/cmd/import.go +++ b/cmd/msgvault/cmd/import.go @@ -47,6 +47,11 @@ Examples: } func runWhatsAppImport(cmd *cobra.Command, sourcePath string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg // Validate source file exists. if _, err := os.Stat(sourcePath); err != nil { return fmt.Errorf("source file not found: %w", err) @@ -67,7 +72,7 @@ func runWhatsAppImport(cmd *cobra.Command, sourcePath string) error { } } - s, cleanup, err := openWritableStoreAndInitForIngest() + s, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) if err != nil { return err } @@ -113,7 +118,7 @@ func runWhatsAppImport(cmd *cobra.Command, sourcePath string) error { if err != nil { if ctx.Err() != nil { fmt.Println("\nImport interrupted. Run again to continue.") - return rebuildCacheAfterWrite(dbPath) + return rebuildCacheAfterWrite(dbPath, state) } return fmt.Errorf("import failed: %w", err) } @@ -121,11 +126,11 @@ func runWhatsAppImport(cmd *cobra.Command, sourcePath string) error { // Auto-default-identity must run BEFORE the legacy migration // retry — see comment in account_identity.go. if !noDefaultIdentityImportWhatsApp && summary.SourceID != 0 { - confirmDefaultIdentity(cmd.OutOrStdout(), s, summary.SourceID, importPhone, importPhone, "phone-e164") + confirmDefaultIdentity(cmd.OutOrStdout(), s, summary.SourceID, importPhone, importPhone, "phone-e164", state.logger) } if summary.SourceID != 0 { - if err := runPostSourceCreateMigrations(s); err != nil { + if err := runPostSourceCreateMigrationsForInvocation(s, state); err != nil { return fmt.Errorf("post-source-create migrations: %w", err) } } @@ -163,7 +168,7 @@ func runWhatsAppImport(cmd *cobra.Command, sourcePath string) error { fmt.Printf(" Rate: %.0f messages/sec\n", rate) } - return rebuildCacheAfterWrite(dbPath) + return rebuildCacheAfterWrite(dbPath, state) } // ImportCLIProgress implements whatsapp.ImportProgress for terminal output. diff --git a/cmd/msgvault/cmd/import_eml.go b/cmd/msgvault/cmd/import_eml.go index c08fe4aac..7e73049ba 100644 --- a/cmd/msgvault/cmd/import_eml.go +++ b/cmd/msgvault/cmd/import_eml.go @@ -42,6 +42,12 @@ duplicate messages receive every mailbox label where they appear.`, return nil }, RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger if !isDaemonCLISubprocess() { return runDaemonCLICommandHTTPFromCobraWithLocalFiles(cmd, args, nil) } @@ -51,7 +57,7 @@ duplicate messages receive every mailbox label where they appear.`, ctx, stop := signal.NotifyContext(cmd.Context(), os.Interrupt, syscall.SIGTERM) defer stop() - st, cleanup, err := openWritableStoreAndInitForIngest() + st, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) if err != nil { return err } @@ -68,17 +74,17 @@ duplicate messages receive every mailbox label where they appear.`, NoResume: flags.noResume, CheckpointInterval: flags.checkpointInterval, AttachmentsDir: attachmentsDir, - RemoteImages: configuredRemoteImageFetcher(), + RemoteImages: configuredRemoteImageFetcher(cfg), Logger: logger, }) if importErr != nil { - return errors.Join(importErr, rebuildCacheAfterWrite(dbPath)) + return errors.Join(importErr, rebuildCacheAfterWrite(dbPath, state)) } if err := runEMLPostImportMigrations( - cmd.OutOrStdout(), st, summary, flags, + cmd.OutOrStdout(), st, summary, flags, state, ); err != nil { - return errors.Join(err, rebuildCacheAfterWrite(dbPath)) + return errors.Join(err, rebuildCacheAfterWrite(dbPath, state)) } out := cmd.OutOrStdout() @@ -93,7 +99,7 @@ duplicate messages receive every mailbox label where they appear.`, if resultErr == nil && summary.HardErrors { resultErr = fmt.Errorf("import completed with %d errors", summary.Errors) } - return errors.Join(resultErr, rebuildCacheAfterWrite(dbPath)) + return errors.Join(resultErr, rebuildCacheAfterWrite(dbPath, state)) }, } @@ -122,10 +128,15 @@ func runEMLPostImportMigrations( st *store.Store, summary *importer.EMLImportSummary, flags importEMLFlags, + state *invocation, ) error { if summary == nil || summary.SourceID == 0 { return nil } + state = invocationState(context.Background(), state) + if state == nil || state.logger == nil { + return errors.New("invocation state is required") + } // Establish the source identifier before retrying the legacy migration, // including after a partial import. Otherwise migrated legacy identities // can suppress the source's own identifier on the next resume. @@ -133,9 +144,10 @@ func runEMLPostImportMigrations( confirmDefaultIdentity( out, st, summary.SourceID, flags.identifier, flags.identifier, "account-identifier", + state.logger, ) } - if err := runPostSourceCreateMigrations(st); err != nil { + if err := runPostSourceCreateMigrationsForInvocation(st, state); err != nil { return fmt.Errorf("post-source-create migrations: %w", err) } return nil diff --git a/cmd/msgvault/cmd/import_eml_test.go b/cmd/msgvault/cmd/import_eml_test.go index 19aa1bcfe..2708f759f 100644 --- a/cmd/msgvault/cmd/import_eml_test.go +++ b/cmd/msgvault/cmd/import_eml_test.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "errors" "io" "os" @@ -10,6 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/importer" "go.kenn.io/msgvault/internal/store" ) @@ -17,6 +19,7 @@ import ( func TestImportEMLCommandRequiresIdentifier(t *testing.T) { t.Setenv(daemonCLISubprocessEnv, strconv.Itoa(os.Getppid())) cmd := newImportEMLCommand() + cmd.SetContext(testInvocationContext(context.Background(), config.NewDefaultConfig(), invocationOptions{})) cmd.SetArgs([]string{t.TempDir()}) err := cmd.Execute() @@ -39,7 +42,7 @@ func TestImportEMLCommandRefreshesCacheAfterPostMigrationFailure(t *testing.T) { dataDir := t.TempDir() testCfg := lifecycleTestConfig(dataDir) testCfg.Identity.Addresses = []string{"legacy@example.com"} - withStoreResolverConfig(t, testCfg) + testCtx := withStoreResolverConfig(t, testCfg) t.Setenv(daemonCLISubprocessEnv, strconv.Itoa(os.Getppid())) st, err := store.Open(testCfg.DatabaseDSN()) @@ -68,6 +71,7 @@ func TestImportEMLCommandRefreshesCacheAfterPostMigrationFailure(t *testing.T) { t.Cleanup(func() { buildCacheBeforeMessagesExportHook = nil }) cmd := newImportEMLCommand() + cmd.SetContext(testCtx) cmd.SetOut(io.Discard) cmd.SetErr(io.Discard) cmd.SetArgs([]string{ @@ -86,7 +90,7 @@ func TestRunEMLPostImportMigrationsConfirmsIdentityAfterHardErrors(t *testing.T) require := require.New(t) testCfg := lifecycleTestConfig(t.TempDir()) testCfg.Identity.Addresses = []string{"legacy@example.com"} - withStoreResolverConfig(t, testCfg) + testCtx := withStoreResolverConfig(t, testCfg) st, err := store.Open(testCfg.DatabaseDSN()) require.NoError(err, "open store") @@ -101,7 +105,7 @@ func TestRunEMLPostImportMigrationsConfirmsIdentityAfterHardErrors(t *testing.T) }, importEMLFlags{ identifier: "archive@example.com", sourceType: "eml", - }) + }, invocationFromContext(testCtx)) require.NoError(err, "post-import migrations") identities, err := st.ListAccountIdentities(src.ID) @@ -115,7 +119,8 @@ func TestRunEMLPostImportMigrationsConfirmsIdentityAfterHardErrors(t *testing.T) func TestRunEMLPostImportMigrationsSkipsDefaultIdentityForNonEmailSource(t *testing.T) { require := require.New(t) testCfg := lifecycleTestConfig(t.TempDir()) - withStoreResolverConfig(t, testCfg) + testCtx := withStoreResolverConfig(t, testCfg) + _ = testCtx st, err := store.Open(testCfg.DatabaseDSN()) require.NoError(err, "open store") @@ -129,7 +134,7 @@ func TestRunEMLPostImportMigrationsSkipsDefaultIdentityForNonEmailSource(t *test }, importEMLFlags{ identifier: "+15550001111", sourceType: "whatsapp", - }) + }, invocationFromContext(testCtx)) require.NoError(err, "post-import migrations") identities, err := st.ListAccountIdentities(src.ID) diff --git a/cmd/msgvault/cmd/import_emlx.go b/cmd/msgvault/cmd/import_emlx.go index 1df0b82f1..ca49673fa 100644 --- a/cmd/msgvault/cmd/import_emlx.go +++ b/cmd/msgvault/cmd/import_emlx.go @@ -62,6 +62,11 @@ Examples: `, Args: cobra.MaximumNArgs(2), RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg if !isDaemonCLISubprocess() { return runDaemonCLICommandHTTPFromCobra(cmd, args) } @@ -150,7 +155,7 @@ Examples: } }() - st, cleanup, err := openWritableStoreAndInitForIngest() + st, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) if err != nil { return err } @@ -171,7 +176,7 @@ Examples: importErr = importAutoAccounts(ctx, cmd, st, mailDir, attachmentsDir) } - return errors.Join(importErr, rebuildCacheAfterWrite(dbPath)) + return errors.Join(importErr, rebuildCacheAfterWrite(dbPath, state)) }, } @@ -181,6 +186,12 @@ func importSingleAccount( st *store.Store, mailDir, identifier, attachmentsDir string, ) error { + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger summary, err := importer.ImportEmlxDir( ctx, st, mailDir, importer.EmlxImportOptions{ SourceType: importEmlxSourceType, @@ -188,7 +199,7 @@ func importSingleAccount( NoResume: importEmlxNoResume, CheckpointInterval: importEmlxCheckpointInterval, AttachmentsDir: attachmentsDir, - RemoteImages: configuredRemoteImageFetcher(), + RemoteImages: configuredRemoteImageFetcher(cfg), Logger: logger, }, ) @@ -200,7 +211,7 @@ func importSingleAccount( // retry — see comment in account_identity.go. if ctx.Err() == nil && !summary.HardErrors && !noDefaultIdentityImportEmlx { if summary.SourceID != 0 { - confirmDefaultIdentity(cmd.OutOrStdout(), st, summary.SourceID, identifier, identifier, "account-identifier") + confirmDefaultIdentity(cmd.OutOrStdout(), st, summary.SourceID, identifier, identifier, "account-identifier", state.logger) } else { logger.Warn("auto-default-identity: missing source id", "identifier", identifier) } @@ -216,7 +227,7 @@ func importSingleAccount( // next invocation retries and prints the summary then. UX // polish tracked separately in // private/drafts/2026-05-02-issue-import-migration-error-ux.md. - if err := runPostSourceCreateMigrations(st); err != nil { + if err := runPostSourceCreateMigrationsForInvocation(st, state); err != nil { return fmt.Errorf("post-source-create migrations: %w", err) } } @@ -231,6 +242,12 @@ func importAutoAccounts( st *store.Store, mailDir, attachmentsDir string, ) error { + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger accountsDBPath := importEmlxAccountsDB if strings.HasPrefix(accountsDBPath, "~/") { home, _ := os.UserHomeDir() @@ -322,7 +339,7 @@ func importAutoAccounts( NoResume: importEmlxNoResume, CheckpointInterval: importEmlxCheckpointInterval, AttachmentsDir: attachmentsDir, - RemoteImages: configuredRemoteImageFetcher(), + RemoteImages: configuredRemoteImageFetcher(cfg), Logger: logger, }, ) @@ -344,7 +361,7 @@ func importAutoAccounts( accountDisplay = account.Email } if summary.SourceID != 0 { - confirmDefaultIdentity(cmd.OutOrStdout(), st, summary.SourceID, accountDisplay, identifier, "account-identifier") + confirmDefaultIdentity(cmd.OutOrStdout(), st, summary.SourceID, accountDisplay, identifier, "account-identifier", state.logger) } else { logger.Warn("auto-default-identity: missing source id", "identifier", identifier) } @@ -356,7 +373,7 @@ func importAutoAccounts( // above. UX polish tracked in // private/drafts/2026-05-02-issue-import-migration-error-ux.md // for a follow-up PR. - if err := runPostSourceCreateMigrations(st); err != nil { + if err := runPostSourceCreateMigrationsForInvocation(st, state); err != nil { importErrors = append(importErrors, fmt.Errorf("%s: post-source-create migrations: %w", identifier, err)) continue } @@ -373,6 +390,7 @@ func importAutoAccounts( grandTotal.MessagesUpdated += summary.MessagesUpdated grandTotal.MessagesSkipped += summary.MessagesSkipped grandTotal.PartialFiles += summary.PartialFiles + grandTotal.AttachmentsRestored += summary.AttachmentsRestored grandTotal.Errors += summary.Errors if summary.HardErrors { grandTotal.HardErrors = true @@ -449,8 +467,9 @@ func printImportStats(out io.Writer, summary importer.EmlxImportSummary) { ) if summary.PartialFiles > 0 { _, _ = fmt.Fprintf(out, - " Partial files: %d (body imported; attachments not cached by Apple Mail)\n", + " Partial files: %d (%d attachments restored from Apple Mail's Attachments/ directory)\n", summary.PartialFiles, + summary.AttachmentsRestored, ) } _, _ = fmt.Fprintf(out, diff --git a/cmd/msgvault/cmd/import_google_groups_test.go b/cmd/msgvault/cmd/import_google_groups_test.go index 040d56d89..c9679f749 100644 --- a/cmd/msgvault/cmd/import_google_groups_test.go +++ b/cmd/msgvault/cmd/import_google_groups_test.go @@ -13,6 +13,12 @@ import ( ) func TestImportMboxCmd_GoogleGroupsTakeout(t *testing.T) { + cfg := testConfigValue() + cfgFile := "" + homeDir := "" + logger := testLoggerValue() + verbose := false + markDaemonCLISubprocessForTest(t) require := require.New(t) diff --git a/cmd/msgvault/cmd/import_gvoice.go b/cmd/msgvault/cmd/import_gvoice.go index a1ccdc063..f8c101372 100644 --- a/cmd/msgvault/cmd/import_gvoice.go +++ b/cmd/msgvault/cmd/import_gvoice.go @@ -2,7 +2,9 @@ package cmd import ( "context" + "errors" "fmt" + "log/slog" "os" "os/signal" "strings" @@ -38,19 +40,24 @@ Examples: } func runImportGvoice(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg if !isDaemonCLISubprocess() { return runDaemonCLICommandHTTPFromCobra(cmd, args) } takeoutDir := args[0] - s, cleanup, err := openWritableStoreAndInitForIngest() + s, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) if err != nil { return err } defer cleanup() - clientOpts, err := buildGvoiceOpts() + clientOpts, err := buildGvoiceOpts(state.logger) if err != nil { return err } @@ -97,7 +104,7 @@ func runImportGvoice(cmd *cobra.Command, args []string) error { if ctx.Err() != nil { fmt.Println("\nImport interrupted.") printGvoiceSummary(summary, startTime) - return rebuildCacheAfterWrite(cfg.DatabaseDSN()) + return rebuildCacheAfterWrite(cfg.DatabaseDSN(), state) } return fmt.Errorf("import failed: %w", err) } @@ -106,17 +113,17 @@ func runImportGvoice(cmd *cobra.Command, args []string) error { // Auto-default-identity must run BEFORE the legacy migration // retry — see comment in account_identity.go. if !noDefaultIdentityImportGVoice && strings.HasPrefix(phone, "+") { - confirmDefaultIdentity(cmd.OutOrStdout(), s, src.ID, phone, phone, "phone-e164") + confirmDefaultIdentity(cmd.OutOrStdout(), s, src.ID, phone, phone, "phone-e164", state.logger) } - if err := runPostSourceCreateMigrations(s); err != nil { + if err := runPostSourceCreateMigrationsForInvocation(s, state); err != nil { return fmt.Errorf("post-source-create migrations: %w", err) } printGvoiceSummary(summary, startTime) - return rebuildCacheAfterWrite(cfg.DatabaseDSN()) + return rebuildCacheAfterWrite(cfg.DatabaseDSN(), state) } -func buildGvoiceOpts() ([]gvoice.ClientOption, error) { +func buildGvoiceOpts(logger *slog.Logger) ([]gvoice.ClientOption, error) { var opts []gvoice.ClientOption opts = append(opts, gvoice.WithLogger(logger)) diff --git a/cmd/msgvault/cmd/import_gvoice_e2e_test.go b/cmd/msgvault/cmd/import_gvoice_e2e_test.go index 0d662c4d8..054652113 100644 --- a/cmd/msgvault/cmd/import_gvoice_e2e_test.go +++ b/cmd/msgvault/cmd/import_gvoice_e2e_test.go @@ -17,6 +17,12 @@ import ( ) func TestImportGvoiceStoresVoicemailAudioEndToEnd(t *testing.T) { + cfg := testConfigValue() + cfgFile := "" + homeDir := "" + logger := testLoggerValue() + verbose := false + assert := assert.New(t) require := require.New(t) markDaemonCLISubprocessForTest(t) @@ -50,6 +56,7 @@ func TestImportGvoiceStoresVoicemailAudioEndToEnd(t *testing.T) { voice, }) require.NoError(rootCmd.ExecuteContext(context.Background())) + runCfg := invocationFromCommand(rootCmd).cfg st, err := store.Open(filepath.Join(home, "msgvault.db")) require.NoError(err) @@ -94,7 +101,7 @@ func TestImportGvoiceStoresVoicemailAudioEndToEnd(t *testing.T) { )) // An interrupted subprocess can persist the import before CLI finalization. // Retrying identical input must still refresh the published attachment data. - client, err := gvoice.NewClient(voice, gvoice.WithAttachmentsDir(cfg.AttachmentsDir())) + client, err := gvoice.NewClient(voice, gvoice.WithAttachmentsDir(runCfg.AttachmentsDir())) require.NoError(err) t.Cleanup(func() { _ = client.Close() }) source, err := st.GetOrCreateSource("google_voice", client.Identifier()) @@ -108,8 +115,9 @@ func TestImportGvoiceStoresVoicemailAudioEndToEnd(t *testing.T) { voice, }) require.NoError(rootCmd.ExecuteContext(context.Background())) + runCfg = invocationFromCommand(rootCmd).cfg - engine, err := query.NewDuckDBEngine(cfg.AnalyticsDir(), "", nil) + engine, err := query.NewDuckDBEngine(runCfg.AnalyticsDir(), "", nil) require.NoError(err) result, queryErr := engine.QuerySQL(context.Background(), ` SELECT size FROM attachments diff --git a/cmd/msgvault/cmd/import_imazing_csv.go b/cmd/msgvault/cmd/import_imazing_csv.go new file mode 100644 index 000000000..276b5c413 --- /dev/null +++ b/cmd/msgvault/cmd/import_imazing_csv.go @@ -0,0 +1,144 @@ +package cmd + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/imazingcsv" +) + +var resolveLocalTimezone = ResolveLocalTimezone + +// ResolveLocalTimezone returns a concrete IANA timezone for offset-free CSV +// timestamps on Unix hosts, where TZ, /etc/localtime, or /etc/timezone name +// the local zone. Go's opaque "Local" label is not stable enough to persist. +// Windows exposes no dependable IANA name for the system timezone, so it +// refuses there and callers must require an explicit --timezone instead. +func ResolveLocalTimezone() (string, error) { + if runtime.GOOS == "windows" { + return "", errors.New("the Windows system timezone has no dependable IANA name") + } + candidates := []string{strings.TrimPrefix(strings.TrimSpace(os.Getenv("TZ")), ":")} + if time.Local != nil { + candidates = append(candidates, time.Local.String()) + } + if target, err := filepath.EvalSymlinks("/etc/localtime"); err == nil { + if _, zone, ok := strings.Cut(filepath.ToSlash(target), "/zoneinfo/"); ok { + candidates = append(candidates, zone) + } + } + if data, err := os.ReadFile("/etc/timezone"); err == nil { + candidates = append(candidates, strings.TrimSpace(string(data))) + } + for _, candidate := range candidates { + if candidate == "" || candidate == "Local" || filepath.IsAbs(candidate) { + continue + } + if _, err := time.LoadLocation(candidate); err == nil { + return candidate, nil + } + } + return "", errors.New("local timezone has no concrete IANA name") +} + +func newImportIMazingCSVCmd() *cobra.Command { + var opts imazingcsv.Options + command := &cobra.Command{ + Use: "import-imazing-csv ", + Short: "Import Messages CSV exports created by iMazing", + Long: `Import iMessage and SMS history from an iMazing Messages CSV export. + +The input can be the export root containing csv/ and attachments/, or the csv/ +directory itself. --me identifies your phone number or email address. Offset-free +message dates use --timezone. On Unix, an omitted --timezone resolves the local +IANA zone; Windows has no dependable local IANA zone, so --timezone is required +there.`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if opts.Timezone == "" { + zone, err := resolveLocalTimezone() + if err != nil { + return fmt.Errorf("resolve local timezone; pass --timezone with an IANA name: %w", err) + } + opts.Timezone = zone + if err := cmd.Flags().Set("timezone", zone); err != nil { + return fmt.Errorf("forward resolved timezone: %w", err) + } + } + if !isDaemonCLISubprocess() { + // The export directory and --contacts file are caller-local + // inputs; a configured remote daemon cannot read them. + return runDaemonCLICommandHTTPFromCobraWithLocalFiles(cmd, args, nil) + } + return runImportIMazingCSV(cmd, args[0], opts) + }, + } + command.Flags().StringVar(&opts.Owner, "me", "", "your phone number or email address") + command.Flags().StringVar(&opts.ContactsPath, "contacts", "", "vCard file used to fill empty participant names") + command.Flags().StringVar(&opts.Timezone, "timezone", "", + "IANA timezone for dates without an offset (required on Windows)") + _ = command.MarkFlagRequired("me") + return command +} + +func runImportIMazingCSV(cmd *cobra.Command, exportDir string, opts imazingcsv.Options) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + info, err := os.Stat(exportDir) + if err != nil { + return fmt.Errorf("inspect iMazing export directory: %w", err) + } + if !info.IsDir() { + return fmt.Errorf("iMazing export path is not a directory: %s", exportDir) + } + dbPath := cfg.DatabaseDSN() + st, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) + if err != nil { + return err + } + defer cleanup() + opts.AttachmentsDir = cfg.AttachmentsDir() + summary, err := imazingcsv.NewImporter(st, opts).ImportPath(cmd.Context(), exportDir) + if err != nil { + // Messages commit one by one; committed rows must still reach the + // analytics cache even though the import as a whole failed. + return errors.Join( + fmt.Errorf("import iMazing CSV: %w", err), + rebuildCacheAfterWrite(dbPath, state), + ) + } + if err := runPostSourceCreateMigrationsForInvocation(st, state); err != nil { + return errors.Join( + fmt.Errorf("post-source-create migrations: %w", err), + rebuildCacheAfterWrite(dbPath, state), + ) + } + _, _ = fmt.Fprintln(cmd.OutOrStdout(), "Import complete") + _, _ = fmt.Fprintf(cmd.OutOrStdout(), " Files: %d\n", summary.Files) + _, _ = fmt.Fprintf(cmd.OutOrStdout(), " Conversations: %d\n", summary.Conversations) + _, _ = fmt.Fprintf(cmd.OutOrStdout(), " Messages: %d\n", summary.Messages) + _, _ = fmt.Fprintf(cmd.OutOrStdout(), " Participants: %d\n", summary.Participants) + _, _ = fmt.Fprintf(cmd.OutOrStdout(), " Attachments stored: %d\n", summary.AttachmentsStored) + _, _ = fmt.Fprintf(cmd.OutOrStdout(), " Attachments missing: %d\n", summary.AttachmentsMissing) + _, _ = fmt.Fprintf(cmd.OutOrStdout(), " Attachments skipped: %d\n", summary.AttachmentsSkipped) + _, _ = fmt.Fprintf(cmd.OutOrStdout(), " Replies linked: %d\n", summary.RepliesLinked) + _, _ = fmt.Fprintf(cmd.OutOrStdout(), " Replies unresolved: %d\n", summary.RepliesUnresolved) + if summary.ContactsTotal > 0 { + _, _ = fmt.Fprintf(cmd.OutOrStdout(), " Contacts matched: %d of %d\n", + summary.ContactsMatched, summary.ContactsTotal) + } + return rebuildCacheAfterWrite(dbPath, state) +} + +func init() { + rootCmd.AddCommand(newImportIMazingCSVCmd()) +} diff --git a/cmd/msgvault/cmd/import_imazing_csv_test.go b/cmd/msgvault/cmd/import_imazing_csv_test.go new file mode 100644 index 000000000..dd24d5443 --- /dev/null +++ b/cmd/msgvault/cmd/import_imazing_csv_test.go @@ -0,0 +1,291 @@ +package cmd + +import ( + "bytes" + "encoding/csv" + "errors" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/query" + "go.kenn.io/msgvault/internal/store" +) + +func TestImportIMazingCSVRequiresMe(t *testing.T) { + command := &cobra.Command{Use: "test"} + command.AddCommand(newImportIMazingCSVCmd()) + command.SetArgs([]string{"import-imazing-csv", t.TempDir()}) + err := command.Execute() + require.Error(t, err) + assert.ErrorContains(t, err, "required flag") +} + +// The export directory and --contacts file live on the caller's machine, so a +// configured remote daemon cannot read them. The command must refuse before +// proxying instead of sending client-side paths for the daemon host to fail on. +func TestImportIMazingCSVRejectsConfiguredRemoteDaemon(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + server, requests := newDaemonCLIRunnerTestServer(t, nil, `{"type":"complete"}`) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx + t.Setenv(daemonCLISubprocessEnv, "") + + command := &cobra.Command{Use: "test"} + command.SetContext(testCtx) + command.AddCommand(newImportIMazingCSVCmd()) + var output bytes.Buffer + command.SetOut(&output) + command.SetErr(&output) + command.SetArgs([]string{ + "import-imazing-csv", t.TempDir(), "--me", "+15550000001", "--timezone", "UTC", + }) + + err := command.Execute() + + require.ErrorContains(err, "run it on the daemon host with --local") + assert.Equal(0, int(requests.Load()), "runner endpoint calls") +} + +func TestImportIMazingCSVReportsSummary(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + markDaemonCLISubprocessForTest(t) + home := t.TempDir() + cfg := config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx + cfg.HomeDir = home + cfg.Data.DataDir = home + exportDir := writeIMazingCSVCommandFixture(t) + command := &cobra.Command{Use: "test"} + command.SetContext(testCtx) + command.AddCommand(newImportIMazingCSVCmd()) + var output bytes.Buffer + command.SetOut(&output) + command.SetErr(&output) + command.SetArgs([]string{ + "import-imazing-csv", exportDir, "--me", "+15550000001", "--timezone", "UTC", + }) + + require.NoError(command.Execute()) + assert.Contains(output.String(), "Files: 1") + assert.Contains(output.String(), "Messages: 1") + assert.Contains(output.String(), "Attachments stored: 0") +} + +// TestImportIMazingCSVRebuildsCacheAfterPartialImportFailure catches the +// failure mode where messages commit one by one but the import later fails +// (here: an unreadable --contacts file). The committed rows must still reach +// the analytics cache, not stay invisible until the next maintenance build, +// even though the command returns an error. +func TestImportIMazingCSVRebuildsCacheAfterPartialImportFailure(t *testing.T) { + cfg := testConfigValue() + + assert := assert.New(t) + require := require.New(t) + markDaemonCLISubprocessForTest(t) + dataDir := t.TempDir() + savedCfg := cfg + cfg = config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx + cfg.HomeDir = dataDir + cfg.Data.DataDir = dataDir + t.Cleanup(func() { cfg = savedCfg }) + + exportDir := writeIMazingCSVCommandFixture(t) + command := &cobra.Command{Use: "test"} + command.SetContext(testCtx) + command.AddCommand(newImportIMazingCSVCmd()) + var output bytes.Buffer + command.SetOut(&output) + command.SetErr(&output) + command.SetArgs([]string{ + "import-imazing-csv", exportDir, "--me", "+15550000001", "--timezone", "UTC", + "--contacts", filepath.Join(dataDir, "missing.vcf"), + }) + err := command.Execute() + require.Error(err) + require.ErrorContains(err, "import iMazing CSV") + require.ErrorContains(err, "parse iMazing contacts") + + st, err := store.OpenForTest(cfg.DatabaseDSN()) + require.NoError(err) + var messages int + require.NoError(st.DB().QueryRow(`SELECT COUNT(*) FROM messages`).Scan(&messages)) + revision, err := st.DerivedDataRevision() + require.NoError(err) + require.NoError(st.Close()) + assert.Equal(1, messages, "the message committed before the import error must be stored") + + cacheState, err := query.ReadCacheSyncState(cfg.AnalyticsDir()) + require.NoError(err, "the analytics cache must exist after a partially failed import") + assert.Equal(revision, cacheState.DerivedDataRevision, + "the analytics cache must catch up to the partial import's committed revision") + staleness := cacheNeedsBuild(cfg.DatabaseDSN(), cfg.AnalyticsDir()) + assert.False(staleness.NeedsBuild, "a failed import must not leave the analytics cache stale") +} + +// --timezone Local names the host's own zone instead of a concrete IANA zone, +// so offset-free timestamps would decode differently on every host while the +// opaque name is persisted with the source. The command must fail before the +// importer creates a source, starts a sync, or persists a message. +func TestImportIMazingCSVRejectsLocalTimezone(t *testing.T) { + cfg := testConfigValue() + + assert := assert.New(t) + require := require.New(t) + markDaemonCLISubprocessForTest(t) + dataDir := t.TempDir() + savedCfg := cfg + cfg = config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx + cfg.HomeDir = dataDir + cfg.Data.DataDir = dataDir + t.Cleanup(func() { cfg = savedCfg }) + + exportDir := writeIMazingCSVCommandFixture(t) + command := &cobra.Command{Use: "test"} + command.SetContext(testCtx) + command.AddCommand(newImportIMazingCSVCmd()) + var output bytes.Buffer + command.SetOut(&output) + command.SetErr(&output) + command.SetArgs([]string{ + "import-imazing-csv", exportDir, "--me", "+15550000001", "--timezone", "Local", + }) + + err := command.Execute() + + require.ErrorContains(err, "host-dependent Local") + st, err := store.OpenForTest(cfg.DatabaseDSN()) + require.NoError(err) + var sources, syncs, messages int + require.NoError(st.DB().QueryRow( + `SELECT COUNT(*) FROM sources WHERE source_type = 'imazing_csv'`).Scan(&sources)) + require.NoError(st.DB().QueryRow(`SELECT COUNT(*) FROM sync_runs`).Scan(&syncs)) + require.NoError(st.DB().QueryRow(`SELECT COUNT(*) FROM messages`).Scan(&messages)) + require.NoError(st.Close()) + assert.Zero(sources, "no iMazing CSV source may be created for --timezone Local") + assert.Zero(syncs, "no sync may be started for --timezone Local") + assert.Zero(messages, "no message may be persisted for --timezone Local") +} + +func TestImportIMazingCSVReportsLocalTimezoneResolutionFailure(t *testing.T) { + previous := resolveLocalTimezone + resolveLocalTimezone = func() (string, error) { return "", errors.New("unknown local zone") } + t.Cleanup(func() { resolveLocalTimezone = previous }) + command := &cobra.Command{Use: "test"} + command.AddCommand(newImportIMazingCSVCmd()) + command.SetArgs([]string{"import-imazing-csv", t.TempDir(), "--me", "+15550000001"}) + err := command.Execute() + require.Error(t, err) + assert.ErrorContains(t, err, "--timezone") +} + +// The omitted-timezone default is a Unix-only capability: Windows exposes no +// dependable IANA name for the system timezone, so the resolver must refuse +// there instead of advertising a local-IANA default it cannot honor. A +// concrete TZ keeps the Unix branch deterministic on any host. +func TestResolveLocalTimezonePlatformContract(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + t.Setenv("TZ", "Europe/Berlin") + + zone, err := ResolveLocalTimezone() + + if runtime.GOOS == "windows" { + require.Error(err) + require.ErrorContains(err, "Windows") + return + } + require.NoError(err) + assert.Equal("Europe/Berlin", zone) +} + +// Omitted --timezone resolves the local IANA zone on Unix and is rejected on +// Windows. Both branches are behavioral: the Unix branch runs a full import +// with the resolved zone, and the Windows branch must fail before any +// source, sync, or message is created. +func TestImportIMazingCSVOmittedTimezonePlatformContract(t *testing.T) { + cfg := testConfigValue() + + assert := assert.New(t) + require := require.New(t) + markDaemonCLISubprocessForTest(t) + dataDir := t.TempDir() + savedCfg := cfg + cfg = config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx + cfg.HomeDir = dataDir + cfg.Data.DataDir = dataDir + t.Cleanup(func() { cfg = savedCfg }) + + exportDir := writeIMazingCSVCommandFixture(t) + command := &cobra.Command{Use: "test"} + command.SetContext(testCtx) + command.AddCommand(newImportIMazingCSVCmd()) + var output bytes.Buffer + command.SetOut(&output) + command.SetErr(&output) + command.SetArgs([]string{"import-imazing-csv", exportDir, "--me", "+15550000001"}) + + if runtime.GOOS == "windows" { + err := command.Execute() + require.Error(err) + require.ErrorContains(err, "--timezone") + require.ErrorContains(err, "Windows") + // The omitted-timezone rejection happens before any store + // initialization, so proving the database file was never created is + // both the strictest check and free of the handle-leak and empty- + // schema pitfalls of opening a store just to count rows. + dbPath, err := cfg.DatabasePath() + require.NoError(err) + require.NoFileExists(dbPath, "a rejected timezone must fail before the database is created") + return + } + + t.Setenv("TZ", "Europe/Berlin") + require.NoError(command.Execute()) + assert.Contains(output.String(), "Import complete") + st, err := store.OpenForTest(cfg.DatabaseDSN()) + require.NoError(err) + source, err := st.GetSourceByTypeAndIdentifier("imazing_csv", "+15550000001") + require.NoError(err) + require.True(source.SyncConfig.Valid) + assert.JSONEq(`{"ambiguous_time_policy":"earlier","timezone":"Europe/Berlin"}`, source.SyncConfig.String) + var messages int + require.NoError(st.DB().QueryRow(`SELECT COUNT(*) FROM messages`).Scan(&messages)) + require.NoError(st.Close()) + assert.Equal(1, messages, "the import runs with the resolved zone") +} + +func writeIMazingCSVCommandFixture(t *testing.T) string { + t.Helper() + root := t.TempDir() + require.NoError(t, os.Mkdir(filepath.Join(root, "csv"), 0o700)) + file, err := os.Create(filepath.Join(root, "csv", "messages.csv")) + require.NoError(t, err) + writer := csv.NewWriter(file) + require.NoError(t, writer.Write([]string{ + "Chat Session", "Message Date", "Delivered Date", "Read Date", "Service", "Type", + "Sender ID", "Sender Name", "Status", "Replying to", "Subject", "Text", "Attachment", "Attachment type", + })) + require.NoError(t, writer.Write([]string{ + "Alice", "2024-06-01 12:00:00", "", "", "iMessage", "Outgoing", + "", "", "", "", "", "hello", "", "", + })) + writer.Flush() + require.NoError(t, writer.Error()) + require.NoError(t, file.Close()) + return root +} diff --git a/cmd/msgvault/cmd/import_imessage.go b/cmd/msgvault/cmd/import_imessage.go index dd50aa0c3..f315106a9 100644 --- a/cmd/msgvault/cmd/import_imessage.go +++ b/cmd/msgvault/cmd/import_imessage.go @@ -2,7 +2,9 @@ package cmd import ( "context" + "errors" "fmt" + "log/slog" "os" "os/signal" "path/filepath" @@ -55,11 +57,15 @@ Examples: } func runImportImessage(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } if !isDaemonCLISubprocess() { return runDaemonCLICommandHTTPFromCobra(cmd, args) } - s, cleanup, err := openWritableStoreAndInitForIngest() + s, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) if err != nil { return err } @@ -70,7 +76,7 @@ func runImportImessage(cmd *cobra.Command, args []string) error { return err } - clientOpts, err := buildImessageOpts() + clientOpts, err := buildImessageOpts(state.logger) if err != nil { return err } @@ -90,7 +96,7 @@ func runImportImessage(cmd *cobra.Command, args []string) error { if err != nil { return fmt.Errorf("get or create source: %w", err) } - if err := runPostSourceCreateMigrations(s); err != nil { + if err := runPostSourceCreateMigrationsForInvocation(s, state); err != nil { return fmt.Errorf("post-source-create migrations: %w", err) } @@ -122,20 +128,23 @@ func runImportImessage(cmd *cobra.Command, args []string) error { if ctx.Err() != nil { fmt.Println("\nImport interrupted.") printImessageSummary(summary, startTime) - return finishImessageImport(s) + return finishImessageImport(s, state, summary) } return fmt.Errorf("import failed: %w", err) } printImessageSummary(summary, startTime) - return finishImessageImport(s) + return finishImessageImport(s, state, summary) } // finishImessageImport runs the post-import name backfill, refreshes -// generated chat titles, and triggers an analytics cache rebuild that picks up -// the participant/conversation changes (the default staleness check only -// notices new/deleted messages, not title or display_name updates). -func finishImessageImport(s *store.Store) error { +// generated chat titles, and rebuilds the analytics cache when those changes +// affect existing rows inside the published message boundary. +func finishImessageImport(s *store.Store, state *invocation, summary *imessage.ImportSummary) error { + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg mutated := false if importImessageContacts != "" { @@ -149,11 +158,11 @@ func finishImessageImport(s *store.Store) error { } dbPath := cfg.DatabaseDSN() - if mutated { + if !store.IsPostgresURL(dbPath) && (mutated || (summary != nil && summary.DatesCleared > 0)) { // Title/display_name updates aren't visible to the message-id-keyed // staleness check, so the standard rebuildCacheAfterWrite would skip. - // Force a full rebuild so conversations.parquet and - // participants.parquet are re-exported and the TUI sees the new names. + // A date clear also leaves the message ID unchanged, so it must re-export + // messages.parquet as well as the affected conversation datasets. if _, err := buildCache( dbPath, cfg.AnalyticsDir(), @@ -165,7 +174,7 @@ func finishImessageImport(s *store.Store) error { return nil } - return rebuildCacheAfterWrite(dbPath) + return rebuildCacheAfterWrite(dbPath, state) } func retitleImessageChats(s *store.Store) bool { @@ -276,7 +285,7 @@ func resolveChatDBPath() (string, error) { return path, nil } -func buildImessageOpts() ([]imessage.ClientOption, error) { +func buildImessageOpts(logger *slog.Logger) ([]imessage.ClientOption, error) { var opts []imessage.ClientOption opts = append(opts, imessage.WithImessageLogger(logger)) @@ -342,6 +351,9 @@ func printImessageSummary( if summary.Skipped > 0 { fmt.Printf(" Skipped: %d\n", summary.Skipped) } + if summary.DatesCleared > 0 { + fmt.Printf(" Dates cleared: %d\n", summary.DatesCleared) + } if summary.MessagesImported > 0 && elapsed.Seconds() > 0 { rate := float64(summary.MessagesImported) / elapsed.Seconds() fmt.Printf(" Rate: %.1f messages/sec\n", rate) diff --git a/cmd/msgvault/cmd/import_imessage_dates_test.go b/cmd/msgvault/cmd/import_imessage_dates_test.go new file mode 100644 index 000000000..6d3b909f2 --- /dev/null +++ b/cmd/msgvault/cmd/import_imessage_dates_test.go @@ -0,0 +1,195 @@ +package cmd + +import ( + "context" + "database/sql" + "math" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/imessage" + "go.kenn.io/msgvault/internal/store" +) + +// Exercise the reported path: real chat.db import followed by a full cache build. +func TestBuildCacheAfterIMessageSentinelDates(t *testing.T) { + assert, require := assert.New(t), require.New(t) + tmp := t.TempDir() + chatPath := filepath.Join(tmp, "chat.db") + chat, err := sql.Open("sqlite3", chatPath+"?_journal_mode=WAL") + require.NoError(err) + t.Cleanup(func() { _ = chat.Close() }) + _, err = chat.Exec(` + CREATE TABLE message (guid TEXT, text TEXT, attributedBody BLOB, date INTEGER, + is_from_me INTEGER, service TEXT, cache_has_attachments INTEGER, handle_id INTEGER); + CREATE TABLE handle (id TEXT); + CREATE TABLE chat (guid TEXT, display_name TEXT, chat_identifier TEXT); + CREATE TABLE chat_message_join (chat_id INTEGER, message_id INTEGER); + CREATE TABLE chat_handle_join (chat_id INTEGER, handle_id INTEGER); + INSERT INTO handle VALUES ('peer@example.test'); + INSERT INTO chat VALUES ('any;-;synthetic', NULL, 'peer@example.test'); + INSERT INTO chat_handle_join VALUES (1,1); + `) + require.NoError(err) + for i, date := range []int64{math.MinInt64, math.MaxInt64, 725760000000000000, 725846400000000000} { + var body any + if i >= 2 { + body = "Synthetic message" + } + _, err = chat.Exec(`INSERT INTO message VALUES ('synthetic', ?, NULL, ?, ?, 'SMS', 0, 1)`, body, date, i%2) + require.NoError(err) + _, err = chat.Exec(`INSERT INTO chat_message_join VALUES (1,?)`, i+1) + require.NoError(err) + } + dbPath := filepath.Join(tmp, "msgvault.db") + st, err := store.Open(dbPath) + require.NoError(err) + t.Cleanup(func() { _ = st.Close() }) + require.NoError(st.InitSchema()) + src, err := resolveImessageSource(st) + require.NoError(err) + require.NoError(st.AddAccountIdentity(src.ID, "owner@example.test", "manual")) + c, err := imessage.NewClient(chatPath, imessage.WithOwnerHandle("owner@example.test")) + require.NoError(err) + t.Cleanup(func() { _ = c.Close() }) + summary, err := c.Import(context.Background(), st, src.ID) + require.NoError(err) + assert.Equal(4, summary.MessagesImported) + assert.Zero(summary.Skipped) + var total, undated int + require.NoError(st.DB().QueryRow(`SELECT COUNT(*) FROM messages`).Scan(&total)) + assert.Equal(4, total) + require.NoError(st.DB().QueryRow(`SELECT COUNT(*) FROM messages WHERE sent_at IS NULL AND internal_date IS NULL`).Scan(&undated)) + assert.Equal(2, undated) + require.NoError(st.Close()) + analyticsDir := filepath.Join(tmp, "analytics") + result, err := buildCache(dbPath, analyticsDir, true) + require.NoError(err, "sentinel placeholders must not break identity validation") + assert.False(result.Skipped) + + // Undated rows stay archived; the dated analytics contain only real dates. + analytics, err := sql.Open("duckdb", "") + require.NoError(err) + t.Cleanup(func() { _ = analytics.Close() }) + var exported int + require.NoError(analytics.QueryRow(`SELECT COUNT(*) FROM read_parquet(?, hive_partitioning=true)`, + filepath.Join(analyticsDir, "messages", "**", "*.parquet")).Scan(&exported)) + assert.Equal(2, exported) +} + +func TestReimportClearedIMessageDateRebuildsExistingCache(t *testing.T) { + assert, require := assert.New(t), require.New(t) + tmp := t.TempDir() + chatPath := filepath.Join(tmp, "chat.db") + chat, err := sql.Open("sqlite3", chatPath+"?_journal_mode=WAL") + require.NoError(err) + t.Cleanup(func() { _ = chat.Close() }) + _, err = chat.Exec(` + CREATE TABLE message (guid TEXT, text TEXT, attributedBody BLOB, date INTEGER, + is_from_me INTEGER, service TEXT, cache_has_attachments INTEGER, handle_id INTEGER); + CREATE TABLE handle (id TEXT); + CREATE TABLE chat (guid TEXT, display_name TEXT, chat_identifier TEXT); + CREATE TABLE chat_message_join (chat_id INTEGER, message_id INTEGER); + CREATE TABLE chat_handle_join (chat_id INTEGER, handle_id INTEGER); + INSERT INTO handle VALUES ('peer@example.test'); + INSERT INTO chat VALUES ('any;-;synthetic', NULL, 'peer@example.test'); + INSERT INTO message VALUES ('synthetic-1', NULL, NULL, 725760000000000000, 0, 'SMS', 0, NULL); + INSERT INTO message VALUES ('synthetic-2', NULL, NULL, 725846400000000000, 0, 'SMS', 0, NULL); + INSERT INTO chat_message_join VALUES (1,1); + INSERT INTO chat_message_join VALUES (1,2); + `) + require.NoError(err) + + cfg := config.NewDefaultConfig() + cfg.Data.DataDir = tmp + dbPath := cfg.DatabaseDSN() + st, err := store.Open(dbPath) + require.NoError(err) + t.Cleanup(func() { _ = st.Close() }) + require.NoError(st.InitSchema()) + src, err := resolveImessageSource(st) + require.NoError(err) + require.NoError(st.AddAccountIdentity(src.ID, "owner@example.test", "manual")) + c, err := imessage.NewClient(chatPath) + require.NoError(err) + t.Cleanup(func() { _ = c.Close() }) + summary, err := c.Import(context.Background(), st, src.ID) + require.NoError(err) + assert.Equal(2, summary.MessagesImported) + assert.Zero(summary.Skipped) + _ = retitleImessageChats(st) + require.NoError(st.Close()) + result, err := buildCache(dbPath, cfg.AnalyticsDir(), true) + require.NoError(err) + assert.False(result.Skipped) + + cachedMessages := func(sourceMessageID string) int { + analytics, err := sql.Open("duckdb", "") + require.NoError(err) + defer func() { _ = analytics.Close() }() + var count int + require.NoError(analytics.QueryRow(` + SELECT COUNT(*) FROM read_parquet(?, hive_partitioning=true) + WHERE source_message_id = ?`, filepath.Join(cfg.AnalyticsDir(), "messages", "**", "*.parquet"), sourceMessageID).Scan(&count)) + return count + } + assert.Equal(1, cachedMessages("1"), "initial cache should contain the dated message") + + st, err = store.Open(dbPath) + require.NoError(err) + _, err = chat.Exec(`UPDATE message SET date = ? WHERE ROWID = 1`, int64(math.MaxInt64)) + require.NoError(err) + summary, err = c.Import(context.Background(), st, src.ID) + require.NoError(err) + assert.Equal(2, summary.MessagesImported) + var sentAt, internalDate sql.NullTime + require.NoError(st.DB().QueryRow(` + SELECT sent_at, internal_date FROM messages WHERE source_id = ? AND source_message_id = '1'`, src.ID).Scan(&sentAt, &internalDate)) + assert.False(sentAt.Valid) + assert.False(internalDate.Valid) + + state := &invocation{cfg: cfg} + require.Equal(1, summary.DatesCleared) + require.NoError(finishImessageImport(st, state, summary)) + assert.Equal(0, cachedMessages("1"), "the cache must drop the message whose date was cleared") +} + +func TestFinishImessageImportSkipsCacheBuildForPostgres(t *testing.T) { + assert, require := assert.New(t), require.New(t) + st, err := store.Open(filepath.Join(t.TempDir(), "msgvault.db")) + require.NoError(err) + t.Cleanup(func() { _ = st.Close() }) + require.NoError(st.InitSchema()) + src, err := resolveImessageSource(st) + require.NoError(err) + participantID, err := st.EnsureParticipant("peer@example.test", "Synthetic Person", "example.test") + require.NoError(err) + conversationID, err := st.EnsureConversationWithType( + src.ID, "synthetic-chat", "direct_chat", "peer@example.test", + ) + require.NoError(err) + require.NoError(st.EnsureConversationParticipant(conversationID, participantID, "member")) + + cfg := config.NewDefaultConfig() + cfg.Data.DatabaseURL = "postgres://user:pass@example.test:5432/msgvault" + state := &invocation{cfg: cfg} + require.NoError(finishImessageImport(st, state, &imessage.ImportSummary{})) + var title string + require.NoError(st.DB().QueryRow(`SELECT title FROM conversations WHERE id = ?`, conversationID).Scan(&title)) + assert.Equal("Synthetic Person", title) +} + +func TestPrintImessageSummaryReportsClearedDates(t *testing.T) { + assert := assert.New(t) + done := captureStdout(t) + printImessageSummary(&imessage.ImportSummary{DatesCleared: 2}, time.Now()) + assert.Contains(done(), " Dates cleared: 2\n") + + done = captureStdout(t) + printImessageSummary(&imessage.ImportSummary{}, time.Now()) + assert.NotContains(done(), "Dates cleared") +} diff --git a/cmd/msgvault/cmd/import_maildir_test.go b/cmd/msgvault/cmd/import_maildir_test.go index d27ba1f78..373a37e1c 100644 --- a/cmd/msgvault/cmd/import_maildir_test.go +++ b/cmd/msgvault/cmd/import_maildir_test.go @@ -17,7 +17,7 @@ func TestImportMaildirCommandArchivesMail(t *testing.T) { require := require.New(t) dataDir := t.TempDir() testCfg := lifecycleTestConfig(dataDir) - withStoreResolverConfig(t, testCfg) + testCtx := withStoreResolverConfig(t, testCfg) t.Setenv(daemonCLISubprocessEnv, strconv.Itoa(os.Getppid())) root := filepath.Join(dataDir, "mail") for _, dir := range []string{"new", "cur", "tmp"} { @@ -25,6 +25,7 @@ func TestImportMaildirCommandArchivesMail(t *testing.T) { } require.NoError(os.WriteFile(filepath.Join(root, "new", "one"), []byte("From: alice@example.com\r\nSubject: Maildir CLI\r\n\r\nbody"), 0600)) command := newImportMaildirCommand() + command.SetContext(testCtx) command.SetOut(io.Discard) command.SetErr(io.Discard) command.SetArgs([]string{root, "--identifier", "alice@example.com"}) diff --git a/cmd/msgvault/cmd/import_mbox.go b/cmd/msgvault/cmd/import_mbox.go index 10944bcb7..bf0e1780f 100644 --- a/cmd/msgvault/cmd/import_mbox.go +++ b/cmd/msgvault/cmd/import_mbox.go @@ -60,6 +60,12 @@ Examples: `, Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger if !isDaemonCLISubprocess() { return runDaemonCLICommandHTTPFromCobra(cmd, args) } @@ -109,7 +115,7 @@ Examples: } }() - st, cleanup, err := openWritableStoreAndInitForIngest() + st, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) if err != nil { return err } @@ -268,7 +274,7 @@ Examples: NoResume: importMboxNoResume, CheckpointInterval: importMboxCheckpointInterval, AttachmentsDir: attachmentsDir, - RemoteImages: configuredRemoteImageFetcher(), + RemoteImages: configuredRemoteImageFetcher(cfg), Logger: logger, }) if err != nil { @@ -316,7 +322,7 @@ Examples: // A Google Groups identifier names a shared archive, not its owner. if ctx.Err() == nil && !hadHardErrors && !noDefaultIdentityImportMbox && importMboxSourceType != "google-groups" { if sourceID != 0 { - confirmDefaultIdentity(cmd.OutOrStdout(), st, sourceID, identifier, identifier, "account-identifier") + confirmDefaultIdentity(cmd.OutOrStdout(), st, sourceID, identifier, identifier, "account-identifier", state.logger) } } @@ -334,7 +340,7 @@ Examples: // tracked in // private/drafts/2026-05-02-issue-import-migration-error-ux.md. if sourceID != 0 { - if err := runPostSourceCreateMigrations(st); err != nil { + if err := runPostSourceCreateMigrationsForInvocation(st, state); err != nil { return fmt.Errorf("post-source-create migrations: %w", err) } } @@ -362,7 +368,7 @@ Examples: _, _ = fmt.Fprintf(out, " Errors: %d\n", totalErrors) _, _ = fmt.Fprintf(out, " Bytes: %.2f MB\n", float64(totalBytes)/(1024*1024)) - cacheErr := rebuildCacheAfterWrite(dbPath) + cacheErr := rebuildCacheAfterWrite(dbPath, state) if ctx.Err() == nil && hadHardErrors { return errors.Join(fmt.Errorf("import completed with %d errors", totalErrors), cacheErr) diff --git a/cmd/msgvault/cmd/import_mbox_e2e_test.go b/cmd/msgvault/cmd/import_mbox_e2e_test.go index 56c19e36e..3005b1d84 100644 --- a/cmd/msgvault/cmd/import_mbox_e2e_test.go +++ b/cmd/msgvault/cmd/import_mbox_e2e_test.go @@ -19,6 +19,12 @@ import ( ) func TestImportMboxCmd_EndToEnd_MboxFile(t *testing.T) { + cfg := testConfigValue() + cfgFile := "" + homeDir := "" + logger := testLoggerValue() + verbose := false + markDaemonCLISubprocessForTest(t) require := require.New(t) @@ -126,6 +132,12 @@ func TestImportMboxCmd_EndToEnd_MboxFile(t *testing.T) { } func TestImportMboxCmd_AttachmentFailureIsBestEffort(t *testing.T) { + cfg := testConfigValue() + cfgFile := "" + homeDir := "" + logger := testLoggerValue() + verbose := false + markDaemonCLISubprocessForTest(t) tmp := t.TempDir() @@ -199,6 +211,12 @@ func TestImportMboxCmd_AttachmentFailureIsBestEffort(t *testing.T) { } func TestImportMboxCmd_ReturnsCanceledWhenContextCanceled(t *testing.T) { + cfg := testConfigValue() + cfgFile := "" + homeDir := "" + logger := testLoggerValue() + verbose := false + markDaemonCLISubprocessForTest(t) tmp := t.TempDir() @@ -280,6 +298,12 @@ func TestImportMboxCmd_ReturnsCanceledWhenContextCanceled(t *testing.T) { } func TestImportMboxCmd_EndToEnd_ZipResumeAcrossFiles(t *testing.T) { + cfg := testConfigValue() + cfgFile := "" + homeDir := "" + logger := testLoggerValue() + verbose := false + markDaemonCLISubprocessForTest(t) require := require.New(t) diff --git a/cmd/msgvault/cmd/import_messenger.go b/cmd/msgvault/cmd/import_messenger.go index 05c1f8b62..1179276a0 100644 --- a/cmd/msgvault/cmd/import_messenger.go +++ b/cmd/msgvault/cmd/import_messenger.go @@ -1,11 +1,9 @@ package cmd import ( - "context" + "errors" "fmt" "os" - "os/signal" - "syscall" "time" "github.com/spf13/cobra" @@ -58,6 +56,12 @@ Examples: } func runImportMessenger(cmd *cobra.Command, rootDir string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger if info, err := os.Stat(rootDir); err != nil { return fmt.Errorf("source directory not found: %w", err) } else if !info.IsDir() { @@ -65,26 +69,14 @@ func runImportMessenger(cmd *cobra.Command, rootDir string) error { } dbPath := cfg.DatabaseDSN() - s, cleanup, err := openWritableStoreAndInitForIngest() + s, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) if err != nil { return err } defer cleanup() - ctx, cancel := context.WithCancel(cmd.Context()) - defer cancel() - - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) - defer signal.Stop(sigChan) - go func() { - select { - case <-sigChan: - _, _ = fmt.Fprintln(cmd.ErrOrStderr(), "\nInterrupted. Saving checkpoint...") - cancel() - case <-ctx.Done(): - } - }() + ctx, stop := withInterruptCancel(cmd, "\nInterrupted. Saving checkpoint...") + defer stop() opts := fbmessenger.ImportOptions{ Me: importMessengerMe, @@ -105,12 +97,12 @@ func runImportMessenger(cmd *cobra.Command, rootDir string) error { if err != nil { if ctx.Err() != nil { _, _ = fmt.Fprintln(cmd.OutOrStdout(), "\nImport interrupted. Re-run to continue.") - return rebuildCacheAfterWrite(dbPath) + return rebuildCacheAfterWrite(dbPath, state) } return fmt.Errorf("import failed: %w", err) } - if err := runPostSourceCreateMigrations(s); err != nil { + if err := runPostSourceCreateMigrationsForInvocation(s, state); err != nil { return fmt.Errorf("post-source-create migrations: %w", err) } @@ -138,7 +130,7 @@ func runImportMessenger(cmd *cobra.Command, rootDir string) error { importMessengerMe, fbmessenger.Slug(fbmessenger.StripDomain(importMessengerMe))) } - return rebuildCacheAfterWrite(dbPath) + return rebuildCacheAfterWrite(dbPath, state) } func init() { diff --git a/cmd/msgvault/cmd/import_messenger_e2e_test.go b/cmd/msgvault/cmd/import_messenger_e2e_test.go index e83888265..136b64277 100644 --- a/cmd/msgvault/cmd/import_messenger_e2e_test.go +++ b/cmd/msgvault/cmd/import_messenger_e2e_test.go @@ -16,6 +16,12 @@ import ( func saveMessengerState(t *testing.T) func() { t.Helper() + cfg := testConfigValue() + cfgFile := "" + homeDir := "" + logger := testLoggerValue() + verbose := false + prevCfg := cfg prevLogger := logger prevMe := importMessengerMe @@ -120,7 +126,7 @@ func TestImportMessengerRunsPostSourceMigrationWithoutMessengerIdentity(t *testi t.Cleanup(saveMessengerState(t)) testCfg := lifecycleTestConfig(tmp) testCfg.Identity.Addresses = []string{"legacy@example.com"} - withStoreResolverConfig(t, testCfg) + testCtx := withStoreResolverConfig(t, testCfg) st, err := store.Open(testCfg.DatabaseDSN()) require.NoError(err, "open seed store") @@ -137,7 +143,8 @@ func TestImportMessengerRunsPostSourceMigrationWithoutMessengerIdentity(t *testi var stdout bytes.Buffer cmd := &cobra.Command{Use: "import-messenger"} - cmd.SetContext(context.Background()) + cmd.SetContext(testCtx) + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetErr(io.Discard) diff --git a/cmd/msgvault/cmd/import_pst.go b/cmd/msgvault/cmd/import_pst.go index 3018e1ca3..8ae0b561a 100644 --- a/cmd/msgvault/cmd/import_pst.go +++ b/cmd/msgvault/cmd/import_pst.go @@ -2,6 +2,7 @@ package cmd import ( "context" + "errors" "fmt" "io" "os" @@ -41,6 +42,12 @@ Examples: `, Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger if !isDaemonCLISubprocess() { return runDaemonCLICommandHTTPFromCobra(cmd, args) } @@ -96,7 +103,7 @@ Examples: } }() - st, cleanup, err := openWritableStoreAndInitForIngest() + st, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) if err != nil { return err } @@ -114,13 +121,13 @@ Examples: NoResume: importPstNoResume, CheckpointInterval: importPstCheckpointInterval, AttachmentsDir: attachmentsDir, - RemoteImages: configuredRemoteImageFetcher(), + RemoteImages: configuredRemoteImageFetcher(cfg), Logger: logger, }) if err != nil { return err } - if err := runPstPostImportMigrations(cmd.OutOrStdout(), st, summary, importPstSourceType, identifier); err != nil { + if err := runPstPostImportMigrations(cmd.OutOrStdout(), st, summary, importPstSourceType, identifier, state); err != nil { return err } @@ -172,18 +179,22 @@ func runPstPostImportMigrations( summary *importer.PstImportSummary, sourceType string, identifier string, + state *invocation, ) error { if summary == nil || summary.SourceID == 0 { return nil } + if state == nil || state.logger == nil { + return errors.New("invocation state is required") + } // Auto-default-identity must run BEFORE the legacy migration retry // whenever the migration will run, including interrupted or hard-error // imports, so migrated legacy [identity] rows cannot suppress the // source's own account identifier on a later resume. if store.SourceTypeUsesEmailIdentity(sourceType) { - confirmDefaultIdentity(out, st, summary.SourceID, identifier, identifier, "account-identifier") + confirmDefaultIdentity(out, st, summary.SourceID, identifier, identifier, "account-identifier", state.logger) } - if err := runPostSourceCreateMigrations(st); err != nil { + if err := runPostSourceCreateMigrationsForInvocation(st, state); err != nil { return fmt.Errorf("post-source-create migrations: %w", err) } return nil diff --git a/cmd/msgvault/cmd/import_pst_test.go b/cmd/msgvault/cmd/import_pst_test.go index 5e3c76edd..371e739bd 100644 --- a/cmd/msgvault/cmd/import_pst_test.go +++ b/cmd/msgvault/cmd/import_pst_test.go @@ -2,7 +2,6 @@ package cmd import ( "bytes" - "context" "io" "path/filepath" "testing" @@ -41,7 +40,7 @@ func TestImportPstRunsPostSourceMigrationForEligibleSourceTypes(t *testing.T) { t.Cleanup(saveImportPstState(t)) testCfg := lifecycleTestConfig(tmp) testCfg.Identity.Addresses = []string{"legacy@example.com"} - withStoreResolverConfig(t, testCfg) + testCtx := withStoreResolverConfig(t, testCfg) st, err := store.Open(testCfg.DatabaseDSN()) require.NoError( @@ -68,7 +67,8 @@ func TestImportPstRunsPostSourceMigrationForEligibleSourceTypes(t *testing.T) { var stdout bytes.Buffer cmd := &cobra.Command{Use: "import-pst"} - cmd.SetContext(context.Background()) + cmd.SetContext(testCtx) + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetErr(io.Discard) @@ -115,7 +115,8 @@ func TestRunPstPostImportMigrationsConfirmsDefaultIdentityBeforeHardErrorMigrati tmp := t.TempDir() testCfg := lifecycleTestConfig(tmp) testCfg.Identity.Addresses = []string{"legacy@example.com"} - withStoreResolverConfig(t, testCfg) + testCtx := withStoreResolverConfig(t, testCfg) + _ = testCtx st, err := store.Open(testCfg.DatabaseDSN()) require.NoError( @@ -132,7 +133,7 @@ func TestRunPstPostImportMigrationsConfirmsDefaultIdentityBeforeHardErrorMigrati err = runPstPostImportMigrations(io.Discard, st, &importer.PstImportSummary{ SourceID: src.ID, HardErrors: true, - }, "mbox", "archive@example.com") + }, "mbox", "archive@example.com", invocationFromContext(testCtx)) require.NoError( err, "post-import migrations") diff --git a/cmd/msgvault/cmd/import_slackdump.go b/cmd/msgvault/cmd/import_slackdump.go index bf3a94ccd..bf9e97566 100644 --- a/cmd/msgvault/cmd/import_slackdump.go +++ b/cmd/msgvault/cmd/import_slackdump.go @@ -10,6 +10,7 @@ import ( "github.com/spf13/cobra" "go.kenn.io/msgvault/internal/attachmentpolicy" + "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/slack" "go.kenn.io/msgvault/internal/store" ) @@ -55,6 +56,11 @@ func newImportSlackdumpCmd() *cobra.Command { } func runImportSlackdump(cmd *cobra.Command, sourcePath string, opts slackdumpCLIOptions) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg info, err := os.Stat(sourcePath) if err != nil { return fmt.Errorf("source path not found: %w", err) @@ -64,7 +70,7 @@ func runImportSlackdump(cmd *cobra.Command, sourcePath string, opts slackdumpCLI } dbPath := cfg.DatabaseDSN() - st, cleanup, err := openWritableStoreAndInitForIngest() + st, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) if err != nil { return err } @@ -79,12 +85,12 @@ func runImportSlackdump(cmd *cobra.Command, sourcePath string, opts slackdumpCLI Limit: opts.Limit, AttachmentsDir: cfg.AttachmentsDir(), MediaPolicyForTeam: func(teamID string) attachmentpolicy.Policy { - return resolveSlackdumpMediaPolicy(teamID, opts.MaxMediaMB) + return resolveSlackdumpMediaPolicy(teamID, opts.MaxMediaMB, cfg) }, Progress: func(line string) { writeSlackProgress(cmd.OutOrStdout(), line) }, }) postImportErr := runSlackdumpPostImportMigrations( - cmd.OutOrStdout(), st, summary, opts.NoDefaultIdentity, + cmd.OutOrStdout(), st, summary, opts.NoDefaultIdentity, state, ) if importErr != nil { if ctx.Err() != nil { @@ -93,11 +99,11 @@ func runImportSlackdump(cmd *cobra.Command, sourcePath string, opts slackdumpCLI return errors.Join( fmt.Errorf("import Slackdump: %w", importErr), postImportErr, - rebuildCacheAfterWrite(dbPath), + rebuildCacheAfterWrite(dbPath, state), ) } if postImportErr != nil { - return errors.Join(postImportErr, rebuildCacheAfterWrite(dbPath)) + return errors.Join(postImportErr, rebuildCacheAfterWrite(dbPath, state)) } _, _ = fmt.Fprintln(cmd.OutOrStdout(), "\nImport complete!") @@ -110,7 +116,7 @@ func runImportSlackdump(cmd *cobra.Command, sourcePath string, opts slackdumpCLI if summary.Errors > 0 { _, _ = fmt.Fprintf(cmd.OutOrStdout(), " Errors: %d\n", summary.Errors) } - return rebuildCacheAfterWrite(dbPath) + return rebuildCacheAfterWrite(dbPath, state) } func runSlackdumpPostImportMigrations( @@ -118,10 +124,14 @@ func runSlackdumpPostImportMigrations( st *store.Store, summary *slack.SlackdumpImportSummary, noDefaultIdentity bool, + state *invocation, ) error { if summary == nil || summary.SourceID == 0 { return nil } + if state == nil || state.logger == nil { + return errors.New("invocation state is required") + } var identityErr error if !noDefaultIdentity { source, err := st.GetSourceByID(summary.SourceID) @@ -131,17 +141,21 @@ func runSlackdumpPostImportMigrations( confirmDefaultIdentity( out, st, source.ID, source.Identifier, source.Identifier, "account-identifier", + state.logger, ) } } - migrationErr := runPostSourceCreateMigrations(st) + migrationErr := runPostSourceCreateMigrationsForInvocation(st, state) if migrationErr != nil { migrationErr = fmt.Errorf("post-source-create migrations: %w", migrationErr) } return errors.Join(identityErr, migrationErr) } -func resolveSlackdumpMediaPolicy(teamID string, overrideMB int64) attachmentpolicy.Policy { +func resolveSlackdumpMediaPolicy(teamID string, overrideMB int64, cfg *config.Config) attachmentpolicy.Policy { + if cfg == nil { + return attachmentpolicy.Policy{} + } policy := cfg.Slack.MediaPolicy(teamID) if overrideMB > 0 { policy.MaxBytes = overrideMB << 20 diff --git a/cmd/msgvault/cmd/import_slackdump_test.go b/cmd/msgvault/cmd/import_slackdump_test.go index 885efffcb..622b5f8a9 100644 --- a/cmd/msgvault/cmd/import_slackdump_test.go +++ b/cmd/msgvault/cmd/import_slackdump_test.go @@ -59,7 +59,7 @@ func TestImportSlackdumpCommandForwardsFlagsAndPath(t *testing.T) { func TestRunImportSlackdumpRejectsMissingSourcePath(t *testing.T) { command := &cobra.Command{Use: "import-slackdump"} - command.SetContext(context.Background()) + command.SetContext(testInvocationContext(context.Background(), config.NewDefaultConfig(), invocationOptions{})) command.SetOut(io.Discard) command.SetErr(io.Discard) @@ -69,9 +69,13 @@ func TestRunImportSlackdumpRejectsMissingSourcePath(t *testing.T) { } func TestResolveSlackdumpMediaPolicyPreservesWorkspaceRulesAndOverridesOnlySize(t *testing.T) { + cfg := testConfigValue() + savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx disabled := false cfg.Slack = config.SlackConfig{ Media: &disabled, @@ -89,7 +93,7 @@ func TestResolveSlackdumpMediaPolicyPreservesWorkspaceRulesAndOverridesOnlySize( MaxBytes: 7 << 20, DisabledReason: attachmentpolicy.SkipPolicyScope, } - assert.Equal(t, want, resolveSlackdumpMediaPolicy("T001", 0)) + assert.Equal(t, want, resolveSlackdumpMediaPolicy("T001", 0, cfg)) want.MaxBytes = 3 << 20 - assert.Equal(t, want, resolveSlackdumpMediaPolicy("T001", 3)) + assert.Equal(t, want, resolveSlackdumpMediaPolicy("T001", 3, cfg)) } diff --git a/cmd/msgvault/cmd/import_synctech_sms.go b/cmd/msgvault/cmd/import_synctech_sms.go index 74dceb91f..0f1a574ee 100644 --- a/cmd/msgvault/cmd/import_synctech_sms.go +++ b/cmd/msgvault/cmd/import_synctech_sms.go @@ -21,6 +21,11 @@ func newImportSynctechSMSCmd() *cobra.Command { SMS Backup & Restore by SyncTech Pty Ltd.`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg if opts.OwnerPhone == "" { return errors.New("--owner-phone is required") } @@ -28,7 +33,7 @@ SMS Backup & Restore by SyncTech Pty Ltd.`, return runDaemonCLICommandHTTPFromCobra(cmd, args) } opts.AttachmentsDir = cfg.AttachmentsDir() - st, cleanup, err := openWritableStoreAndInitForIngest() + st, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) if err != nil { return err } diff --git a/cmd/msgvault/cmd/import_synctech_sms_test.go b/cmd/msgvault/cmd/import_synctech_sms_test.go index 44a37f729..8361751f0 100644 --- a/cmd/msgvault/cmd/import_synctech_sms_test.go +++ b/cmd/msgvault/cmd/import_synctech_sms_test.go @@ -11,10 +11,13 @@ import ( func TestImportSynctechSMSRequiresOwnerPhone(t *testing.T) { dir := t.TempDir() - cfg = config.NewDefaultConfig() + cfg := config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.HomeDir = t.TempDir() cfg.Data.DataDir = cfg.HomeDir cmd := newTestRootCmd() + cmd.SetContext(testCtx) cmd.AddCommand(newImportSynctechSMSCmd()) cmd.SetArgs([]string{"import-synctech-sms", dir}) err := cmd.Execute() @@ -29,10 +32,13 @@ func TestImportSynctechSMSCommandRuns(t *testing.T) { input := filepath.Join(t.TempDir(), "sms.xml") err := os.WriteFile(input, []byte(``), 0o600) require.NoError(t, err, "write fixture") - cfg = config.NewDefaultConfig() + cfg := config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.HomeDir = home cfg.Data.DataDir = home cmd := newTestRootCmd() + cmd.SetContext(testCtx) cmd.AddCommand(newImportSynctechSMSCmd()) cmd.SetArgs([]string{"import-synctech-sms", "--owner-phone", "+15550000001", input}) require.NoError(t, cmd.Execute(), "Execute") diff --git a/cmd/msgvault/cmd/import_whatsapp_identity_test.go b/cmd/msgvault/cmd/import_whatsapp_identity_test.go index ef84492f9..febc6ce1c 100644 --- a/cmd/msgvault/cmd/import_whatsapp_identity_test.go +++ b/cmd/msgvault/cmd/import_whatsapp_identity_test.go @@ -2,7 +2,6 @@ package cmd import ( "bytes" - "context" "database/sql" "io" "os" @@ -99,6 +98,7 @@ func runWhatsAppIdentityCommand( noDefaultIdentity bool, ) (string, error) { t.Helper() + cfg := testConfigValue() oldCfg := cfg oldNoDefaultIdentity := noDefaultIdentityImportWhatsApp @@ -117,7 +117,7 @@ func runWhatsAppIdentityCommand( importDisplayName = oldImportDisplayName }) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ HomeDir: home, Data: config.DataConfig{ DataDir: home, @@ -136,7 +136,8 @@ func runWhatsAppIdentityCommand( var output strings.Builder cmd := &cobra.Command{} - cmd.SetContext(context.Background()) + cmd.SetContext(testCtx) + cmd.SetContext(testCtx) cmd.SetOut(&output) cmd.SetErr(io.Discard) oldStdout := os.Stdout diff --git a/cmd/msgvault/cmd/initdb.go b/cmd/msgvault/cmd/initdb.go index a528097f3..a1c8a5763 100644 --- a/cmd/msgvault/cmd/initdb.go +++ b/cmd/msgvault/cmd/initdb.go @@ -1,6 +1,7 @@ package cmd import ( + "errors" "fmt" "io" @@ -17,6 +18,12 @@ This command creates all necessary tables for storing messages, attachments, labels, and sync state. It is safe to run multiple times - tables are only created if they don't already exist.`, RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger dbPath := cfg.DatabaseDSN() logger.Info("initializing database", "path", dbPath) diff --git a/cmd/msgvault/cmd/initdb_test.go b/cmd/msgvault/cmd/initdb_test.go index f0730831f..11cd0b98e 100644 --- a/cmd/msgvault/cmd/initdb_test.go +++ b/cmd/msgvault/cmd/initdb_test.go @@ -16,6 +16,8 @@ import ( ) func TestInitDBUsesConfiguredRemoteHTTPAndPreservesOutput(t *testing.T) { + logger := testLoggerValue() + assert := assert.New(t) var requests atomic.Int32 @@ -40,7 +42,7 @@ func TestInitDBUsesConfiguredRemoteHTTPAndPreservesOutput(t *testing.T) { t.Cleanup(server.Close) dataDir := t.TempDir() - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ HomeDir: dataDir, Data: config.DataConfig{DataDir: dataDir}, Remote: config.RemoteConfig{ @@ -55,6 +57,7 @@ func TestInitDBUsesConfiguredRemoteHTTPAndPreservesOutput(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer cmd := &cobra.Command{Use: initDBCmd.Use, RunE: initDBCmd.RunE} + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetErr(&stderr) diff --git a/cmd/msgvault/cmd/invocation.go b/cmd/msgvault/cmd/invocation.go new file mode 100644 index 000000000..f81490c16 --- /dev/null +++ b/cmd/msgvault/cmd/invocation.go @@ -0,0 +1,239 @@ +package cmd + +import ( + "context" + "log/slog" + "os" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/logging" +) + +// invocationOptions is the set of root-owned options that belong to one +// execution. Cobra owns the parsed flag values; this snapshot survives child +// command forwarding and delayed work after the command tree has moved on. +type invocationOptions struct { + cfgFile string + homeDir string + verbose bool + useLocal bool + logFile string + logLevel string + noLogFile bool + logSQL bool + logSQLSlow int64 + + cfgFileChanged bool + homeDirChanged bool + verboseChanged bool + useLocalChanged bool + logFileChanged bool + logLevelChanged bool + noLogChanged bool + logSQLChanged bool + logSlowChanged bool + + agentURL string + agentTokenFile string + agentAllowInsecure bool + agentURLChanged bool + agentTokenChanged bool +} + +type invocation struct { + options invocationOptions + cfg *config.Config + logger *slog.Logger + logResult *logging.Result +} + +type invocationContextKey struct{} + +func newInvocation() *invocation { + return &invocation{ + logger: slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo})), + } +} + +func withInvocation(ctx context.Context, inv *invocation) context.Context { + if ctx == nil { + ctx = context.Background() + } + return context.WithValue(ctx, invocationContextKey{}, inv) +} + +func invocationBoundJobRun(inv *invocation, run func(context.Context) error) func(context.Context) error { + return func(ctx context.Context) error { + return run(withInvocation(ctx, inv)) + } +} + +func invocationFromContext(ctx context.Context) *invocation { + if ctx == nil { + return nil + } + inv, _ := ctx.Value(invocationContextKey{}).(*invocation) + return inv +} + +func invocationFromCommand(cmd *cobra.Command) *invocation { + if cmd == nil { + return nil + } + if inv := invocationFromContext(cmd.Context()); inv != nil { + return inv + } + return invocationFromContext(cmd.Root().Context()) +} + +func loggerFromContext(ctx context.Context) *slog.Logger { + if inv := invocationFromContext(ctx); inv != nil && inv.logger != nil { + return inv.logger + } + return slog.New(slog.DiscardHandler) +} + +func optionsFromContext(ctx context.Context) invocationOptions { + if inv := invocationFromContext(ctx); inv != nil { + return inv.options + } + return invocationOptions{} +} + +func prepareInvocation(cmd *cobra.Command) *invocation { + if cmd == nil { + return nil + } + root := cmd.Root() + inv := invocationFromContext(root.Context()) + if inv == nil { + inv = newInvocation() + root.SetContext(withInvocation(root.Context(), inv)) + } + if inv.logger == nil { + inv.logger = newInvocation().logger + } + + // Read root-owned options from the executing root. A reusable leaf can + // retain inherited flags from an earlier parent, so cmd.Flags() may expose + // stale values from another invocation tree. + flags := root.PersistentFlags() + if cmd == root && flags.Lookup("agent-url") == nil { + flags = cmd.Flags() + } + if flags.Lookup("config") != nil { + inv.options.cfgFile = invocationStringFlag(flags, "config") + } + if flags.Lookup("home") != nil { + inv.options.homeDir = invocationStringFlag(flags, "home") + } + if flags.Lookup("verbose") != nil { + inv.options.verbose = invocationBoolFlag(flags, "verbose") + } + if flags.Lookup(localValue) != nil { + inv.options.useLocal = invocationBoolFlag(flags, localValue) + } + if flags.Lookup("log-file") != nil { + inv.options.logFile = invocationStringFlag(flags, "log-file") + } + if flags.Lookup("log-level") != nil { + inv.options.logLevel = invocationStringFlag(flags, "log-level") + } + if flags.Lookup("no-log-file") != nil { + inv.options.noLogFile = invocationBoolFlag(flags, "no-log-file") + } + if flags.Lookup("log-sql") != nil { + inv.options.logSQL = invocationBoolFlag(flags, "log-sql") + } + if flags.Lookup("log-sql-slow-ms") != nil { + inv.options.logSQLSlow = invocationInt64Flag(flags, "log-sql-slow-ms") + } + if flags.Lookup("agent-url") != nil { + inv.options.agentURL = invocationStringFlag(flags, "agent-url") + } + if flags.Lookup("agent-token-file") != nil { + inv.options.agentTokenFile = invocationStringFlag(flags, "agent-token-file") + } + if flags.Lookup("agent-allow-insecure") != nil { + inv.options.agentAllowInsecure = invocationBoolFlag(flags, "agent-allow-insecure") + } + if flag := flags.Lookup("agent-url"); flag != nil { + inv.options.agentURLChanged = flag.Changed + } + if flag := flags.Lookup("agent-token-file"); flag != nil { + inv.options.agentTokenChanged = flag.Changed + } + inv.options.cfgFileChanged = invocationFlagChanged(flags, "config") + inv.options.homeDirChanged = invocationFlagChanged(flags, "home") + inv.options.verboseChanged = invocationFlagChanged(flags, "verbose") + inv.options.useLocalChanged = invocationFlagChanged(flags, localValue) + inv.options.logFileChanged = invocationFlagChanged(flags, "log-file") + inv.options.logLevelChanged = invocationFlagChanged(flags, "log-level") + inv.options.noLogChanged = invocationFlagChanged(flags, "no-log-file") + inv.options.logSQLChanged = invocationFlagChanged(flags, "log-sql") + inv.options.logSlowChanged = invocationFlagChanged(flags, "log-sql-slow-ms") + + // Cobra retains a leaf's context between Execute calls. Refresh it from + // the root so a repeated execution cannot keep the prior run's state. + cmd.SetContext(root.Context()) + return inv +} + +func invocationStringFlag(flags *pflag.FlagSet, name string) string { + if flags == nil { + return "" + } + value, err := flags.GetString(name) + if err != nil { + return "" + } + return value +} + +func invocationBoolFlag(flags *pflag.FlagSet, name string) bool { + if flags == nil { + return false + } + value, err := flags.GetBool(name) + if err != nil { + return false + } + return value +} + +func invocationInt64Flag(flags *pflag.FlagSet, name string) int64 { + if flags == nil { + return 0 + } + value, err := flags.GetInt64(name) + if err != nil { + return 0 + } + return value +} + +func invocationFlagChanged(flags *pflag.FlagSet, name string) bool { + if flags == nil { + return false + } + flag := flags.Lookup(name) + return flag != nil && flag.Changed +} + +func clearInvocationFlags(root *cobra.Command) { + if root == nil { + return + } + for _, name := range []string{ + "config", "home", "verbose", localValue, "log-file", "log-level", + "no-log-file", "log-sql", "log-sql-slow-ms", "agent-url", + "agent-token-file", "agent-allow-insecure", + } { + if flag := root.PersistentFlags().Lookup(name); flag != nil { + _ = flag.Value.Set(flag.DefValue) + flag.Changed = false + } + } +} diff --git a/cmd/msgvault/cmd/invocation_test.go b/cmd/msgvault/cmd/invocation_test.go new file mode 100644 index 000000000..33b3e768d --- /dev/null +++ b/cmd/msgvault/cmd/invocation_test.go @@ -0,0 +1,359 @@ +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/logging" + "go.kenn.io/msgvault/internal/mcpdiscovery" + "go.kenn.io/msgvault/internal/scheduler" +) + +func testConfigValue() *config.Config { return config.NewDefaultConfig() } + +func testDiscardLogger() *slog.Logger { return slog.New(slog.DiscardHandler) } + +func testInvocationWithConfig(cfg *config.Config) *invocation { + return &invocation{cfg: cfg, logger: testDiscardLogger()} +} + +func testLoggerValue() *slog.Logger { return slog.New(slog.DiscardHandler) } + +func testInvocationContext(ctx context.Context, cfg *config.Config, options invocationOptions) context.Context { + state := newInvocation() + state.cfg = cfg + state.options = options + return withInvocation(ctx, state) +} + +func TestInvocationBoundJobRun(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + state := newInvocation() + state.cfg = testConfigValue() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + job := scheduler.Job{ + Run: invocationBoundJobRun(state, func(ctx context.Context) error { + assert.Same(state, invocationFromContext(ctx)) + assert.Same(state.cfg, invocationFromContext(ctx).cfg) + assert.ErrorIs(ctx.Err(), context.Canceled) + return nil + }), + } + require.NoError(job.Run(ctx)) +} + +func TestInvocationIsolation(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + defaultHome := t.TempDir() + t.Setenv("MSGVAULT_HOME", defaultHome) + explicitHome := t.TempDir() + root := newRootCommand() + root.SetOut(io.Discard) + root.SetErr(io.Discard) + + var homes []string + root.AddCommand(&cobra.Command{ + Use: "probe", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + inv := invocationFromCommand(command) + require.NotNil(inv) + require.NotNil(inv.cfg) + homes = append(homes, inv.cfg.HomeDir) + return nil + }, + }) + + root.SetArgs([]string{"--home", explicitHome, "--no-log-file", "probe"}) + require.NoError(executeRootContext(context.Background(), root)) + root.SetArgs([]string{"probe"}) + require.NoError(executeRootContext(context.Background(), root)) + + require.Len(homes, 2) + assert.Equal(explicitHome, homes[0]) + assert.Equal(defaultHome, homes[1]) +} + +func TestInvocationRepeatedExecution(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + defaultHome := t.TempDir() + explicitHome := t.TempDir() + t.Setenv("MSGVAULT_HOME", defaultHome) + + root := rootCmd + root.SetOut(io.Discard) + root.SetErr(io.Discard) + var seenHomes []string + var seenInvocations []*invocation + probe := &cobra.Command{ + Use: "invocation-probe", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + inv := invocationFromCommand(command) + require.NotNil(inv) + require.NotNil(inv.cfg) + require.NotNil(inv.logResult) + seenInvocations = append(seenInvocations, inv) + seenHomes = append(seenHomes, inv.cfg.HomeDir) + return nil + }, + } + root.AddCommand(probe) + t.Cleanup(func() { root.RemoveCommand(probe) }) + + root.SetArgs([]string{"--home", explicitHome, "--no-log-file", "invocation-probe"}) + require.NoError(executeRootContext(context.Background(), root)) + root.SetArgs([]string{"invocation-probe"}) + require.NoError(executeRootContext(context.Background(), root)) + + require.Len(seenHomes, 2) + assert.Equal(explicitHome, seenHomes[0]) + assert.Equal(defaultHome, seenHomes[1]) + assert.NotSame(seenInvocations[0], seenInvocations[1]) +} + +func TestInvocationLogLifecycle(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + root := newRootCommand() + root.SetOut(io.Discard) + root.SetErr(io.Discard) + var skippedConfig, skippedResult bool + root.AddCommand(&cobra.Command{ + Use: "version", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + inv := invocationFromCommand(command) + require.NotNil(inv) + skippedConfig = inv.cfg != nil + skippedResult = inv.logResult != nil + return nil + }, + }) + root.AddCommand(&cobra.Command{ + Use: "probe ", + Args: cobra.ExactArgs(1), + RunE: func(_ *cobra.Command, args []string) error { + if args[0] == "error" { + return errors.New("probe failure") + } + slog.Info("probe body", "mode", args[0]) + return nil + }, + }) + home := t.TempDir() + for _, test := range []struct { + mode, outcome string + }{ + {mode: "ok", outcome: "ok"}, + {mode: "error", outcome: "error"}, + } { + logPath := filepath.Join(t.TempDir(), test.mode+".log") + root.SetArgs([]string{ + "--home", home, "--log-file", logPath, "probe", test.mode, + }) + err := executeRootContext(context.Background(), root) + if test.outcome == "ok" { + require.NoError(err) + } else { + require.ErrorContains(err, "probe failure") + } + data, readErr := os.ReadFile(logPath) + require.NoError(readErr) + lines := strings.Split(strings.TrimSpace(string(data)), "\n") + require.NotEmpty(lines) + last := lines[len(lines)-1] + assert.Contains(last, `"msg":"msgvault exit"`) + assert.Contains(last, `"outcome":"`+test.outcome+`"`) + require.NoError(os.Rename(logPath, logPath+".closed"), "log file must be closed after execution") + } + + skippedHome := t.TempDir() + root.SetArgs([]string{"--home", skippedHome, "version"}) + require.NoError(executeRootContext(context.Background(), root)) + assert.False(skippedConfig) + assert.False(skippedResult) + assert.NoDirExists(filepath.Join(skippedHome, "logs")) + assert.NoFileExists(filepath.Join(skippedHome, "config.toml")) + + badConfig := filepath.Join(t.TempDir(), "bad.toml") + require.NoError(os.WriteFile(badConfig, []byte("not = [valid"), 0o600)) + root.SetArgs([]string{"--config", badConfig, "probe", "ok"}) + require.Error(executeRootContext(context.Background(), root)) + inv := invocationFromContext(root.Context()) + require.NotNil(inv) + assert.Nil(inv.logResult, "a failed load must not reuse a prior logging result") + assert.Nil(inv.cfg, "a failed load must not reuse a prior config") +} + +func TestInvocationPanicCleanup(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + logPath := filepath.Join(t.TempDir(), "panic.log") + cmd := exec.Command(os.Args[0], "-test.run=^TestInvocationPanicHelper$") //nolint:gosec // test binary and fixed test selector. + cmd.Env = append(os.Environ(), + "MSGVAULT_INVOCATION_PANIC_HELPER=1", + "MSGVAULT_INVOCATION_PANIC_LOG="+logPath, + ) + output, err := cmd.CombinedOutput() + var exitErr *exec.ExitError + require.ErrorAs(err, &exitErr) + assert.Equal(2, exitErr.ExitCode(), string(output)) + data, readErr := os.ReadFile(logPath) + require.NoError(readErr) + assert.Contains(string(data), `"msg":"msgvault panic"`) + require.NoError(os.Rename(logPath, logPath+".closed"), "panic cleanup must close the log file") +} + +func TestInvocationPanicHelper(t *testing.T) { + if os.Getenv("MSGVAULT_INVOCATION_PANIC_HELPER") != "1" { + return + } + root := newRootCommand() + root.AddCommand(&cobra.Command{ + Use: "panic", + RunE: func(*cobra.Command, []string) error { + panic("invocation panic") + }, + }) + root.SetArgs([]string{ + "--home", filepath.Dir(os.Getenv("MSGVAULT_INVOCATION_PANIC_LOG")), + "--log-file", os.Getenv("MSGVAULT_INVOCATION_PANIC_LOG"), "panic", + }) + _ = executeRootContext(context.Background(), root) + os.Exit(3) +} + +func TestTUILoggerRestoration(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + logPath := filepath.Join(t.TempDir(), "tui.log") + result, err := logging.BuildHandler(logging.Options{ + FilePath: logPath, Stderr: io.Discard, + }) + require.NoError(err) + t.Cleanup(result.Close) + previous := slog.New(slog.DiscardHandler) + slog.SetDefault(previous) + t.Cleanup(func() { slog.SetDefault(previous) }) + inside := false + require.NoError(withTUIFileLogger(result, func() error { + inside = slog.Default() != previous + return nil + })) + assert.True(inside) + assert.Same(previous, slog.Default()) +} + +func TestInvocationUsageContract(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + newRoot := func(command *cobra.Command, args ...string) (string, error) { + root := newRootCommand() + var stderr bytes.Buffer + root.SetOut(&stderr) + root.SetErr(&stderr) + root.AddCommand(command) + root.SetArgs(args) + err := executeRootContext(context.Background(), root) + return stderr.String(), err + } + + usage, err := newRoot(&cobra.Command{ + Use: "required ", + Args: cobra.ExactArgs(1), + RunE: func(*cobra.Command, []string) error { return nil }, + }, "required") + require.Error(err) + assert.Contains(usage, "Usage:") + + usage, err = newRoot(&cobra.Command{ + Use: "contract", + RunE: func(command *cobra.Command, _ []string) error { + return usageErr(command, errors.New("contract failure")) + }, + }, "--no-log-file", "--log-level", "error", "contract") + require.ErrorContains(err, "contract failure") + assert.Contains(usage, "Usage:") + + usage, err = newRoot(&cobra.Command{ + Use: "runtime", + RunE: func(*cobra.Command, []string) error { return errors.New("runtime failure") }, + }, "--no-log-file", "--log-level", "error", "runtime") + require.ErrorContains(err, "runtime failure") + assert.NotContains(usage, "Usage:") +} + +func TestInvocationMCPStatusOptions(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + defaultHome := t.TempDir() + homeA := t.TempDir() + homeB := t.TempDir() + t.Setenv("MSGVAULT_HOME", defaultHome) + + publish := func(home, address string) func() error { + cleanup, err := mcpdiscovery.Publish(filepath.Join(home, "mcp"), address, "", "") + require.NoError(err) + return cleanup + } + cleanupDefault := publish(defaultHome, "127.0.0.1:9100") + defer func() { require.NoError(cleanupDefault()) }() + cleanupA := publish(homeA, "127.0.0.1:9101") + defer func() { require.NoError(cleanupA()) }() + cleanupB := publish(homeB, "127.0.0.1:9102") + defer func() { require.NoError(cleanupB()) }() + + configA := filepath.Join(t.TempDir(), "a.toml") + configB := filepath.Join(t.TempDir(), "b.toml") + require.NoError(os.WriteFile(configA, []byte("[server]\napi_port = 9103\n"), 0o600)) + require.NoError(os.WriteFile(configB, []byte("[server]\napi_port = 9104\n"), 0o600)) + cleanupConfigA := publish(filepath.Dir(configA), "127.0.0.1:9103") + defer func() { require.NoError(cleanupConfigA()) }() + cleanupConfigB := publish(filepath.Dir(configB), "127.0.0.1:9104") + defer func() { require.NoError(cleanupConfigB()) }() + + root := rootCmd + root.SetErr(io.Discard) + root.SetOut(io.Discard) + status := func(args ...string) mcpdiscovery.Endpoint { + var output bytes.Buffer + root.SetOut(&output) + root.SetArgs(args) + require.NoError(executeRootContext(context.Background(), root)) + var endpoints []mcpdiscovery.Endpoint + require.NoError(json.Unmarshal(output.Bytes(), &endpoints)) + require.Len(endpoints, 1) + return endpoints[0] + } + + assert.Equal("http://127.0.0.1:9101/mcp", status("--home", homeA, "mcp", "status", "--json").URL) + assert.Equal("http://127.0.0.1:9102/mcp", status("--home", homeB, "mcp", "status", "--json").URL) + assert.Equal("http://127.0.0.1:9103/mcp", status("--config", configA, "mcp", "status", "--json").URL) + assert.Equal("http://127.0.0.1:9104/mcp", status("--config", configB, "mcp", "status", "--json").URL) + assert.Equal("http://127.0.0.1:9100/mcp", status("mcp", "status", "--json").URL) + + for _, home := range []string{homeA, homeB, filepath.Dir(configA), filepath.Dir(configB), defaultHome} { + assert.NoDirExists(filepath.Join(home, "logs")) + } + for _, home := range []string{homeA, homeB, defaultHome} { + assert.NoFileExists(filepath.Join(home, "config.toml")) + } +} diff --git a/cmd/msgvault/cmd/list_accounts_test.go b/cmd/msgvault/cmd/list_accounts_test.go index 44bfe6e7e..f4c6a2b84 100644 --- a/cmd/msgvault/cmd/list_accounts_test.go +++ b/cmd/msgvault/cmd/list_accounts_test.go @@ -14,6 +14,9 @@ import ( ) func TestListAccountsUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) dataDir := t.TempDir() @@ -33,11 +36,15 @@ func TestListAccountsUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { HomeDir: dataDir, Data: config.DataConfig{DataDir: dataDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true listAccountsJSON = false done := captureStdout(t) cmd := &cobra.Command{Use: "list-accounts", RunE: listAccountsCmd.RunE} + cmd.SetContext(testCtx) err := cmd.Execute() out := done() diff --git a/cmd/msgvault/cmd/list_aggregates_test.go b/cmd/msgvault/cmd/list_aggregates_test.go index 13d702273..c655752e0 100644 --- a/cmd/msgvault/cmd/list_aggregates_test.go +++ b/cmd/msgvault/cmd/list_aggregates_test.go @@ -14,6 +14,9 @@ import ( ) func TestListSendersUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) dataDir := t.TempDir() @@ -40,7 +43,10 @@ func TestListSendersUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { Data: config.DataConfig{DataDir: dataDir}, Remote: config.RemoteConfig{URL: "http://configured-daemonclient.invalid"}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true aggLimit = 50 aggAfter = "" aggBefore = "" @@ -48,6 +54,7 @@ func TestListSendersUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { done := captureStdout(t) cmd := &cobra.Command{Use: "list-senders", RunE: listSendersCmd.RunE} + cmd.SetContext(testCtx) err := cmd.Execute() out := done() diff --git a/cmd/msgvault/cmd/listfolders.go b/cmd/msgvault/cmd/listfolders.go index 99309300c..0af3c8847 100644 --- a/cmd/msgvault/cmd/listfolders.go +++ b/cmd/msgvault/cmd/listfolders.go @@ -4,9 +4,11 @@ import ( "context" "errors" "fmt" + "log/slog" "strings" "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/config" imapclient "go.kenn.io/msgvault/internal/imap" "go.kenn.io/msgvault/internal/microsoft" "go.kenn.io/msgvault/internal/store" @@ -33,7 +35,13 @@ IMAP accounts, along with message count. This helps you } func runListFoldersLocal(cmd *cobra.Command, args []string) error { - s, cleanup, err := openWritableStoreAndInit() + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger + s, cleanup, err := openWritableStoreAndInitForInvocation(state) if err != nil { return err } @@ -70,14 +78,14 @@ func runListFoldersLocal(cmd *cobra.Command, args []string) error { if i > 0 { fmt.Println() } - if err := listFolders(ctx, src); err != nil { + if err := listFolders(ctx, src, cfg, logger); err != nil { return err } } return nil } -func listFolders(ctx context.Context, src *store.Source) error { +func listFolders(ctx context.Context, src *store.Source, cfg *config.Config, logger *slog.Logger) error { if cfg == nil { return errors.New("configuration not loaded") } @@ -87,7 +95,7 @@ func listFolders(ctx context.Context, src *store.Source) error { } fmt.Printf("Account: %s\n", displayID) - skip, err := imapSkipReason(src) + skip, err := imapSkipReason(src, cfg, logger) if err != nil { return err } diff --git a/cmd/msgvault/cmd/listfolders_test.go b/cmd/msgvault/cmd/listfolders_test.go index fcd43d0c8..67c327dbd 100644 --- a/cmd/msgvault/cmd/listfolders_test.go +++ b/cmd/msgvault/cmd/listfolders_test.go @@ -14,6 +14,9 @@ import ( ) func TestListFoldersCmd_NoIMAPAccounts(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + require := require.New(t) assert := assert.New(t) t.Setenv(daemonCLISubprocessEnv, strconv.Itoa(os.Getppid())) @@ -41,9 +44,12 @@ func TestListFoldersCmd_NoIMAPAccounts(t *testing.T) { HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(newListFoldersCmd()) root.SetArgs([]string{"list-folders"}) @@ -53,6 +59,9 @@ func TestListFoldersCmd_NoIMAPAccounts(t *testing.T) { } func TestListFoldersCmd_GmailIdentifier(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + require := require.New(t) assert := assert.New(t) t.Setenv(daemonCLISubprocessEnv, strconv.Itoa(os.Getppid())) @@ -79,9 +88,12 @@ func TestListFoldersCmd_GmailIdentifier(t *testing.T) { HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(newListFoldersCmd()) root.SetArgs([]string{"list-folders", "g@example.com"}) @@ -91,6 +103,9 @@ func TestListFoldersCmd_GmailIdentifier(t *testing.T) { } func TestListFoldersCmd_IMAPNoCredentials(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + require := require.New(t) assert := assert.New(t) t.Setenv(daemonCLISubprocessEnv, strconv.Itoa(os.Getppid())) @@ -122,12 +137,15 @@ func TestListFoldersCmd_IMAPNoCredentials(t *testing.T) { Data: config.DataConfig{DataDir: tmpDir}, OAuth: config.OAuthConfig{ClientSecrets: secretsPath}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) // Capture stdout getOutput := captureStdout(t) root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(newListFoldersCmd()) root.SetArgs([]string{"list-folders", "i@example.com"}) @@ -140,6 +158,9 @@ func TestListFoldersCmd_IMAPNoCredentials(t *testing.T) { } func TestListFoldersCmd_ListAllPrintsEachSource(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + require := require.New(t) assert := assert.New(t) t.Setenv(daemonCLISubprocessEnv, strconv.Itoa(os.Getppid())) @@ -173,12 +194,15 @@ func TestListFoldersCmd_ListAllPrintsEachSource(t *testing.T) { Data: config.DataConfig{DataDir: tmpDir}, OAuth: config.OAuthConfig{ClientSecrets: secretsPath}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) // Both sources should have "Credentials not found" in stdout getOutput := captureStdout(t) root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(newListFoldersCmd()) root.SetArgs([]string{"list-folders"}) @@ -192,6 +216,9 @@ func TestListFoldersCmd_ListAllPrintsEachSource(t *testing.T) { } func TestListFoldersCmd_BrokenOAuthDoesNotBlockIMAP(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + require := require.New(t) assert := assert.New(t) t.Setenv(daemonCLISubprocessEnv, strconv.Itoa(os.Getppid())) @@ -225,11 +252,14 @@ func TestListFoldersCmd_BrokenOAuthDoesNotBlockIMAP(t *testing.T) { Data: config.DataConfig{DataDir: tmpDir}, OAuth: config.OAuthConfig{ClientSecrets: secretsPath}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) getOutput := captureStdout(t) root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(newListFoldersCmd()) root.SetArgs([]string{"list-folders"}) diff --git a/cmd/msgvault/cmd/logs.go b/cmd/msgvault/cmd/logs.go index af15938a1..a070e05a3 100644 --- a/cmd/msgvault/cmd/logs.go +++ b/cmd/msgvault/cmd/logs.go @@ -4,6 +4,7 @@ import ( "bufio" "context" "encoding/json/v2" + "errors" "fmt" "io" "os" @@ -70,6 +71,11 @@ func runLogsCmd(cmd *cobra.Command, args []string) error { if !isDaemonCLISubprocess() { return runDaemonCLICommandHTTPFromCobra(cmd, args) } + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg dir := cfg.LogsDir() serveLogPath := filepath.Join(cfg.Data.DataDir, "serve.log") diff --git a/cmd/msgvault/cmd/logs_routing_test.go b/cmd/msgvault/cmd/logs_routing_test.go index 53e41225a..45fd74630 100644 --- a/cmd/msgvault/cmd/logs_routing_test.go +++ b/cmd/msgvault/cmd/logs_routing_test.go @@ -24,11 +24,13 @@ func TestLogsCommandUsesDaemonRunner(t *testing.T) { "--run-id=abc123", }, req.Args, "args") }, `{"type":"stdout","data":"12:00:00 WARN abc123 sync failed\n"}`, `{"type":"stderr","data":"tail warning\n"}`, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx var stdout bytes.Buffer var stderr bytes.Buffer cmd := newLogsRoutingTestCommand() + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetErr(&stderr) cmd.SetArgs([]string{ diff --git a/cmd/msgvault/cmd/mcp.go b/cmd/msgvault/cmd/mcp.go index 7334d9209..9f0b3cccc 100644 --- a/cmd/msgvault/cmd/mcp.go +++ b/cmd/msgvault/cmd/mcp.go @@ -4,15 +4,19 @@ import ( "context" "errors" "fmt" + "log/slog" "net" "path/filepath" "strconv" "strings" "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/daemonclient" "go.kenn.io/msgvault/internal/deletion" mcpserver "go.kenn.io/msgvault/internal/mcp" + "go.kenn.io/msgvault/internal/providercredentials" "go.kenn.io/msgvault/internal/vector/visual" "go.kenn.io/msgvault/pkg/client/generated" ) @@ -23,6 +27,12 @@ var mcpHTTPAddr string var mcpHTTPAllowInsecure bool var mcpHTTPAllowWrites bool var mcpAllowProfileWrites bool +var mcpAllowIdentityDecisions bool +var mcpAllowIdentityScoring bool +var mcpAllowPersonMerges bool +var mcpAllowCardDAVWrites bool +var mcpAllowCalendarWrites bool +var serveMCPStdioWithOptions = mcpserver.ServeWithOptions var serveMCPHTTPWithOptions = mcpserver.ServeHTTPWithOptions var mcpCmd = &cobra.Command{ @@ -31,8 +41,8 @@ var mcpCmd = &cobra.Command{ Long: `Start an MCP (Model Context Protocol) server over stdio. This allows Claude Desktop (or any MCP client) to query your archive -using tools like search_metadata, search_message_bodies, search_document_attachments, semantic_search_messages, get_message, list_messages, get_stats, -aggregate, list_saved_views, run_saved_view, and stage_deletion. +using tools like search_metadata, search_message_bodies, search_document_attachments, semantic_search_messages, get_message, list_messages, list_thread, export_eml, get_stats, +aggregate, get_person_agenda, list_saved_views, run_saved_view, and stage_deletion. Add to Claude Desktop config: { @@ -44,11 +54,34 @@ Add to Claude Desktop config: } }`, RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || (state.cfg == nil && !isAgentMode(state)) { + return errors.New("configuration is unavailable") + } + if isAgentMode(state) && mcpHTTPAddr != "" { + return usageErr(cmd, errors.New("delegated MCP supports stdio only")) + } + cfg := state.cfg + httpAddr, inboundKey, err := prepareMCPHTTP(cmd, cfg) + if err != nil { + return usageErr(cmd, err) + } st, info, err := OpenHTTPStore(cmd.Context()) if err != nil { return fmt.Errorf("open daemon: %w", err) } defer func() { _ = st.Close() }() + if httpAddr != "" { + if !cmd.Flags().Changed("http-token-file") && !cmd.Flags().Changed("http-token-env") { + // Local startup may have created the key. OpenHTTPStore refreshes + // it after discovering or starting the daemon that owns the archive. + inboundKey = cfg.Server.AuthenticationKey() + } + httpAddr, err = normalizeMCPHTTPAddr(httpAddr, mcpHTTPAllowInsecure, inboundKey != "") + if err != nil { + return usageErr(cmd, err) + } + } // Derive from cmd.Context() so signal handling installed by // the cobra root command (SIGINT/SIGTERM → ctx.Done()) reaches @@ -57,51 +90,136 @@ Add to Claude Desktop config: ctx, cancel := context.WithCancel(cmd.Context()) defer cancel() - opts, err := daemonMCPServeOptions(ctx, st) - if err != nil { - return err + opts := daemonMCPServeOptions(ctx, st, state) + if isAgentMode(state) && opts.Calendar == nil { + return fmt.Errorf("calendar delegation requires a daemon with API schema %s or later", calendarControlMinAPISchemaVersion) } opts.AllowProfileWrites = mcpAllowProfileWrites + opts.AllowIdentityDecisions = mcpAllowIdentityDecisions + opts.AllowIdentityScoring = mcpAllowIdentityScoring + opts.AllowPersonMerges = mcpAllowPersonMerges + opts.AllowCardDAVWrites = mcpAllowCardDAVWrites + opts.AllowCalendarWrites = mcpAllowCalendarWrites - if mcpHTTPAddr != "" { - normalized, err := normalizeMCPHTTPAddr( - mcpHTTPAddr, - mcpHTTPAllowInsecure, - cfg.Server.APIKey != "", - ) - if err != nil { - return usageErr(cmd, err) - } + if httpAddr != "" { return serveMCPHTTPWithOptions(ctx, opts, mcpserver.HTTPOptions{ - Addr: normalized, + Addr: httpAddr, DiscoveryDirectory: filepath.Join(cfg.HomeDir, "mcp"), BackendURL: info.URL, - APIKey: cfg.Server.APIKey, + APIKey: inboundKey, AllowWrites: mcpHTTPAllowWrites, }) } - return mcpserver.ServeWithOptions(ctx, opts) + return serveMCPStdioWithOptions(ctx, opts) }, } +// prepareMCPHTTP validates inbound sources and the address without creating keys. +// Explicit inbound sources leave the daemon's unused server credential alone. +func prepareMCPHTTP(cmd *cobra.Command, cfg *config.Config) (string, string, error) { + fileSet := cmd.Flags().Changed("http-token-file") + envSet := cmd.Flags().Changed("http-token-env") + if mcpHTTPAddr == "" { + if fileSet || envSet || cmd.Flags().Changed("http") { + return "", "", errors.New("HTTP token flags require --http with an address") + } + return "", "", nil + } + var key string + var err error + var deferKeyCheck bool + switch { + case fileSet: + path, _ := cmd.Flags().GetString("http-token-file") + if path == "" { + return "", "", errors.New("--http-token-file must not be empty") + } + key, err = providercredentials.ReadSecretFile(path) + case envSet: + name, _ := cmd.Flags().GetString("http-token-env") + if name == "" { + return "", "", errors.New("--http-token-env must not be empty") + } + key, err = providercredentials.ResolveSecret("", "", name) + default: + err = cfg.ResolveServerKey() + key = cfg.Server.AuthenticationKey() + // A local daemon may create the default key during startup. Enforce + // the inbound key requirement after OpenHTTPStore has resolved it. + deferKeyCheck = !isRemoteModeFor(invocationFromCommand(cmd)) + } + if err != nil { + return "", "", fmt.Errorf("MCP inbound credential: %w", err) + } + address, err := normalizeMCPHTTPAddr(mcpHTTPAddr, mcpHTTPAllowInsecure, key != "" || deferKeyCheck) + return address, key, err +} + // savedViewsMinAPISchemaVersion is the first daemon API schema that runs Saved // Views through POST /api/v1/saved-views/{id}/run. const savedViewsMinAPISchemaVersion = "2.21.0" +const personCardDAVMinAPISchemaVersion = "2.32.0" +const identityReviewMinAPISchemaVersion = "3.0.0" +const identityScoringMinAPISchemaVersion = "3.0.0" + +// personAgendaMinAPISchemaVersion adds live task-backed person agendas. +const personAgendaMinAPISchemaVersion = "2.30.0" -func daemonMCPServeOptions(ctx context.Context, st *daemonclient.Client) (mcpserver.ServeOptions, error) { +// archiveSQLMinAPISchemaVersion adds SQL confined to archive analytics files. +const archiveSQLMinAPISchemaVersion = "2.31.0" + +// calendarControlMinAPISchemaVersion adds delegated Calendar tools. +const calendarControlMinAPISchemaVersion = "3.1.0" + +// Schema 2.28.0 adds independent configured-lane facts to authenticated +// health. Older health responses cannot distinguish text from visual search. +const vectorLaneHealthMinAPISchemaVersion = "2.28.0" + +// Schema 2.4.0 added the visual attachment search route. +const visualSearchMinAPISchemaVersion = "2.4.0" + +func daemonMCPServeOptions(ctx context.Context, st *daemonclient.Client, state *invocation) mcpserver.ServeOptions { + if state == nil { + state = invocationFromContext(ctx) + } + var cfg *config.Config + log := slog.New(slog.DiscardHandler) + if state != nil { + cfg = state.cfg + if state.logger != nil { + log = state.logger + } + } engine := daemonclient.NewEngineAdapter(st) opts := mcpserver.ServeOptions{ Engine: engine, - AttachmentsDir: cfg.AttachmentsDir(), AttachmentReader: st, ManifestSaver: daemonMCPManifestSaver{client: st}, DocumentSearcher: st, PersonFileSearcher: daemonMCPPersonFileSearcher{client: st}, - DataDir: cfg.Data.DataDir, } - schemaVersion, capabilityErr := st.APISchemaVersion(ctx) + if cfg != nil { + opts.AttachmentsDir = cfg.AttachmentsDir() + opts.DataDir = cfg.Data.DataDir + } + health, capabilityErr := st.Health(ctx) + var schemaVersion string + if health != nil && health.APISchemaVersion != nil { + schemaVersion = *health.APISchemaVersion + } + if capabilityErr == nil && health != nil && health.Vector != nil && + daemonclient.APISchemaVersionAtLeast(schemaVersion, vectorLaneHealthMinAPISchemaVersion) { + if health.Vector.TextEnabled != nil && *health.Vector.TextEnabled { + opts.HybridSearcher = daemonMCPHybridSearcher{client: st} + opts.SimilarSearcher = daemonMCPSimilarSearcher{client: st} + } + if health.Vector.VisualEnabled != nil && *health.Vector.VisualEnabled && + daemonclient.APISchemaVersionAtLeast(schemaVersion, visualSearchMinAPISchemaVersion) { + opts.VisualSearcher = daemonMCPVisualSearcher{client: st} + } + } if capabilityErr != nil { - logger.Warn("people tools disabled because the daemon capability probe failed", "error", capabilityErr) + log.Warn("people tools disabled because the daemon capability probe failed", "error", capabilityErr) } else if daemonclient.APISchemaVersionAtLeast(schemaVersion, peopleMinAPISchemaVersion) { people := daemonclient.NewPeopleBrowser(engine) if daemonclient.APISchemaVersionAtLeast(schemaVersion, directoryPeopleMinAPISchemaVersion) { @@ -112,35 +230,39 @@ func daemonMCPServeOptions(ctx context.Context, st *daemonclient.Client) (mcpser // The daemon executes Saved Views itself, so the tools need a daemon that // serves the run endpoint; an older daemon simply omits them. if capabilityErr != nil { - logger.Warn("Saved View tools disabled because the daemon capability probe failed", "error", capabilityErr) + log.Warn("Saved View tools disabled because the daemon capability probe failed", "error", capabilityErr) } else if daemonclient.APISchemaVersionAtLeast(schemaVersion, savedViewsMinAPISchemaVersion) { opts.SavedViews = st } + if capabilityErr != nil { + log.Warn("meeting tools disabled because the daemon capability probe failed", "error", capabilityErr) + } else if daemonclient.APISchemaVersionAtLeast(schemaVersion, meetingsMinAPISchemaVersion) { + opts.Meetings = st + } + if capabilityErr == nil && daemonclient.APISchemaVersionAtLeast(schemaVersion, calendarControlMinAPISchemaVersion) { + opts.Calendar = st + } + if isAgentMode(state) { + return mcpserver.ServeOptions{Calendar: opts.Calendar, CalendarOnly: true} + } + if capabilityErr == nil && daemonclient.APISchemaVersionAtLeast(schemaVersion, personAgendaMinAPISchemaVersion) { + opts.PersonAgendaBackend = st + } + if capabilityErr == nil && daemonclient.APISchemaVersionAtLeast(schemaVersion, archiveSQLMinAPISchemaVersion) && + (health.AnalyticsEngine == nil || *health.AnalyticsEngine != api.AnalyticsModePostgres) { + opts.ArchiveSQLQuerier = engine + } + if capabilityErr == nil && daemonclient.APISchemaVersionAtLeast(schemaVersion, identityReviewMinAPISchemaVersion) { + opts.IdentityReview = st + } + if capabilityErr == nil && daemonclient.APISchemaVersionAtLeast(schemaVersion, personCardDAVMinAPISchemaVersion) { + opts.PersonCardDAV = st + } + if capabilityErr == nil && daemonclient.APISchemaVersionAtLeast(schemaVersion, identityScoringMinAPISchemaVersion) { + opts.IdentityScoring = st + } - vectorAvailable, err := st.VectorSearchAvailable(ctx) - if err != nil { - return mcpserver.ServeOptions{}, fmt.Errorf("check daemon vector search: %w", err) - } - if vectorAvailable { - opts.HybridSearcher = daemonMCPHybridSearcher{client: st} - opts.SimilarSearcher = daemonMCPSimilarSearcher{client: st} - } - // The daemon owns the multimodal lane; a remote-only MCP client's local - // config says nothing about it, so availability is probed, not assumed. - // The stats lane field distinguishes configured (including still - // initializing) from disabled, so a transient 503 during asynchronous - // vector init cannot permanently omit the tool — per-request errors - // report readiness instead. Older daemons lack the field; fall back to - // the visual status endpoint answering at all. - visualAvailable, laneReported, visualErr := st.VisualSearchAvailable(ctx) - if visualErr == nil && !laneReported { - _, statusErr := st.VisualStatus(ctx) - visualAvailable = statusErr == nil - } - if visualErr == nil && visualAvailable { - opts.VisualSearcher = daemonMCPVisualSearcher{client: st} - } - return opts, nil + return opts } type daemonMCPPersonFileSearcher struct{ client *daemonclient.Client } @@ -226,7 +348,14 @@ func (s daemonMCPHybridSearcher) SearchHybrid( return &mcpserver.HybridSearchResult{ Hits: hits, PoolSaturated: resp.PoolSaturated, + Accelerator: resp.Accelerator, HasMore: resp.HasMore, + TookMS: resp.TookMS, + Timings: mcpserver.HybridSearchTimings{ + QueryEmbeddingMS: resp.Timings.QueryEmbeddingMS, + RetrievalMS: resp.Timings.RetrievalMS, + HydrationMS: resp.Timings.HydrationMS, + }, Generation: mcpserver.HybridGeneration{ ID: resp.Generation.ID, Model: resp.Generation.Model, @@ -281,21 +410,34 @@ func init() { mcpCmd.Flags().StringVar(&mcpHTTPAddr, "http", "", "Serve over StreamableHTTP on this address (e.g. 127.0.0.1:8080) "+ "instead of stdio. Bare port forms (':8080', '8080') bind to "+ - "loopback only; non-loopback hosts require [server].api_key or "+ + "loopback only; non-loopback hosts require an inbound key or "+ "--http-allow-insecure.") + mcpCmd.Flags().String("http-token-file", "", "Read an independent inbound bearer key from an owner-only file (requires --http)") + mcpCmd.Flags().String("http-token-env", "", "Name the environment variable holding an inbound bearer key (requires --http; file takes priority)") mcpCmd.Flags().BoolVar(&mcpHTTPAllowInsecure, "http-allow-insecure", false, - "Allow --http to bind a non-loopback address without [server].api_key. "+ + "Allow --http to bind a non-loopback address without an inbound key. "+ "Any configured key still requires bearer authentication. Without a "+ "key, any reachable client can read your archive; only set this behind "+ "a trusted network boundary or authenticating reverse proxy.") mcpCmd.Flags().BoolVar(&mcpHTTPAllowWrites, "http-allow-writes", false, "Expose write-class MCP tools over HTTP. This permits attachment exports, "+ "deletion manifests, Saved View management, and profile writes separately enabled with "+ - "--allow-profile-writes; enable it only for trusted, authenticated clients.") + "--allow-profile-writes, identity decisions, identity scoring, person merges, CardDAV writes, and calendar writes enabled "+ + "with their separate opt-ins; enable it only for trusted, authenticated clients.") mcpCmd.Flags().BoolVar(&mcpAllowProfileWrites, "allow-profile-writes", false, "Expose person promotion and private Notes writes. Model tool calls "+ "can persist profile data, so enable this only for sessions where the user "+ "has explicitly authorized profile writes.") + mcpCmd.Flags().BoolVar(&mcpAllowIdentityDecisions, "allow-identity-decisions", false, + "Expose identity match accept/reject tools. Each call requires MCP client confirmation; the client must obtain user approval.") + mcpCmd.Flags().BoolVar(&mcpAllowIdentityScoring, "allow-identity-scoring", false, + "Expose manual identity scoring that sends evidence to the fixed Jev provider. Each call requires MCP client confirmation; the client must obtain user approval.") + mcpCmd.Flags().BoolVar(&mcpAllowPersonMerges, "allow-person-merges", false, + "Expose local person merge tools. Each call requires MCP client confirmation; the client must obtain user approval.") + mcpCmd.Flags().BoolVar(&mcpAllowCardDAVWrites, "allow-carddav-writes", false, + "Expose CardDAV publication and sync tools. Each call requires MCP client confirmation; the client must obtain user approval.") + mcpCmd.Flags().BoolVar(&mcpAllowCalendarWrites, "allow-calendar-writes", false, + "Expose calendar event mutation tools. Calendar event text is untrusted input; enable only when the user explicitly authorizes calendar writes.") _ = mcpCmd.Flags().MarkDeprecated("force-sql", "deprecated in 0.17.0; set [analytics].engine = \"sql\" in config.toml") _ = mcpCmd.Flags().MarkDeprecated("no-sqlite-scanner", "deprecated in 0.17.0; cache engine selection is daemon-managed; use [analytics].engine = \"sql\" for live SQL") _ = mcpCmd.Flags().MarkHidden("force-sql") @@ -343,7 +485,7 @@ func normalizeMCPHTTPAddr(addr string, allowInsecure, authenticated bool) (strin if !authenticated && !allowInsecure { return "", fmt.Errorf( "--http %q: refusing to bind a non-loopback address without "+ - "[server].api_key or --http-allow-insecure (configure an API key "+ + "an inbound key or --http-allow-insecure (configure an API key "+ "for bearer authentication, or only opt into unauthenticated "+ "access behind a trusted network boundary)", trimmed) } diff --git a/cmd/msgvault/cmd/mcp_startup_test.go b/cmd/msgvault/cmd/mcp_startup_test.go new file mode 100644 index 000000000..866b898bb --- /dev/null +++ b/cmd/msgvault/cmd/mcp_startup_test.go @@ -0,0 +1,196 @@ +package cmd + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/config" +) + +const ( + mcpStartupChildEnv = "MSGVAULT_MCP_STARTUP_CHILD" + mcpStartupHomeEnv = "MSGVAULT_MCP_STARTUP_HOME" +) + +// TestMCPInitializeWithoutStats exercises the real mcp command in a child +// process so stdio startup and the daemon request boundary stay in the test. +func TestMCPInitializeWithoutStats(t *testing.T) { + testMCPStartupCatalog(t, api.AnalyticsModeDuckDB) +} + +func TestMCPPostgresCatalogWithoutSQL(t *testing.T) { + testMCPStartupCatalog(t, api.AnalyticsModePostgres) +} + +func testMCPStartupCatalog(t *testing.T, analyticsEngine string) { + t.Helper() + require := require.New(t) + assert := assert.New(t) + + releaseStats := make(chan struct{}) + var releaseStatsOnce sync.Once + statsSeen := make(chan struct{}, 1) + daemon := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v1/health": + w.Header().Set("Content-Type", "application/json") + _, _ = fmt.Fprintf(w, `{"status":"ok","api_schema_version":%q,"analytics_engine":%q}`, api.APISchemaVersion, analyticsEngine) + case "/api/v1/stats": + statsSeen <- struct{}{} + <-releaseStats + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"total_messages":0,"total_threads":0,"total_accounts":0,"total_labels":0,"total_attachments":0,"database_size_bytes":0}`)) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(func() { + releaseStatsOnce.Do(func() { close(releaseStats) }) + daemon.Close() + }) + + home := t.TempDir() + configText := fmt.Sprintf("[remote]\nurl = %q\nallow_insecure = true\n", daemon.URL) + require.NoError(os.WriteFile(filepath.Join(home, "config.toml"), []byte(configText), 0o600)) + + ctx, cancel := context.WithTimeout(t.Context(), serveLifecycleTestTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestMCPStartupChild$") //nolint:gosec // the test binary and fixed test selector are local. + cmd.Env = append(os.Environ(), + mcpStartupChildEnv+"=1", + mcpStartupHomeEnv+"="+home, + "MSGVAULT_TEST_DB=", + ) + stdin, err := cmd.StdinPipe() + require.NoError(err) + stdout, err := cmd.StdoutPipe() + require.NoError(err) + var stderr bytes.Buffer + cmd.Stderr = &stderr + require.NoError(cmd.Start()) + var waitOnce sync.Once + var waitErr error + waitChild := func() error { + waitOnce.Do(func() { waitErr = cmd.Wait() }) + if waitErr != nil { + return fmt.Errorf("wait for MCP child: %w", waitErr) + } + return nil + } + t.Cleanup(func() { + _ = stdin.Close() + cancel() + _ = waitChild() + }) + + reader := bufio.NewReader(stdout) + waitForResponse := make(chan []byte, 1) + go func() { + line, readErr := reader.ReadBytes('\n') + if readErr != nil { + waitForResponse <- nil + return + } + waitForResponse <- line + }() + + initialize := `{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"target-896","version":"test"}}}` + "\n" + _, err = stdin.Write([]byte(initialize)) + require.NoError(err) + + var response []byte + select { + case <-statsSeen: + require.FailNow("MCP initialization must not request /api/v1/stats") + case response = <-waitForResponse: + case <-ctx.Done(): + require.FailNow("MCP initialize did not return or request stats within the watchdog") + } + require.NotEmpty(response, "MCP initialize returned no response") + var envelope map[string]any + require.NoError(json.Unmarshal(response, &envelope)) + assert.InDelta(float64(1), envelope["id"], 0) + require.Nil(envelope["error"]) + + _, err = stdin.Write([]byte(`{"jsonrpc":"2.0","method":"notifications/initialized"}` + "\n" + + `{"jsonrpc":"2.0","id":2,"method":"tools/list"}` + "\n")) + require.NoError(err) + response, err = reader.ReadBytes('\n') + require.NoError(err) + var catalog struct { + Result struct { + Tools []struct { + Name string `json:"name"` + Description string `json:"description"` + } `json:"tools"` + } `json:"result"` + } + require.NoError(json.Unmarshal(response, &catalog)) + require.NotEmpty(catalog.Result.Tools) + var names []string + for _, tool := range catalog.Result.Tools { + names = append(names, tool.Name) + if tool.Name == "semantic_search_messages" { + assert.Contains(tool.Description, "unavailable: vector search is not configured") + } + } + assert.Contains(names, "semantic_search_messages") + assert.NotContains(names, "find_similar_messages") + assert.NotContains(names, "search_visual_attachments") + if analyticsEngine == api.AnalyticsModePostgres { + assert.NotContains(names, "query_sql") + } else { + assert.Contains(names, "query_sql") + } + assert.Empty(statsSeen, "tool discovery must not request /api/v1/stats") + + _ = stdin.Close() + waitDone := make(chan error, 1) + go func() { waitDone <- waitChild() }() + select { + case waitErr = <-waitDone: + case <-ctx.Done(): + require.FailNow("MCP child did not exit after stdin closed") + } + require.NoError(waitErr, stderr.String()) +} + +func TestMCPStartupChild(t *testing.T) { + if os.Getenv(mcpStartupChildEnv) != "1" { + return + } + runMCPStartupChild(t) +} + +func runMCPStartupChild(t *testing.T) { + t.Helper() + home := os.Getenv(mcpStartupHomeEnv) + require.NotEmpty(t, home) + loaded, err := config.Load("", home) + require.NoError(t, err) + cfg := loaded + remoteAPISchemaCheckEnabled = true + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + ctx = testInvocationContext(ctx, cfg, invocationOptions{}) + mcpCmd.SetContext(ctx) + err = mcpCmd.RunE(mcpCmd, nil) + if err != nil && !errors.Is(err, context.Canceled) { + assert.NoError(t, err) + } +} diff --git a/cmd/msgvault/cmd/mcp_status.go b/cmd/msgvault/cmd/mcp_status.go index 615a702d5..cae6bd9b0 100644 --- a/cmd/msgvault/cmd/mcp_status.go +++ b/cmd/msgvault/cmd/mcp_status.go @@ -3,6 +3,7 @@ package cmd import ( "encoding/json/jsontext" "encoding/json/v2" + "errors" "fmt" "path/filepath" @@ -14,15 +15,30 @@ import ( func newMCPStatusCommand() *cobra.Command { var jsonOutput bool command := &cobra.Command{ - Use: "status", - Short: "List running HTTP MCP listeners", - Args: cobra.NoArgs, - PersistentPreRunE: func(*cobra.Command, []string) error { return nil }, + Use: "status", + Short: "List running HTTP MCP listeners", + Args: cobra.NoArgs, + PersistentPreRunE: func(command *cobra.Command, _ []string) error { + inv := prepareInvocation(command) + if inv == nil { + return errors.New("missing invocation state") + } + // MCP status is intentionally a config reader. It must not run the + // root owner lifecycle, create the home directory, or initialize + // logging just to inspect listener records. + command.SilenceUsage = true + return nil + }, RunE: func(command *cobra.Command, _ []string) error { - cfg, err := config.Load(cfgFile, homeDir) + inv := invocationFromCommand(command) + if inv == nil { + return errors.New("missing invocation state") + } + cfg, err := config.Load(inv.options.cfgFile, inv.options.homeDir) if err != nil { return err } + inv.cfg = cfg directory := filepath.Join(cfg.HomeDir, "mcp") endpoints, err := mcpdiscovery.List(directory) if err != nil { diff --git a/cmd/msgvault/cmd/mcp_test.go b/cmd/msgvault/cmd/mcp_test.go index bdda7cc46..d2cc1252c 100644 --- a/cmd/msgvault/cmd/mcp_test.go +++ b/cmd/msgvault/cmd/mcp_test.go @@ -8,10 +8,12 @@ import ( "log/slog" "net/http" "net/http/httptest" + "os" "path/filepath" "sync/atomic" "testing" + "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.kenn.io/msgvault/internal/api" @@ -23,25 +25,36 @@ import ( func TestMCPWriteHelpDisclosesMutationClassesAndProfileOptIn(t *testing.T) { assert := assert.New(t) - require.NotNil(t, mcpCmd.Flags().Lookup("allow-profile-writes")) + require := require.New(t) + require.NotNil(mcpCmd.Flags().Lookup("allow-profile-writes")) + for _, name := range []string{"allow-identity-decisions", "allow-identity-scoring", "allow-person-merges", "allow-carddav-writes"} { + require.NotNil(mcpCmd.Flags().Lookup(name)) + } + require.NotNil(mcpCmd.Flags().Lookup("allow-calendar-writes")) var output bytes.Buffer previousOutput := mcpCmd.OutOrStdout() mcpCmd.SetOut(&output) t.Cleanup(func() { mcpCmd.SetOut(previousOutput) }) - require.NoError(t, mcpCmd.Help()) + require.NoError(mcpCmd.Help()) help := output.String() assert.Contains(help, "attachment exports") assert.Contains(help, "deletion manifests") assert.Contains(help, "person promotion") assert.Contains(help, "private Notes writes") + assert.Contains(help, "allow-identity-decisions") + assert.Contains(help, "allow-identity-scoring") + assert.Contains(help, "allow-person-merges") + assert.Contains(help, "allow-carddav-writes") + assert.Contains(help, "calendar event mutation tools") } func TestMCPCommandUsesDaemonInsteadOfOpeningLocalDatabase(t *testing.T) { require := require.New(t) assert := assert.New(t) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ + HomeDir: t.TempDir(), Data: config.DataConfig{ DataDir: filepath.Join(t.TempDir(), "missing-parent", "data"), }, @@ -60,10 +73,11 @@ func TestMCPCommandUsesDaemonInsteadOfOpeningLocalDatabase(t *testing.T) { mcpHTTPAllowInsecure = savedAllowInsecure }) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(testCtx) cancel() cmd := mcpCmd + cmd.SetContext(testCtx) cmd.SetContext(ctx) err := cmd.RunE(cmd, nil) @@ -87,22 +101,12 @@ func TestMCPCommandForwardsHTTPPolicy(t *testing.T) { http.Error(w, `{"error":"visual_search_not_ready"}`, http.StatusServiceUnavailable) return } - assert.Equal("/api/v1/stats", r.URL.Path) - assert.Equal("daemon-key", r.Header.Get("X-Api-Key")) - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{ - "total_messages": 0, - "total_threads": 0, - "total_accounts": 0, - "total_labels": 0, - "total_attachments": 0, - "database_size_bytes": 0 - }`)) + http.NotFound(w, r) })) t.Cleanup(daemon.Close) home := t.TempDir() - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ HomeDir: home, Data: config.DataConfig{DataDir: t.TempDir()}, Server: config.ServerConfig{APIKey: "mcp-http-key"}, @@ -116,18 +120,36 @@ func TestMCPCommandForwardsHTTPPolicy(t *testing.T) { savedHTTPAddr := mcpHTTPAddr savedAllowInsecure := mcpHTTPAllowInsecure savedAllowProfileWrites := mcpAllowProfileWrites + savedAllowIdentityDecisions := mcpAllowIdentityDecisions + savedAllowIdentityScoring := mcpAllowIdentityScoring + savedAllowPersonMerges := mcpAllowPersonMerges + savedAllowCardDAVWrites := mcpAllowCardDAVWrites + savedAllowCalendarWrites := mcpAllowCalendarWrites savedServeHTTP := serveMCPHTTPWithOptions allowWritesFlag := mcpCmd.Flags().Lookup("http-allow-writes") require.NotNil(allowWritesFlag, "mcp command must define --http-allow-writes") require.NoError(allowWritesFlag.Value.Set("true")) + allowCalendarWritesFlag := mcpCmd.Flags().Lookup("allow-calendar-writes") + require.NotNil(allowCalendarWritesFlag, "mcp command must define --allow-calendar-writes") + require.NoError(allowCalendarWritesFlag.Value.Set("true")) mcpHTTPAddr = "0.0.0.0:8081" mcpHTTPAllowInsecure = true mcpAllowProfileWrites = true + mcpAllowIdentityDecisions = true + mcpAllowIdentityScoring = true + mcpAllowPersonMerges = true + mcpAllowCardDAVWrites = true t.Cleanup(func() { assert.NoError(allowWritesFlag.Value.Set("false")) + assert.NoError(allowCalendarWritesFlag.Value.Set("false")) mcpHTTPAddr = savedHTTPAddr mcpHTTPAllowInsecure = savedAllowInsecure mcpAllowProfileWrites = savedAllowProfileWrites + mcpAllowIdentityDecisions = savedAllowIdentityDecisions + mcpAllowIdentityScoring = savedAllowIdentityScoring + mcpAllowPersonMerges = savedAllowPersonMerges + mcpAllowCardDAVWrites = savedAllowCardDAVWrites + mcpAllowCalendarWrites = savedAllowCalendarWrites serveMCPHTTPWithOptions = savedServeHTTP }) @@ -140,11 +162,16 @@ func TestMCPCommandForwardsHTTPPolicy(t *testing.T) { return wantErr } - mcpCmd.SetContext(context.Background()) + mcpCmd.SetContext(testCtx) err := mcpCmd.RunE(mcpCmd, nil) require.ErrorIs(err, wantErr) assert.True(gotServeOpts.AllowProfileWrites) + assert.True(gotServeOpts.AllowIdentityDecisions) + assert.True(gotServeOpts.AllowIdentityScoring) + assert.True(gotServeOpts.AllowPersonMerges) + assert.True(gotServeOpts.AllowCardDAVWrites) + assert.True(gotServeOpts.AllowCalendarWrites) assert.Equal(mcpserver.HTTPOptions{ Addr: "0.0.0.0:8081", DiscoveryDirectory: filepath.Join(home, "mcp"), @@ -154,42 +181,194 @@ func TestMCPCommandForwardsHTTPPolicy(t *testing.T) { }, gotHTTPOpts) } -func TestDaemonMCPServeOptionsDisablesVectorToolsWhenDaemonVectorUnavailable(t *testing.T) { +func TestMCPDelegatedModeRejectsDaemonWithoutCalendarAPI(t *testing.T) { + assertions := assert.New(t) + requirements := require.New(t) + previousHTTPAddr := mcpHTTPAddr + previousServe := serveMCPStdioWithOptions + previousSchemaCheck := remoteAPISchemaCheckEnabled + mcpHTTPAddr = "" + remoteAPISchemaCheckEnabled = true + var serveCalled bool + serveMCPStdioWithOptions = func(_ context.Context, options mcpserver.ServeOptions) error { + serveCalled = true + assertions.True(options.CalendarOnly) + assertions.Nil(options.Calendar) + return errors.New("stdio serving started without calendar tools") + } + t.Cleanup(func() { + mcpHTTPAddr = previousHTTPAddr + serveMCPStdioWithOptions = previousServe + remoteAPISchemaCheckEnabled = previousSchemaCheck + }) + + var healthRequests atomic.Int32 + daemon := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v1/health": + healthRequests.Add(1) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"status": "ok", "api_schema_version": "3.0.0"}) + case "/api/session": + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]string{"auth_mode": "delegated"}) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(daemon.Close) + tokenFile := filepath.Join(t.TempDir(), "agent-token") + requirements.NoError(os.WriteFile(tokenFile, []byte("mva1_test-token"), 0o600)) + ctx := testInvocationContext(t.Context(), config.NewDefaultConfig(), invocationOptions{ + agentURL: daemon.URL, agentTokenFile: tokenFile, agentAllowInsecure: true, + agentURLChanged: true, agentTokenChanged: true, + }) + cmd := &cobra.Command{} + cmd.SetContext(ctx) + + err := mcpCmd.RunE(cmd, nil) + + requirements.ErrorContains(err, "calendar delegation requires a daemon with API schema 3.1.0 or later") + assertions.False(serveCalled) + assertions.Equal(int32(2), healthRequests.Load()) +} + +func TestDaemonMCPHybridSearcherPreservesPhaseTimings(t *testing.T) { assert := assert.New(t) + client := newMCPDaemonClient(t, func(w http.ResponseWriter, r *http.Request) { + assert.Equal("/api/v1/search", r.URL.Path) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "query":"semantic terms", + "mode":"hybrid", + "returned":0, + "pool_saturated":false, + "accelerator":"vec1_ivf_opq", + "has_more":false, + "generation":{"id":7,"model":"fake","dimension":4,"fingerprint":"fake:4","state":"active"}, + "took_ms":12, + "timings":{"query_embedding_ms":2,"retrieval_ms":7,"hydration_ms":3}, + "results":[] + }`)) + }) - withStoreResolverConfig(t, &config.Config{ - Data: config.DataConfig{DataDir: t.TempDir()}, + result, err := (daemonMCPHybridSearcher{client: client}).SearchHybrid(t.Context(), mcpserver.HybridSearchRequest{ + Query: "semantic terms", Mode: "hybrid", Limit: 10, }) - client := newMCPStatsDaemonClient(t, `{ - "total_messages": 0, - "total_threads": 0, - "total_accounts": 0, - "total_labels": 0, - "total_attachments": 0, - "database_size_bytes": 0 - }`) - - opts, err := daemonMCPServeOptions(context.Background(), client) require.NoError(t, err) - assert.NotNil(opts.Engine, "engine") - assert.NotNil(opts.AttachmentReader, "attachment reader") - assert.NotNil(opts.ManifestSaver, "manifest saver") - assert.NotNil(opts.DocumentSearcher, "document searcher") - assert.NotNil(opts.PersonFileSearcher, "person file searcher") - assert.Nil(opts.HybridSearcher, "hybrid searcher") - assert.Nil(opts.SimilarSearcher, "similar searcher") + assert.Equal(int64(12), result.TookMS) + assert.Equal("vec1_ivf_opq", result.Accelerator) + assert.Equal(mcpserver.HybridSearchTimings{ + QueryEmbeddingMS: 2, + RetrievalMS: 7, + HydrationMS: 3, + }, result.Timings) +} + +func TestDaemonMCPServeOptionsUsesHealthForVectorTools(t *testing.T) { + testCtx := withStoreResolverConfig(t, &config.Config{ + Data: config.DataConfig{DataDir: t.TempDir()}, + }) + _ = testCtx + tests := []struct { + name string + health string + wantText bool + wantVisual bool + }{ + {name: "both disabled", health: `{"status":"ok","api_schema_version":"2.28.0","vector":{"status":"disabled","text_enabled":false,"visual_enabled":false}}`}, + {name: "text only", health: `{"status":"ok","api_schema_version":"2.28.0","vector":{"status":"ready","text_enabled":true,"visual_enabled":false}}`, wantText: true}, + {name: "visual only", health: `{"status":"ok","api_schema_version":"2.28.0","vector":{"status":"ready","text_enabled":false,"visual_enabled":true}}`, wantVisual: true}, + {name: "both enabled", health: `{"status":"ok","api_schema_version":"2.28.0","vector":{"status":"ready","text_enabled":true,"visual_enabled":true}}`, wantText: true, wantVisual: true}, + {name: "initializing lanes stay registered", health: `{"status":"ok","api_schema_version":"2.28.0","vector":{"status":"initializing","text_enabled":true,"visual_enabled":true}}`, wantText: true, wantVisual: true}, + {name: "failed lanes stay registered", health: `{"status":"ok","api_schema_version":"2.28.0","vector":{"status":"error","text_enabled":true,"visual_enabled":true}}`, wantText: true, wantVisual: true}, + {name: "stale lanes stay registered", health: `{"status":"ok","api_schema_version":"2.28.0","vector":{"status":"stale","text_enabled":true,"visual_enabled":true}}`, wantText: true, wantVisual: true}, + {name: "legacy health without lane fields", health: `{"status":"ok","api_schema_version":"2.26.0","vector":{"status":"ready"}}`}, + {name: "visual route predecessor", health: `{"status":"ok","api_schema_version":"2.3.0","vector":{"status":"ready","text_enabled":true,"visual_enabled":true}}`}, + {name: "lane facts predecessor", health: `{"status":"ok","api_schema_version":"2.27.0","vector":{"status":"ready","text_enabled":true,"visual_enabled":true}}`}, + {name: "missing schema", health: `{"status":"ok","vector":{"status":"ready","text_enabled":true,"visual_enabled":true}}`}, + {name: "malformed schema", health: `{"status":"ok","api_schema_version":"unknown","vector":{"status":"ready","text_enabled":true,"visual_enabled":true}}`}, + {name: "health unavailable"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + var healthRequests atomic.Int32 + client := newMCPDaemonClient(t, func(w http.ResponseWriter, r *http.Request) { + assert.Equal("/api/v1/health", r.URL.Path, "startup must not request archive statistics") + healthRequests.Add(1) + if tt.health == "" { + http.Error(w, `{"error":"temporarily_unavailable"}`, http.StatusServiceUnavailable) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(tt.health)) + }) + + opts := daemonMCPServeOptions(t.Context(), client, invocationFromContext(t.Context())) + assert.Equal(tt.wantText, opts.HybridSearcher != nil, "semantic search") + assert.Equal(tt.wantText, opts.SimilarSearcher != nil, "similar messages") + assert.Equal(tt.wantVisual, opts.VisualSearcher != nil, "visual search") + assert.Equal(int32(1), healthRequests.Load(), "reuse the schema probe") + }) + } +} + +func TestDaemonMCPVectorReadinessIsCheckedAtRequestTime(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + testCtx := withStoreResolverConfig(t, &config.Config{ + Data: config.DataConfig{DataDir: t.TempDir()}, + }) + + requests := make(chan string, 8) + client := newMCPDaemonClient(t, func(w http.ResponseWriter, r *http.Request) { + requests <- r.URL.Path + if r.URL.Path == "/api/v1/health" { + _ = json.NewEncoder(w).Encode(map[string]any{ + "status": "ok", "api_schema_version": api.APISchemaVersion, + "vector": map[string]any{ + "status": "initializing", + "text_enabled": true, + "visual_enabled": false, + }, + }) + return + } + http.Error(w, `{"error":"vector_initializing","message":"Vector search is initializing"}`, http.StatusServiceUnavailable) + }) + + opts := daemonMCPServeOptions(testCtx, client, invocationFromContext(testCtx)) + _, err := opts.HybridSearcher.SearchHybrid(testCtx, mcpserver.HybridSearchRequest{ + Query: "term", + Mode: "hybrid", + }) + var coded interface{ APIErrorCode() string } + require.ErrorAs(err, &coded) + assert.Equal("vector_initializing", coded.APIErrorCode()) + path := <-requests + assert.Equal("/api/v1/health", path, "startup should only probe health") + path = <-requests + assert.Equal("/api/v1/search", path, "vector readiness belongs to the request") } func TestDaemonMCPServeOptionsGatesPeopleToolsByAPISchema(t *testing.T) { - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Data: config.DataConfig{DataDir: t.TempDir()}, }) + _ = testCtx tests := []struct { name string schemaVersion string wantPeople bool wantDirectory bool wantSavedViews bool + wantMeetings bool + wantAgenda bool + wantArchiveSQL bool + wantReview bool + wantScoring bool + wantCardDAV bool }{ {name: "people schema", schemaVersion: "2.10.0", wantPeople: true}, {name: "directory predecessor", schemaVersion: "2.12.9", wantPeople: true}, @@ -197,6 +376,16 @@ func TestDaemonMCPServeOptionsGatesPeopleToolsByAPISchema(t *testing.T) { {name: "newer schema", schemaVersion: "2.14.0", wantPeople: true, wantDirectory: true}, {name: "schema before the Saved View run endpoint", schemaVersion: "2.20.0", wantPeople: true, wantDirectory: true}, {name: "saved view run schema", schemaVersion: "2.21.0", wantPeople: true, wantDirectory: true, wantSavedViews: true}, + {name: "schema before the meeting endpoints", schemaVersion: "2.24.0", wantPeople: true, wantDirectory: true, wantSavedViews: true}, + {name: "meeting predecessor 2.25.x", schemaVersion: "2.25.0", wantPeople: true, wantDirectory: true, wantSavedViews: true}, + {name: "meeting predecessor 2.26.x", schemaVersion: "2.26.0", wantPeople: true, wantDirectory: true, wantSavedViews: true}, + {name: "meeting schema", schemaVersion: "2.27.0", wantPeople: true, wantDirectory: true, wantSavedViews: true, wantMeetings: true}, + {name: "person agenda predecessor", schemaVersion: "2.29.0", wantPeople: true, wantDirectory: true, wantSavedViews: true, wantMeetings: true}, + {name: "person agenda schema", schemaVersion: "2.30.0", wantPeople: true, wantDirectory: true, wantSavedViews: true, wantMeetings: true, wantAgenda: true}, + {name: "archive SQL schema", schemaVersion: "2.31.0", wantPeople: true, wantDirectory: true, wantSavedViews: true, wantMeetings: true, wantAgenda: true, wantArchiveSQL: true}, + {name: "person CardDAV schema", schemaVersion: "2.32.0", wantPeople: true, wantDirectory: true, wantSavedViews: true, wantMeetings: true, wantAgenda: true, wantArchiveSQL: true, wantCardDAV: true}, + {name: "identity review and scoring predecessor", schemaVersion: "2.35.0", wantPeople: true, wantDirectory: true, wantSavedViews: true, wantMeetings: true, wantAgenda: true, wantArchiveSQL: true, wantCardDAV: true}, + {name: "identity review and scoring schema", schemaVersion: "3.0.0", wantPeople: true, wantDirectory: true, wantSavedViews: true, wantMeetings: true, wantAgenda: true, wantArchiveSQL: true, wantCardDAV: true, wantReview: true, wantScoring: true}, {name: "older same-major schema", schemaVersion: "2.9.9"}, {name: "malformed schema", schemaVersion: "not-a-version"}, {name: "missing schema"}, @@ -212,17 +401,12 @@ func TestDaemonMCPServeOptionsGatesPeopleToolsByAPISchema(t *testing.T) { body["api_schema_version"] = tt.schemaVersion } _ = json.NewEncoder(w).Encode(body) - case "/api/v1/stats": - _, _ = w.Write([]byte(`{"total_messages":0}`)) - case "/api/v1/multimodal/status": - http.Error(w, `{"error":"visual_search_not_ready"}`, http.StatusServiceUnavailable) default: - assert.Failf("unexpected request", "%s %s", r.Method, r.URL.Path) + http.NotFound(w, r) } }) - opts, err := daemonMCPServeOptions(t.Context(), client) - require.NoError(t, err) + opts := daemonMCPServeOptions(t.Context(), client, invocationFromContext(t.Context())) if tt.wantPeople { assert.NotNil(opts.PeopleBackend) } else { @@ -234,43 +418,52 @@ func TestDaemonMCPServeOptionsGatesPeopleToolsByAPISchema(t *testing.T) { assert.Nil(opts.SavedViews, "an older daemon cannot run Saved Views") } assert.Equal(tt.wantDirectory, opts.DirectoryBackend != nil) + assert.Equal(tt.wantMeetings, opts.Meetings != nil) + assert.Equal(tt.wantAgenda, opts.PersonAgendaBackend != nil) + assert.Equal(tt.wantArchiveSQL, opts.ArchiveSQLQuerier != nil) + assert.Equal(tt.wantReview, opts.IdentityReview != nil) + assert.Equal(tt.wantCardDAV, opts.PersonCardDAV != nil) + assert.Equal(tt.wantScoring, opts.IdentityScoring != nil) }) } } func TestDaemonMCPServeOptionsWarnsWhenPeopleCapabilityProbeFails(t *testing.T) { + logger := testLoggerValue() + assert := assert.New(t) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Data: config.DataConfig{DataDir: t.TempDir()}, }) var logs bytes.Buffer previousLogger := logger logger = slog.New(slog.NewTextHandler(&logs, nil)) + inv := invocationFromContext(testCtx) + previousInvocationLogger := inv.logger + inv.logger = logger t.Cleanup(func() { logger = previousLogger }) + t.Cleanup(func() { inv.logger = previousInvocationLogger }) client := newMCPDaemonClient(t, func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/api/v1/health": + if r.URL.Path == "/api/v1/health" { http.Error(w, `{"error":"temporarily_unavailable"}`, http.StatusServiceUnavailable) - case "/api/v1/stats": - _, _ = w.Write([]byte(`{"total_messages":0}`)) - case "/api/v1/multimodal/status": - http.Error(w, `{"error":"visual_search_not_ready"}`, http.StatusServiceUnavailable) - default: - assert.Failf("unexpected request", "%s %s", r.Method, r.URL.Path) + return } + http.NotFound(w, r) }) - opts, err := daemonMCPServeOptions(t.Context(), client) - require.NoError(t, err) + opts := daemonMCPServeOptions(testCtx, client, invocationFromContext(testCtx)) assert.Nil(opts.PeopleBackend) assert.Nil(opts.DirectoryBackend) + assert.Nil(opts.ArchiveSQLQuerier) assert.Contains(logs.String(), "people tools disabled") } func TestDaemonMCPServeOptionsUsesOneCapabilityProbe(t *testing.T) { + logger := testLoggerValue() + assert := assert.New(t) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Data: config.DataConfig{DataDir: t.TempDir()}, }) var logs bytes.Buffer @@ -283,25 +476,28 @@ func TestDaemonMCPServeOptionsUsesOneCapabilityProbe(t *testing.T) { case "/api/v1/health": if healthRequests.Add(1) == 1 { _ = json.NewEncoder(w).Encode(map[string]any{ - "status": "ok", "api_schema_version": "2.21.0", + "status": "ok", "api_schema_version": "2.28.0", + "vector": map[string]any{ + "status": "ready", + "text_enabled": true, + "visual_enabled": true, + }, }) return } http.Error(w, `{"error":"temporarily_unavailable"}`, http.StatusServiceUnavailable) - case "/api/v1/stats": - _, _ = w.Write([]byte(`{"total_messages":0}`)) - case "/api/v1/multimodal/status": - http.Error(w, `{"error":"visual_search_not_ready"}`, http.StatusServiceUnavailable) default: - assert.Failf("unexpected request", "%s %s", r.Method, r.URL.Path) + http.NotFound(w, r) } }) - opts, err := daemonMCPServeOptions(t.Context(), client) - require.NoError(t, err) + opts := daemonMCPServeOptions(testCtx, client, invocationFromContext(testCtx)) assert.NotNil(opts.PeopleBackend) assert.NotNil(opts.DirectoryBackend) assert.NotNil(opts.SavedViews) + assert.NotNil(opts.HybridSearcher) + assert.NotNil(opts.SimilarSearcher) + assert.NotNil(opts.VisualSearcher) assert.Equal(int32(1), healthRequests.Load()) assert.Empty(logs.String()) } @@ -309,7 +505,7 @@ func TestDaemonMCPServeOptionsUsesOneCapabilityProbe(t *testing.T) { func TestDaemonMCPServeOptionsSavesDeletionManifestsThroughDaemon(t *testing.T) { require := require.New(t) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Data: config.DataConfig{DataDir: t.TempDir()}, }) @@ -320,94 +516,25 @@ func TestDaemonMCPServeOptionsSavesDeletionManifestsThroughDaemon(t *testing.T) _ = json.NewEncoder(w).Encode(map[string]any{ "status": "ok", "api_schema_version": api.APISchemaVersion, }) - case "/api/v1/multimodal/status": - http.Error(w, `{"error":"visual_search_not_ready"}`, http.StatusServiceUnavailable) - return - case "/api/v1/stats": - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{ - "total_messages": 0, - "total_threads": 0, - "total_accounts": 0, - "total_labels": 0, - "total_attachments": 0, - "database_size_bytes": 0 - }`)) case "/api/v1/cli/deletion-manifests": manifestRequests.Add(1) assert.Equal(t, http.MethodPost, r.Method, "method") w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"id":"batch-1","message_count":1}`)) default: - t.Fatalf("unexpected path: %s", r.URL.Path) + http.NotFound(w, r) } }) - opts, err := daemonMCPServeOptions(context.Background(), client) - require.NoError(err) + opts := daemonMCPServeOptions(testCtx, client, invocationFromContext(testCtx)) require.NotNil(opts.ManifestSaver, "manifest saver") manifest := deletion.NewManifest("mcp test", []string{"gmail-001"}) - err = opts.ManifestSaver.SaveManifest(context.Background(), manifest) + err := opts.ManifestSaver.SaveManifest(testCtx, manifest) require.NoError(err) assert.Equal(t, int32(1), manifestRequests.Load(), "manifest requests") } -func TestDaemonMCPServeOptionsEnablesVectorToolsWhenDaemonVectorAvailable(t *testing.T) { - withStoreResolverConfig(t, &config.Config{ - Data: config.DataConfig{DataDir: t.TempDir()}, - }) - client := newMCPStatsDaemonClient(t, `{ - "total_messages": 0, - "total_threads": 0, - "total_accounts": 0, - "total_labels": 0, - "total_attachments": 0, - "database_size_bytes": 0, - "vector_search": { - "enabled": true, - "active_generation": { - "id": 1, - "model": "text-embedding-3-small", - "dimension": 1536, - "fingerprint": "text-embedding-3-small:1536", - "state": "active", - "message_count": 10 - }, - "missing_embeddings_total": 0 - } - }`) - - opts, err := daemonMCPServeOptions(context.Background(), client) - require.NoError(t, err) - - assert.NotNil(t, opts.HybridSearcher, "hybrid searcher") - assert.NotNil(t, opts.SimilarSearcher, "similar searcher") -} - -func newMCPStatsDaemonClient(t *testing.T, statsJSON string) *daemonclient.Client { - t.Helper() - - return newMCPDaemonClient(t, func(w http.ResponseWriter, r *http.Request) { - // Registration also probes the daemon's multimodal status; these - // fixtures model a daemon without the visual lane. - if r.URL.Path == "/api/v1/health" { - _ = json.NewEncoder(w).Encode(map[string]any{ - "status": "ok", "api_schema_version": api.APISchemaVersion, - }) - return - } - if r.URL.Path == "/api/v1/multimodal/status" { - http.Error(w, `{"error":"visual_search_not_ready"}`, http.StatusServiceUnavailable) - return - } - assert.Equal(t, "/api/v1/stats", r.URL.Path, "path") - assert.Equal(t, "key", r.Header.Get("X-Api-Key"), "api key") - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(statsJSON)) - }) -} - func newMCPDaemonClient(t *testing.T, handler http.HandlerFunc) *daemonclient.Client { t.Helper() diff --git a/cmd/msgvault/cmd/mcp_token_test.go b/cmd/msgvault/cmd/mcp_token_test.go new file mode 100644 index 000000000..4feb5e5dc --- /dev/null +++ b/cmd/msgvault/cmd/mcp_token_test.go @@ -0,0 +1,355 @@ +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/kit/daemon" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/fileutil" + "go.kenn.io/msgvault/internal/mcpdiscovery" + "go.kenn.io/msgvault/internal/providercredentials" +) + +func setMCPTokenTestFlags(t *testing.T, values map[string]string) { + t.Helper() + oldAddr, oldInsecure := mcpHTTPAddr, mcpHTTPAllowInsecure + mcpHTTPAllowInsecure = false + oldContext := mcpCmd.Context() + t.Cleanup(func() { mcpHTTPAddr, mcpHTTPAllowInsecure = oldAddr, oldInsecure; mcpCmd.SetContext(oldContext) }) + for _, name := range []string{"http-token-file", "http-token-env"} { + flag := mcpCmd.Flags().Lookup(name) + require.NotNil(t, flag, "MCP must expose independent inbound credential sources") + oldValue, oldChanged := flag.Value.String(), flag.Changed + t.Cleanup(func() { assert.NoError(t, flag.Value.Set(oldValue)); flag.Changed = oldChanged }) + require.NoError(t, flag.Value.Set("")) + flag.Changed = false + if value, ok := values[name]; ok { + require.NoError(t, flag.Value.Set(value)) + flag.Changed = true + } + } +} + +func TestMCPTokenFailuresBeforeBackendConnection(t *testing.T) { + for _, tc := range []struct { + name, address string + flags map[string]string + }{ + {"missing file", "127.0.0.1:0", map[string]string{"http-token-file": "missing-secret", "http-token-env": "MSGVAULT_TEST_MCP_LOWER_KEY"}}, + {"empty file flag", "127.0.0.1:0", map[string]string{"http-token-file": ""}}, + {"empty env flag", "127.0.0.1:0", map[string]string{"http-token-env": ""}}, + {"missing named environment", "127.0.0.1:0", map[string]string{"http-token-env": "MSGVAULT_TEST_MCP_MISSING_KEY"}}, + {"token without HTTP", "", map[string]string{"http-token-env": "MSGVAULT_TEST_MCP_LOWER_KEY"}}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("MSGVAULT_TEST_MCP_LOWER_KEY", "lower-key") + t.Setenv("MSGVAULT_TEST_MCP_MISSING_KEY", "") + var calls atomic.Int32 + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { calls.Add(1); w.WriteHeader(http.StatusOK) })) + defer backend.Close() + cfg := credentialsTestConfig(t) + cfg.Remote = config.RemoteConfig{URL: backend.URL, AllowInsecure: true} + setMCPTokenTestFlags(t, tc.flags) + mcpHTTPAddr = tc.address + mcpCmd.SetContext(withStoreResolverConfig(t, cfg)) + require.Error(t, mcpCmd.RunE(mcpCmd, nil)) + assert.Zero(t, calls.Load(), "invalid inbound credentials must fail before opening the backend") + }) + } +} + +func TestMCPLoopbackInterfaceReusesKeylessDaemon(t *testing.T) { //nolint:paralleltest // process environment and MCP flags + assert := assert.New(t) + require := require.New(t) + clearServerKeyEnvironment(t) + interfaces, err := net.Interfaces() + require.NoError(err) + var loopback string + for _, iface := range interfaces { + if iface.Flags&net.FlagLoopback != 0 && iface.Flags&net.FlagUp != 0 { + loopback = iface.Name + break + } + } + if loopback == "" { + t.Skip("no active loopback interface") + } + home := t.TempDir() + path := filepath.Join(home, "config.toml") + require.NoError(os.WriteFile(path, []byte("[server]\nbind_addr = 'iface:"+loopback+"'\ndaemon_auto_start = false\ndaemon_auto_restart = 'never'\n"), 0o600)) + cfg, err := config.Load(path, home) + require.NoError(err) + mux := http.NewServeMux() + mux.Handle("/api/ping", daemon.NewPingHandler(daemon.PingHandlerOptions{Service: daemonService, Version: Version})) + mux.HandleFunc("/api/v1/health", func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"status":"ok"}`)) + }) + backend := httptest.NewServer(mux) + t.Cleanup(backend.Close) + rt := daemonRuntimeForHTTPServer(t, backend, daemonAPIKeyFingerprint("")) + _, err = daemonRuntimeStore(home).Write(rt.Record) + require.NoError(err) + + setMCPTokenTestFlags(t, nil) + mcpHTTPAddr = "127.0.0.1:0" + mcpCmd.SetContext(withStoreResolverConfig(t, cfg)) + _, key, err := prepareMCPHTTP(mcpCmd, cfg) + require.NoError(err) + assert.Empty(key, "a loopback interface must keep the existing keyless daemon usable") + _, err = os.Stat(cfg.ServerKeyFilePath()) + require.ErrorIs(err, os.ErrNotExist) + + // A new client must still discover and authenticate the same daemon. + fresh, err := config.Load(path, home) + require.NoError(err) + client, _, err := OpenHTTPStore(withStoreResolverConfig(t, fresh)) + require.NoError(err) + require.NoError(client.Close()) +} + +func TestMCPLocalDaemonKeyLifecycle(t *testing.T) { //nolint:paralleltest // process environment and MCP flags + for _, tc := range []struct { + name, address string + autostart bool + wantError bool + }{ + {"reuse loopback daemon with bind override", "127.0.0.1:0", false, false}, + {"reject public MCP with keyless daemon", "0.0.0.0:0", false, true}, + {"use key created during daemon startup", "0.0.0.0:0", true, false}, + } { + t.Run(tc.name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + clearServerKeyEnvironment(t) + t.Setenv("MSGVAULT_REMOTE_URL", "") + t.Setenv("MSGVAULT_ALLOW_INSECURE", "false") + home := t.TempDir() + port := freeTCPPort(t) + path := filepath.Join(home, "config.toml") + require.NoError(os.WriteFile(path, []byte(fmt.Sprintf("[server]\nbind_addr = '0.0.0.0'\napi_port = %d\ndaemon_auto_start = %t\ndaemon_auto_restart = 'never'\n[analytics]\nengine = 'sql'\n", port, tc.autostart)), 0o600)) + cfg, err := config.Load(path, home) + require.NoError(err) + daemonCtx, stopDaemon := context.WithCancel(t.Context()) + daemonDone := make(chan error, 1) + started := false + startDaemon := func() (*backgroundServeProcess, error) { + overrides := config.RuntimeOverrides{} + if !tc.autostart { + overrides.BindAddr = new("127.0.0.1") + } + owner, err := config.LoadWithOverrides(path, home, overrides) + if err != nil { + return nil, err + } + command := &cobra.Command{Use: "serve"} + command.SetContext(testInvocationContext(daemonCtx, owner, invocationOptions{})) + started = true + go func() { daemonDone <- runServe(command, nil) }() + return &backgroundServeProcess{PID: os.Getpid(), Wait: daemonDone}, nil + } + t.Cleanup(func() { + stopDaemon() + if started { + select { + case err := <-daemonDone: + assert.NoError(err) + case <-time.After(serveLifecycleTestTimeout): + assert.Fail("daemon did not stop") + } + } + }) + if tc.autostart { + // Replace only process spawning; run the real daemon, including + // ownership, key creation, runtime publication, and HTTP handlers. + stubStartServeBackgroundProcess(t, func(*config.Config, backgroundServeStartOptions) (*backgroundServeProcess, error) { + return startDaemon() + }) + } else { + _, err = startDaemon() + require.NoError(err) + waitForServeHealthBounded(t, port, daemonDone) + } + setMCPTokenTestFlags(t, nil) + mcpHTTPAddr = tc.address + ctx, cancel := context.WithCancel(withStoreResolverConfig(t, cfg)) + defer cancel() + mcpCmd.SetContext(ctx) + if tc.wantError { + require.ErrorContains(mcpCmd.RunE(mcpCmd, nil), "refusing to bind a non-loopback address") + } else { + done := make(chan error, 1) + exited := false + go func() { done <- mcpCmd.RunE(mcpCmd, nil) }() + t.Cleanup(func() { + cancel() + if exited { + return + } + select { + case err := <-done: + assert.ErrorIs(err, context.Canceled) + case <-time.After(serveLifecycleTestTimeout): + assert.Fail("MCP did not stop") + } + }) + var endpoint string + var serveErr error + require.Eventually(func() bool { + select { + case serveErr = <-done: + exited = true + return true + default: + } + entries, err := mcpdiscovery.List(filepath.Join(home, "mcp")) + if err != nil || len(entries) != 1 { + return false + } + endpoint = entries[0].URL + return true + }, serveLifecycleTestTimeout, 20*time.Millisecond) + require.False(exited, "MCP stopped before becoming ready: %v", serveErr) + parsed, err := url.Parse(endpoint) + require.NoError(err) + _, listenPort, err := net.SplitHostPort(parsed.Host) + require.NoError(err) + parsed.Host = net.JoinHostPort("127.0.0.1", listenPort) + var key string + if tc.autostart { + key, err = providercredentials.ReadSecretFile(cfg.ServerKeyFilePath()) + require.NoError(err) + } + client := &http.Client{Timeout: serveLifecycleTestTimeout} + for _, token := range []string{"", key} { + request, err := http.NewRequestWithContext(ctx, http.MethodPost, parsed.String(), strings.NewReader(`{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"example-client","version":"test"}}}`)) + require.NoError(err) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Accept", "application/json, text/event-stream") + if token != "" { + request.Header.Set("Authorization", "Bearer "+token) + } + response, err := client.Do(request) + require.NoError(err) + body, err := io.ReadAll(response.Body) + require.NoError(err) + require.NoError(response.Body.Close()) + if tc.autostart && token == "" { + assert.Equal(http.StatusUnauthorized, response.StatusCode) + } else { + assert.Equal(http.StatusOK, response.StatusCode, string(body)) + assert.Contains(string(body), `"protocolVersion"`) + } + } + } + if !tc.autostart { + _, err = os.Stat(cfg.ServerKeyFilePath()) + require.ErrorIs(err, os.ErrNotExist) + } + fresh, err := config.Load(path, home) + require.NoError(err) + client, _, err := OpenHTTPStore(withStoreResolverConfig(t, fresh)) + require.NoError(err, "fresh clients must still connect to the same daemon") + require.NoError(client.Close()) + }) + } +} + +func TestMCPIndependentInboundTokenWithEnvironmentOnlyBackend(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + var wrongBackendKey atomic.Bool + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("X-Api-Key") != "backend-key" { + wrongBackendKey.Store(true) + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + w.Header().Set("Content-Type", "application/json") + if r.URL.Path == "/api/v1/health" { + _ = json.NewEncoder(w).Encode(map[string]any{"status": "ok", "api_schema_version": api.APISchemaVersion}) + return + } + http.NotFound(w, r) + })) + defer backend.Close() + home := t.TempDir() + file := filepath.Join(t.TempDir(), "inbound-key") + require.NoError(fileutil.SecureWriteFile(file, []byte("inbound-key\n"), 0o400)) + t.Setenv("MSGVAULT_REMOTE_URL", backend.URL) + t.Setenv("MSGVAULT_REMOTE_API_KEY", "backend-key") + t.Setenv("MSGVAULT_REMOTE_ALLOW_INSECURE", "true") + t.Setenv("MSGVAULT_API_KEY_FILE", filepath.Join(home, "unused-missing-server-key")) + t.Setenv("MSGVAULT_TEST_MCP_LOWER_KEY", "lower-priority-key") + cfg, err := config.Load("", home) + require.NoError(err) + setMCPTokenTestFlags(t, map[string]string{"http-token-file": file, "http-token-env": "MSGVAULT_TEST_MCP_LOWER_KEY"}) + mcpHTTPAddr = "0.0.0.0:0" + ctx, cancel := context.WithCancel(withStoreResolverConfig(t, cfg)) + mcpCmd.SetContext(ctx) + done := make(chan error, 1) + go func() { done <- mcpCmd.RunE(mcpCmd, nil) }() + t.Cleanup(func() { + cancel() + select { + case err := <-done: + assert.ErrorIs(err, context.Canceled) + case <-time.After(serveLifecycleTestTimeout): + assert.Fail("MCP did not stop") + } + }) + var endpoint string + require.Eventually(func() bool { + entries, err := mcpdiscovery.List(filepath.Join(home, "mcp")) + if err != nil || len(entries) != 1 { + return false + } + endpoint = entries[0].URL + return true + }, serveLifecycleTestTimeout, 20*time.Millisecond) + parsed, err := url.Parse(endpoint) + require.NoError(err) + _, port, err := net.SplitHostPort(parsed.Host) + require.NoError(err) + parsed.Host = net.JoinHostPort("127.0.0.1", port) + endpoint = parsed.String() + client := &http.Client{Timeout: serveLifecycleTestTimeout} + for _, token := range []string{"", "backend-key", "lower-priority-key", "inbound-key"} { + request, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, strings.NewReader(`{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"example-client","version":"test"}}}`)) + require.NoError(err) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Accept", "application/json, text/event-stream") + if token != "" { + request.Header.Set("Authorization", "Bearer "+token) + } + response, err := client.Do(request) + require.NoError(err) + body, err := io.ReadAll(response.Body) + require.NoError(err) + require.NoError(response.Body.Close()) + if token == "inbound-key" { + assert.Equal(http.StatusOK, response.StatusCode, string(body)) + assert.Contains(string(body), `"protocolVersion"`) + } else { + assert.Equal(http.StatusUnauthorized, response.StatusCode) + } + } + assert.False(wrongBackendKey.Load()) +} diff --git a/cmd/msgvault/cmd/media_summary_test.go b/cmd/msgvault/cmd/media_summary_test.go index 0dcde2fa7..661c80ed6 100644 --- a/cmd/msgvault/cmd/media_summary_test.go +++ b/cmd/msgvault/cmd/media_summary_test.go @@ -42,6 +42,19 @@ func TestProviderSyncSummariesReportPolicySkips(t *testing.T) { } } +func TestBeeperSummariesReportUnavailableMedia(t *testing.T) { + sum := &beeper.ImportSummary{AttachmentsUnavailable: 3} + var syncOut bytes.Buffer + cmd := &cobra.Command{} + cmd.SetOut(&syncOut) + printBeeperSummary(cmd, "account", sum) + assert.Contains(t, syncOut.String(), ", 3 media no longer available at source") + + var backfillOut bytes.Buffer + writeBeeperMediaBackfillSummary(&backfillOut, "account", sum) + assert.Contains(t, backfillOut.String(), ", 3 media no longer available at source") +} + func TestRemainingProviderSummariesReportPolicySkips(t *testing.T) { tests := []struct { name string diff --git a/cmd/msgvault/cmd/meeting_source_test.go b/cmd/msgvault/cmd/meeting_source_test.go index 0915e0b37..355a17c42 100644 --- a/cmd/msgvault/cmd/meeting_source_test.go +++ b/cmd/msgvault/cmd/meeting_source_test.go @@ -37,6 +37,7 @@ func TestMeetingConfigurationHintsLoad(t *testing.T) { {name: "Granola", hint: granolaConfigHint}, {name: "Circleback", hint: circlebackConfigHint}, {name: "Notion", hint: notionMeetingsConfigHint}, + {name: "Muesli", hint: muesliConfigHint}, } { t.Run(tt.name, func(t *testing.T) { assert := assert.New(t) @@ -57,6 +58,9 @@ func TestMeetingConfigurationHintsLoad(t *testing.T) { case "Notion": require.Len(cfg.NotionMeetings, 1) assert.Equal("you@example.com", cfg.NotionMeetings[0].AccountEmail) + case "Muesli": + require.Len(cfg.Muesli, 1) + assert.Equal("you@example.com", cfg.Muesli[0].AccountEmail) } }) } diff --git a/cmd/msgvault/cmd/meetings.go b/cmd/msgvault/cmd/meetings.go new file mode 100644 index 000000000..c267c7026 --- /dev/null +++ b/cmd/msgvault/cmd/meetings.go @@ -0,0 +1,475 @@ +package cmd + +import ( + "context" + "encoding/json/jsontext" + "encoding/json/v2" + "errors" + "fmt" + "io" + "net/http" + "strings" + "text/tabwriter" + "time" + "unicode/utf8" + + "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/daemonclient" + mcpserver "go.kenn.io/msgvault/internal/mcp" + "go.kenn.io/msgvault/internal/meetingcontent" + "go.kenn.io/msgvault/pkg/client/generated" +) + +const meetingsMinAPISchemaVersion = "2.27.0" + +type meetingCommandClient interface { + mcpserver.MeetingBackend + APISchemaVersion(ctx context.Context) (string, error) +} + +type meetingCommandDeps struct { + open func(context.Context) (meetingCommandClient, func(), error) +} + +func defaultMeetingCommandDeps() meetingCommandDeps { + return meetingCommandDeps{open: func(ctx context.Context) (meetingCommandClient, func(), error) { + client, _, err := OpenHTTPStore(ctx) + if err != nil { + return nil, func() {}, fmt.Errorf("open daemon: %w", err) + } + return client, func() { _ = client.Close() }, nil + }} +} + +func newMeetingsCommand(deps meetingCommandDeps) *cobra.Command { + command := &cobra.Command{ + Use: "meetings", + Short: "Read archived meeting context, actions, and metrics", + } + command.AddCommand( + newMeetingContextCommand(deps), + newMeetingActionsCommand(deps), + newMeetingMetricsCommand(deps), + ) + return command +} + +func openMeetingCommandClient( + ctx context.Context, + deps meetingCommandDeps, +) (meetingCommandClient, func(), error) { + if deps.open == nil { + return nil, func() {}, errors.New("meeting daemon client is unavailable") + } + client, cleanup, err := deps.open(ctx) + if err != nil { + return nil, func() {}, err + } + if cleanup == nil { + cleanup = func() {} + } + version, err := client.APISchemaVersion(ctx) + if err != nil { + cleanup() + return nil, func() {}, fmt.Errorf("check daemon meeting capability: %w", err) + } + if !daemonclient.APISchemaVersionAtLeast(version, meetingsMinAPISchemaVersion) { + cleanup() + return nil, func() {}, fmt.Errorf( + "meeting commands require daemon API schema %s or newer (daemon reports %q); upgrade the daemon", + meetingsMinAPISchemaVersion, version, + ) + } + return client, cleanup, nil +} + +func newMeetingContextCommand(deps meetingCommandDeps) *cobra.Command { + var messageIDs []int64 + var format string + var includeTranscript bool + var maxBytes int64 + var output string + command := &cobra.Command{ + Use: "context", + Short: "Export deterministic context for selected meetings", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + if err := validateMeetingIDs("--id", messageIDs, true); err != nil { + return usageErr(command, err) + } + format = strings.ToLower(strings.TrimSpace(format)) + if format != "json" && format != "markdown" { + return usageErr(command, errors.New("--format must be json or markdown")) + } + if maxBytes < 4096 || maxBytes > 1048576 { + return usageErr(command, errors.New("--max-bytes must be between 4096 and 1048576")) + } + client, cleanup, err := openMeetingCommandClient(command.Context(), deps) + if err != nil { + return err + } + defer cleanup() + requestFormat := generated.MeetingContextRequestFormat(format) + result, err := client.GetMeetingContext(command.Context(), generated.GetMeetingContextBody{ + MessageIds: &messageIDs, Format: &requestFormat, + IncludeTranscript: &includeTranscript, MaxBytes: &maxBytes, + }) + if err != nil { + return meetingCommandError(err) + } + if result == nil { + return errors.New("meeting context response was empty") + } + return writeMeetingContext(command, output, result.Content) + }, + } + command.Flags().Int64SliceVar(&messageIDs, "id", nil, "Meeting message ID (repeatable)") + command.Flags().StringVar(&format, "format", "markdown", "Context format: json or markdown") + command.Flags().BoolVar(&includeTranscript, "include-transcript", false, "Include archived transcript evidence") + command.Flags().Int64Var(&maxBytes, "max-bytes", 131072, "Maximum UTF-8 content bytes") + command.Flags().StringVarP(&output, "output", "o", "", "Output file (default stdout, use - for stdout)") + return command +} + +func writeMeetingContext(command *cobra.Command, output, content string) error { + if output == "" || output == "-" { + if _, err := io.WriteString(command.OutOrStdout(), content); err != nil { + return fmt.Errorf("write meeting context: %w", err) + } + return nil + } + if _, err := writeAttachmentStreamToFile(output, strings.NewReader(content)); err != nil { + return fmt.Errorf("write meeting context: %w", err) + } + if _, err := fmt.Fprintf(command.ErrOrStderr(), "Wrote meeting context to %s\n", output); err != nil { + return fmt.Errorf("write meeting context notice: %w", err) + } + return nil +} + +type meetingScopeFlags struct { + messageIDs []int64 + sourceIDs []int64 + domains []string + participantIDs []int64 + personID int64 + after string + before string + deletion string +} + +func addMeetingScopeFlags(command *cobra.Command, scope *meetingScopeFlags) { + command.Flags().Int64SliceVar(&scope.messageIDs, "id", nil, "Meeting message ID (repeatable)") + command.Flags().Int64SliceVar(&scope.sourceIDs, "source-id", nil, "Source ID (repeatable)") + command.Flags().StringSliceVar(&scope.domains, "domain", nil, "Exact participant domain (repeatable)") + command.Flags().Int64SliceVar(&scope.participantIDs, "participant-id", nil, "Exact participant ID (repeatable)") + command.Flags().Int64Var(&scope.personID, "person-id", 0, "Durable person ID") + command.Flags().StringVar(&scope.after, "after", "", "Meetings on or after YYYY-MM-DD") + command.Flags().StringVar(&scope.before, "before", "", "Meetings before YYYY-MM-DD") + command.Flags().StringVar(&scope.deletion, "deletion", "any", "Source deletion state: any, active, or deleted") + command.MarkFlagsMutuallyExclusive("person-id", "participant-id") +} + +func (flags *meetingScopeFlags) request(command *cobra.Command) (*generated.MeetingScopeRequest, error) { + if err := validateMeetingIDs("--id", flags.messageIDs, false); err != nil { + return nil, err + } + if err := validateMeetingIDs("--source-id", flags.sourceIDs, false); err != nil { + return nil, err + } + if err := validateMeetingIDs("--participant-id", flags.participantIDs, false); err != nil { + return nil, err + } + if flags.personID < 0 || flags.personID > int64(maxJSONSafeInteger) { + return nil, errors.New("--person-id must be a positive JavaScript-safe integer") + } + if command.Flags().Changed("person-id") && flags.personID == 0 { + return nil, errors.New("--person-id must be positive") + } + if len(flags.domains) > 100 { + return nil, errors.New("--domain accepts at most 100 values") + } + for i := range flags.domains { + flags.domains[i] = strings.TrimSpace(flags.domains[i]) + if flags.domains[i] == "" { + return nil, errors.New("--domain values must be nonempty") + } + } + after, err := parseMeetingCLIDate(flags.after) + if err != nil { + return nil, fmt.Errorf("invalid --after: %w", err) + } + before, err := parseMeetingCLIDate(flags.before) + if err != nil { + return nil, fmt.Errorf("invalid --before: %w", err) + } + if after != nil && before != nil && !after.Before(*before) { + return nil, errors.New("--after must be before --before") + } + flags.deletion = strings.ToLower(strings.TrimSpace(flags.deletion)) + if flags.deletion != "any" && flags.deletion != "active" && flags.deletion != "deleted" { + return nil, errors.New("--deletion must be any, active, or deleted") + } + deletion := generated.MeetingScopeRequestDeletion(flags.deletion) + scope := &generated.MeetingScopeRequest{ + SourceIds: flags.sourceIDs, ParticipantIds: flags.participantIDs, + Domains: flags.domains, After: after, Before: before, Deletion: &deletion, + } + if len(flags.messageIDs) > 0 { + scope.MessageIds = &flags.messageIDs + } + if command.Flags().Changed("person-id") { + scope.PersonID = &flags.personID + } + return scope, nil +} + +const maxJSONSafeInteger = 9007199254740991 + +func validateMeetingIDs(name string, ids []int64, required bool) error { + if required && len(ids) == 0 { + return fmt.Errorf("at least one %s is required", name) + } + if len(ids) > 100 { + return fmt.Errorf("%s accepts at most 100 values", name) + } + for _, id := range ids { + if id < 1 || id > maxJSONSafeInteger { + return fmt.Errorf("%s values must be positive JavaScript-safe integers", name) + } + } + return nil +} + +func parseMeetingCLIDate(value string) (*time.Time, error) { + value = strings.TrimSpace(value) + if value == "" { + return nil, nil //nolint:nilnil // an omitted date is intentionally unbounded + } + parsed, err := time.Parse("2006-01-02", value) + if err != nil { + return nil, errors.New("expected YYYY-MM-DD") + } + return &parsed, nil +} + +func newMeetingActionsCommand(deps meetingCommandDeps) *cobra.Command { + var scopeFlags meetingScopeFlags + var assignee, status, queryText, cursor string + var limit int64 + var jsonOutput bool + command := &cobra.Command{ + Use: "actions", + Short: "List archived meeting action evidence", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + scope, err := scopeFlags.request(command) + if err != nil { + return usageErr(command, err) + } + if limit < 1 || limit > 200 { + return usageErr(command, errors.New("--limit must be between 1 and 200")) + } + status = strings.ToLower(strings.TrimSpace(status)) + if status != "" && status != "pending" && status != "completed" && status != "cancelled" && status != "unknown" { + return usageErr(command, errors.New("--status must be pending, completed, cancelled, or unknown")) + } + queryText = strings.TrimSpace(queryText) + if utf8.RuneCountInString(queryText) > 256 { + return usageErr(command, errors.New("--query must be at most 256 characters")) + } + client, cleanup, err := openMeetingCommandClient(command.Context(), deps) + if err != nil { + return err + } + defer cleanup() + body := generated.ListMeetingActionItemsBody{Scope: scope, Limit: &limit} + if assignee = strings.TrimSpace(assignee); assignee != "" { + body.AssigneeEmail = &assignee + } + if status != "" { + value := generated.MeetingActionsRequestStatus(status) + body.Status = &value + } + if queryText != "" { + body.Query = &queryText + } + if cursor = strings.TrimSpace(cursor); cursor != "" { + body.Cursor = &cursor + } + result, err := client.ListMeetingActionItems(command.Context(), body) + if err != nil { + return meetingCommandError(err) + } + if result == nil { + return errors.New("meeting actions response was empty") + } + if jsonOutput { + return writeMeetingJSON(command, result) + } + return writeMeetingActions(command, result) + }, + } + addMeetingScopeFlags(command, &scopeFlags) + command.Flags().StringVar(&assignee, "assignee", "", "Exact assignee email") + command.Flags().StringVar(&status, "status", "", "Normalized source status") + command.Flags().StringVar(&queryText, "query", "", "Literal title or description substring") + command.Flags().Int64Var(&limit, "limit", 50, "Maximum action rows") + command.Flags().StringVar(&cursor, "cursor", "", "Opaque continuation cursor") + command.Flags().BoolVar(&jsonOutput, flagJSON, false, "Output as JSON") + return command +} + +func writeMeetingActions(command *cobra.Command, page *meetingcontent.ActionsPage) error { + out := tabwriter.NewWriter(command.OutOrStdout(), 0, 4, 2, ' ', 0) + if _, err := fmt.Fprintf(out, "MEETING\tDATE\tSTATUS\tASSIGNEE\tACTION\n"); err != nil { + return fmt.Errorf("write meeting actions: %w", err) + } + for _, row := range page.Rows { + date := "-" + if row.Meeting.OccurredAt != nil { + date = row.Meeting.OccurredAt.UTC().Format("2006-01-02") + } + assignee := row.Action.AssigneeEmail + if assignee == "" { + assignee = row.Action.AssigneeName + } + if assignee == "" { + assignee = "-" + } + if _, err := fmt.Fprintf(out, "%s\t%s\t%s\t%s\t%s\n", + row.Meeting.Title, date, row.Action.Status, assignee, row.Action.Title); err != nil { + return fmt.Errorf("write meeting actions: %w", err) + } + } + if _, err := fmt.Fprintf(out, + "\nTotal actions: %d\tMeetings: %d\tCoverage: %d available, %d partial, %d unsupported, %d unavailable\n", + page.TotalCount, page.Coverage.MeetingCount, page.Coverage.Available, + page.Coverage.Partial, page.Coverage.Unsupported, page.Coverage.Unavailable, + ); err != nil { + return fmt.Errorf("write meeting action totals: %w", err) + } + if page.NextCursor != "" { + if _, err := fmt.Fprintf(out, "Next cursor: %s\n", page.NextCursor); err != nil { + return fmt.Errorf("write meeting action cursor: %w", err) + } + } + if err := out.Flush(); err != nil { + return fmt.Errorf("flush meeting actions: %w", err) + } + return nil +} + +func newMeetingMetricsCommand(deps meetingCommandDeps) *cobra.Command { + var scopeFlags meetingScopeFlags + var jsonOutput bool + command := &cobra.Command{ + Use: "metrics", + Short: "Show archived meeting duration metrics", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + scope, err := scopeFlags.request(command) + if err != nil { + return usageErr(command, err) + } + client, cleanup, err := openMeetingCommandClient(command.Context(), deps) + if err != nil { + return err + } + defer cleanup() + result, err := client.GetMeetingMetrics(command.Context(), generated.GetMeetingMetricsBody{Scope: scope}) + if err != nil { + return meetingCommandError(err) + } + if result == nil { + return errors.New("meeting metrics response was empty") + } + if jsonOutput { + return writeMeetingJSON(command, result) + } + return writeMeetingMetrics(command, result) + }, + } + addMeetingScopeFlags(command, &scopeFlags) + command.Flags().BoolVar(&jsonOutput, flagJSON, false, "Output as JSON") + return command +} + +func writeMeetingMetrics(command *cobra.Command, metrics *meetingcontent.Metrics) error { + out := command.OutOrStdout() + if _, err := fmt.Fprintf(out, + "Meetings: %d\nKnown duration: %d\nUnknown duration: %d\nTotal known seconds: %.0f\n", + metrics.Totals.MeetingCount, metrics.Totals.KnownDurationCount, + metrics.Totals.UnknownDurationCount, metrics.Totals.TotalKnownSeconds, + ); err != nil { + return fmt.Errorf("write meeting metric totals: %w", err) + } + if metrics.Totals.AverageKnownSeconds == nil { + if _, err := fmt.Fprintln(out, "Average known seconds: unavailable"); err != nil { + return fmt.Errorf("write meeting metric average: %w", err) + } + } else if _, err := fmt.Fprintf(out, "Average known seconds: %.0f\n", *metrics.Totals.AverageKnownSeconds); err != nil { + return fmt.Errorf("write meeting metric average: %w", err) + } + if _, err := fmt.Fprintln(out, "\nDuration by basis:"); err != nil { + return fmt.Errorf("write meeting duration heading: %w", err) + } + for _, basis := range metrics.DurationByBasis { + if _, err := fmt.Fprintf(out, " %s: %d meetings, %.0f seconds\n", basis.Basis, basis.Count, basis.TotalSeconds); err != nil { + return fmt.Errorf("write meeting duration basis: %w", err) + } + } + if _, err := fmt.Fprintln(out, "\nMonths:"); err != nil { + return fmt.Errorf("write meeting month heading: %w", err) + } + for _, month := range metrics.Months { + average := "unavailable" + if month.Totals.AverageKnownSeconds != nil { + average = fmt.Sprintf("%.0f", *month.Totals.AverageKnownSeconds) + } + if _, err := fmt.Fprintf(out, + " %s: %d meetings, %d known, %d unknown, %.0f total seconds, %s average seconds\n", + month.Month, month.Totals.MeetingCount, month.Totals.KnownDurationCount, + month.Totals.UnknownDurationCount, month.Totals.TotalKnownSeconds, average, + ); err != nil { + return fmt.Errorf("write meeting month: %w", err) + } + } + return nil +} + +func writeMeetingJSON(command *cobra.Command, value any) error { + encoded, err := json.Marshal(value, jsontext.EscapeForHTML(false)) + if err != nil { + return fmt.Errorf("write meeting JSON: %w", err) + } + if _, err := fmt.Fprintln(command.OutOrStdout(), string(encoded)); err != nil { + return fmt.Errorf("write meeting JSON: %w", err) + } + return nil +} + +func meetingCommandError(err error) error { + var apiErr *daemonclient.APIError + if !errors.As(err, &apiErr) { + return err + } + switch apiErr.Status { + case http.StatusNotFound: + if apiErr.APIErrorCode() != "not_found" { + return err + } + return fmt.Errorf( + "meeting intelligence is unavailable from this daemon; upgrade it to API schema %s or newer: %w", + meetingsMinAPISchemaVersion, err, + ) + case http.StatusServiceUnavailable: + return fmt.Errorf("meeting intelligence is unavailable: %w", err) + default: + return err + } +} + +func init() { + rootCmd.AddCommand(newMeetingsCommand(defaultMeetingCommandDeps())) +} diff --git a/cmd/msgvault/cmd/meetings_test.go b/cmd/msgvault/cmd/meetings_test.go new file mode 100644 index 000000000..195b18002 --- /dev/null +++ b/cmd/msgvault/cmd/meetings_test.go @@ -0,0 +1,237 @@ +package cmd + +import ( + "bytes" + "context" + "fmt" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/daemonclient" + "go.kenn.io/msgvault/internal/meetingimport" + "go.kenn.io/msgvault/internal/testutil" + "go.kenn.io/msgvault/pkg/client/generated" +) + +func newMeetingCommandClient(t *testing.T) (*daemonclient.Client, []meetingimport.Result) { + t.Helper() + st := testutil.NewTestStore(t) + importer := meetingimport.NewImporter(st, meetingimport.Hooks{}) + actions := []meetingimport.MeetingActionItem{ + {SourceID: "send", Title: "Send draft", AssigneeEmail: "alice@example.com", Status: "open"}, + {SourceID: "done", Title: "Publish notes", Status: "completed"}, + } + unrelatedActions := []meetingimport.MeetingActionItem{ + {SourceID: "outside", Title: "Send outside draft", AssigneeEmail: "alice@example.com", Status: "open"}, + } + requests := []meetingimport.Request{ + { + Source: meetingimport.Source{Identifier: "meeting-command-fixture", AccountEmail: "owner@example.com"}, + Meeting: meetingimport.Meeting{ + ExternalID: "one", Title: "Planning review", StartedAt: "2026-03-10T10:00:00Z", + EndedAt: "2026-03-10T10:45:00Z", SummaryText: "Selected the launch owner.", + Transcript: "Alice: I will send the draft.", ActionItems: &actions, + Organizer: &meetingimport.MeetingPerson{Name: "Owner", Email: "owner@example.com"}, + Attendees: []meetingimport.MeetingPerson{{Name: "Alice", Email: "alice@example.com"}}, + }, + }, + { + Source: meetingimport.Source{Identifier: "meeting-command-unrelated", AccountEmail: "owner@other.example"}, + Meeting: meetingimport.Meeting{ + ExternalID: "two", Title: "Status review", StartedAt: "2026-04-11T10:00:00Z", + SummaryText: "Reviewed progress.", + Organizer: &meetingimport.MeetingPerson{Name: "Other Owner", Email: "owner@other.example"}, + Attendees: []meetingimport.MeetingPerson{{Name: "Bob", Email: "bob@other.example"}}, + ActionItems: &unrelatedActions, + }, + }, + } + results := make([]meetingimport.Result, len(requests)) + for i, request := range requests { + var err error + results[i], err = importer.Import(t.Context(), request) + require.NoError(t, err) + } + server := httptest.NewServer(api.NewServer( + &config.Config{}, st, nil, slog.New(slog.DiscardHandler), + ).Router()) + t.Cleanup(server.Close) + client, err := daemonclient.New(daemonclient.Config{ + URL: server.URL, AllowInsecure: true, HTTPClient: server.Client(), + }) + require.NoError(t, err) + t.Cleanup(func() { assert.NoError(t, client.Close()) }) + return client, results +} + +func meetingTestInt64(value int64) string { return strconv.FormatInt(value, 10) } + +func executeMeetingsCommand( + t *testing.T, client meetingCommandClient, args ...string, +) (string, string, error) { + t.Helper() + command := newMeetingsCommand(meetingCommandDeps{open: func(context.Context) (meetingCommandClient, func(), error) { + return client, func() {}, nil + }}) + var stdout, stderr bytes.Buffer + command.SetOut(&stdout) + command.SetErr(&stderr) + command.SetArgs(args) + err := command.ExecuteContext(t.Context()) + return stdout.String(), stderr.String(), err +} + +func TestMeetingsCommandsUseDaemonResultsAndFilters(t *testing.T) { + assertions := assert.New(t) + requirements := require.New(t) + client, imported := newMeetingCommandClient(t) + + _, _, err := executeMeetingsCommand(t, client, + "context", "--id", "999999", "--format", "markdown") + requirements.Error(err) + requirements.ErrorContains(err, "not found") + + messageIDs := []int64{imported[0].MessageID} + format := generated.Markdown + includeTranscript := true + expected, err := client.GetMeetingContext(t.Context(), generated.GetMeetingContextBody{ + MessageIds: &messageIDs, Format: &format, IncludeTranscript: &includeTranscript, + }) + requirements.NoError(err) + stdout, stderr, err := executeMeetingsCommand(t, client, + "context", "--id", meetingTestInt64(imported[0].MessageID), "--include-transcript", "--format", "markdown") + requirements.NoError(err) + assertions.Empty(stderr) + assertions.Equal(expected.Content, stdout) + assertions.Contains(stdout, "Planning review") + assertions.Contains(stdout, "Alice: I will send the draft.") + + stdout, _, err = executeMeetingsCommand(t, client, + "actions", "--assignee", "alice@example.com", "--status", "pending", "--json") + requirements.NoError(err) + assertions.Contains(stdout, `"title":"Send draft"`) + assertions.Contains(stdout, `"title":"Send outside draft"`) + assertions.NotContains(stdout, "Publish notes") + + stdout, _, err = executeMeetingsCommand(t, client, + "actions", "--domain", "example.com", "--assignee", "alice@example.com", "--status", "pending", "--json") + requirements.NoError(err) + assertions.Contains(stdout, `"title":"Send draft"`) + assertions.NotContains(stdout, "Send outside draft") + assertions.NotContains(stdout, "Publish notes") + assertions.NotEqual(imported[0].SourceID, imported[1].SourceID) + stdout, _, err = executeMeetingsCommand(t, client, "metrics") + requirements.NoError(err) + assertions.Contains(stdout, "Meetings: 2") + assertions.Contains(stdout, "Known duration: 1") + assertions.Contains(stdout, "Unknown duration: 1") + assertions.Contains(stdout, "2026-03") + assertions.Contains(stdout, "2026-04") + + stdout, _, err = executeMeetingsCommand(t, client, + "metrics", "--source-id", meetingTestInt64(imported[0].SourceID)) + requirements.NoError(err) + assertions.Contains(stdout, "Meetings: 1") + assertions.Contains(stdout, "Known duration: 1") + assertions.Contains(stdout, "Unknown duration: 0") + assertions.Contains(stdout, "provider") + assertions.Contains(stdout, "2026-03") + assertions.NotContains(stdout, "2026-04") +} + +func TestMeetingContextWritesExactContentAtomically(t *testing.T) { + assertions := assert.New(t) + requirements := require.New(t) + client, imported := newMeetingCommandClient(t) + output := filepath.Join(t.TempDir(), "context.md") + messageIDs := []int64{imported[0].MessageID} + format := generated.Markdown + expected, err := client.GetMeetingContext(t.Context(), generated.GetMeetingContextBody{ + MessageIds: &messageIDs, Format: &format, + }) + requirements.NoError(err) + + stdout, stderr, err := executeMeetingsCommand(t, client, + "context", "--id", meetingTestInt64(imported[0].MessageID), "--format", "markdown", "--output", output) + requirements.NoError(err) + assertions.Empty(stdout) + assertions.Contains(stderr, output) + content, err := os.ReadFile(output) + requirements.NoError(err) + assertions.Equal(expected.Content, string(content)) + assertions.Contains(string(content), "Planning review") +} + +func TestMeetingsCommandsFailClosedForOlderDaemon(t *testing.T) { + tests := []struct { + name string + schemaVersion string + }{ + {name: "well before the meeting endpoints", schemaVersion: "2.23.0"}, + {name: "meeting predecessor 2.25.x", schemaVersion: "2.25.0"}, + {name: "meeting predecessor 2.26.x", schemaVersion: "2.26.0"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + routeCalls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v1/health" { + w.Header().Set("Content-Type", "application/json") + _, err := fmt.Fprintf(w, `{"status":"ok","api_schema_version":%q}`, tt.schemaVersion) + assert.NoError(err) + return + } + routeCalls++ + http.NotFound(w, r) + })) + t.Cleanup(server.Close) + client, err := daemonclient.New(daemonclient.Config{ + URL: server.URL, AllowInsecure: true, HTTPClient: server.Client(), + }) + require.NoError(err) + + _, _, err = executeMeetingsCommand(t, client, "metrics", "--json") + require.Error(err, "daemon schema %s should fail closed", tt.schemaVersion) + require.ErrorContains(err, "API schema 2.27.0 or newer") + require.ErrorContains(err, "upgrade the daemon") + assert.Zero(routeCalls) + }) + } +} + +func TestMeetingsCommandsReportMissingDaemonRoute(t *testing.T) { + requirements := require.New(t) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.URL.Path == "/api/v1/health" { + _, err := w.Write([]byte(`{"status":"ok","api_schema_version":"2.27.0"}`)) + assert.NoError(t, err) + return + } + w.WriteHeader(http.StatusNotFound) + _, err := w.Write([]byte(`{"error":"not_found","message":"route not found"}`)) + assert.NoError(t, err) + })) + t.Cleanup(server.Close) + client, err := daemonclient.New(daemonclient.Config{ + URL: server.URL, AllowInsecure: true, HTTPClient: server.Client(), + }) + requirements.NoError(err) + t.Cleanup(func() { assert.NoError(t, client.Close()) }) + + _, _, err = executeMeetingsCommand(t, client, "metrics", "--json") + requirements.Error(err) + requirements.ErrorContains(err, "meeting intelligence is unavailable") + requirements.ErrorContains(err, "upgrade it to API schema 2.27.0 or newer") +} diff --git a/cmd/msgvault/cmd/microsoft_oauth.go b/cmd/msgvault/cmd/microsoft_oauth.go index b9e14af64..b77004409 100644 --- a/cmd/msgvault/cmd/microsoft_oauth.go +++ b/cmd/msgvault/cmd/microsoft_oauth.go @@ -1,10 +1,15 @@ package cmd import ( + "context" "errors" "fmt" + "log/slog" "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/microsoft" + "go.kenn.io/msgvault/internal/teams" ) // oauthPreflightedFlag marks that the frontend CLI already completed the @@ -28,8 +33,8 @@ func oauthPreflighted(cmd *cobra.Command) (bool, error) { return preflighted, nil } -func requireMicrosoftOAuthConfig() error { - if cfg.Microsoft.ClientID == "" { +func requireMicrosoftOAuthConfig(cfg *config.Config) error { + if cfg == nil || cfg.Microsoft.ClientID == "" { return errors.New("microsoft OAuth not configured\n\n" + "Add to your config.toml:\n\n" + " [microsoft]\n" + @@ -39,11 +44,29 @@ func requireMicrosoftOAuthConfig() error { return nil } +// newTeamsClient builds a Graph client for email's persisted Teams token. +func newTeamsClient(ctx context.Context, cfg *config.Config, logger *slog.Logger, email string) (*teams.Client, error) { + mgr := microsoft.NewGraphManager(cfg.Microsoft.ClientID, cfg.Microsoft.EffectiveTenantID(), + cfg.Microsoft.EffectiveRedirectURI(), cfg.TokensDir(), logger) + tokenFn, err := mgr.TokenSource(ctx, email) + if err != nil { + return nil, err + } + qps := float64(cfg.Sync.RateLimitQPS) + if qps <= 0 { + qps = 5 + } + return teams.NewClient("https://graph.microsoft.com/v1.0", tokenFn, qps), nil +} + // microsoftTenantID resolves the tenant, letting a per-command flag // override the configured default. -func microsoftTenantID(flagTenant string) string { +func microsoftTenantID(flagTenant string, cfg *config.Config) string { if flagTenant != "" { return flagTenant } + if cfg == nil { + return "" + } return cfg.Microsoft.EffectiveTenantID() } diff --git a/cmd/msgvault/cmd/muesli.go b/cmd/msgvault/cmd/muesli.go new file mode 100644 index 000000000..c20f4788d --- /dev/null +++ b/cmd/msgvault/cmd/muesli.go @@ -0,0 +1,295 @@ +package cmd + +import ( + "context" + "errors" + "fmt" + "io" + "strings" + "time" + + "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/muesli" + "go.kenn.io/msgvault/internal/store" +) + +var ( + syncMuesliLimit int + syncMuesliAfter string + syncMuesliFull bool +) + +var ( + rebuildMuesliCacheAfterWrite = rebuildCacheAfterWrite + rebuildMuesliCacheAfterScheduledSync = rebuildCacheAfterScheduledSync +) + +const muesliConfigHint = `Add to your config.toml: + + [[muesli]] + identifier = "mac" # stable label for this Mac's Muesli database + account_email = "you@example.com" # you, the person who records the meetings + enabled = true + # db_path = "~/Library/Application Support/Muesli/muesli.db" # default shown + # schedule = "*/30 * * * *" # optional daemon schedule + # phone_country_code = "1" # convert national-format Contacts phones + # contacts = false # skip Apple Contacts attendee lookup` + +// resolveMuesliSources picks [[muesli]] entries: an explicit identifier must +// match one entry; with no argument every configured entry is returned. +func resolveMuesliSources(args []string, cfg *config.Config) ([]config.MuesliSource, error) { + if len(cfg.Muesli) == 0 { + return nil, errors.New("no [[muesli]] sources configured\n\n" + muesliConfigHint) + } + if len(args) == 0 { + return cfg.Muesli, nil + } + source := cfg.GetMuesliSource(args[0]) + if source == nil { + identifiers := make([]string, 0, len(cfg.Muesli)) + for _, candidate := range cfg.Muesli { + identifiers = append(identifiers, candidate.Identifier) + } + return nil, fmt.Errorf("no [[muesli]] entry with identifier %q (configured: %s)", + args[0], strings.Join(identifiers, ", ")) + } + return []config.MuesliSource{*source}, nil +} + +// probeMuesliDatabase proves the configured database opens read-only and +// has Muesli's meetings table. +func probeMuesliDatabase(ctx context.Context, path string) error { + reader, err := muesli.Open(ctx, path) + if err != nil { + return fmt.Errorf("%w\n\nCheck db_path in the [[muesli]] entry. msgvault reads the database on the "+ + "daemon's host; if macOS blocks the read, grant the daemon Full Disk Access", err) + } + return reader.Close() +} + +var addMuesliCmd = &cobra.Command{ + Use: "add-muesli [identifier]", + Short: "Register a local Muesli meeting database", + Long: `Register a configured Muesli database as a msgvault meeting source. + +Reads db_path from the matching [[muesli]] entry in config.toml (default: +~/Library/Application Support/Muesli/muesli.db) and checks that it opens +read-only as a Muesli database. The daemon reads the file on its own host, +so msgvault must run on the Mac where Muesli records. + +Examples: + msgvault add-muesli + msgvault add-muesli mac`, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + if !isDaemonCLISubprocess() { + return runDaemonCLICommandHTTPFromCobra(cmd, args) + } + sources, err := resolveMuesliSources(args, cfg) + if err != nil { + return err + } + if len(sources) > 1 { + return errors.New("multiple [[muesli]] sources configured; pass an identifier") + } + source := sources[0] + accountEmail, err := source.EffectiveAccountEmail() + if err != nil { + return err + } + if err := probeMuesliDatabase(cmd.Context(), source.EffectiveDBPath()); err != nil { + return err + } + if source.ContactsEnabled() { + contacts, err := muesli.OpenContacts(cmd.Context(), source.EffectiveContactsPath()) + if err != nil { + return err + } + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Apple Contacts: %s\n", contacts.State()) + if contacts.State() != muesli.ContactsComplete { + _, _ = fmt.Fprintln(cmd.OutOrStdout(), + "Grant the msgvault daemon Full Disk Access so attendees picked from Contacts link to people.") + } + } + st, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) + if err != nil { + return err + } + defer cleanup() + if _, err := registerMeetingSource(cmd.OutOrStdout(), st, sourceTypeMuesli, + source.Identifier, accountEmail); err != nil { + return err + } + if err := runPostSourceCreateMigrationsForInvocation(st, state); err != nil { + return fmt.Errorf("post-source-create migrations: %w", err) + } + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "\nMuesli source %s registered.\n", source.Identifier) + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Run: msgvault sync-muesli %s\n", source.Identifier) + return nil + }, +} + +var syncMuesliCmd = &cobra.Command{ + Use: "sync-muesli [identifier]", + Short: "Sync meetings from a local Muesli database", + Long: `Archive completed Muesli meetings: AI notes, typed notes, transcript, and +participant emails. With no identifier, every configured [[muesli]] source is +synced. + +Every run reads the whole database read-only and updates meetings that +changed in place. Meetings still recording or processing wait for a later run. +Meetings deleted in Muesli stay archived. + +Examples: + msgvault sync-muesli + msgvault sync-muesli mac --limit 5 + msgvault sync-muesli --after 2026-01-01 # UTC date + msgvault sync-muesli --full`, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + if !isDaemonCLISubprocess() { + return runDaemonCLICommandHTTPFromCobra(cmd, args) + } + sources, err := resolveMuesliSources(args, cfg) + if err != nil { + return err + } + var after time.Time + if syncMuesliAfter != "" { + parsed, err := time.Parse(time.DateOnly, syncMuesliAfter) + if err != nil { + return usageErr(cmd, fmt.Errorf("invalid --after %q (expected YYYY-MM-DD): %w", syncMuesliAfter, err)) + } + after = parsed.UTC() + } + for _, source := range sources { + if _, err := source.EffectiveAccountEmail(); err != nil { + return err + } + } + + st, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) + if err != nil { + return err + } + defer cleanup() + dbPath := cfg.DatabaseDSN() + + ctx, stop := withInterruptCancel(cmd, "\nInterrupted. Finishing current meeting...") + defer stop() + + pendingWrites := &muesli.ImportSummary{} + for _, source := range sources { + accountEmail, _ := source.EffectiveAccountEmail() + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Syncing Muesli for %s\n\n", source.Identifier) + options := muesliImportOptions(source) + options.AccountEmail = accountEmail + options.Full, options.Limit, options.StartedAfter = syncMuesliFull, syncMuesliLimit, after + options.Progress = func(line string) { _, _ = fmt.Fprintln(cmd.OutOrStdout(), " "+line) } + summary, importErr := muesli.NewImporter(st).Import(ctx, options) + if summary != nil { + pendingWrites.MeetingsAdded += summary.MeetingsAdded + pendingWrites.MeetingsUpdated += summary.MeetingsUpdated + } + if err := finishMuesliImport(source.Identifier, pendingWrites, importErr, + func() error { return rebuildMuesliCacheAfterWrite(dbPath, state) }); err != nil { + return err + } + writeMuesliSummary(cmd.OutOrStdout(), summary) + } + return rebuildMuesliCacheAfterWrite(dbPath, state) + }, +} + +// finishMuesliImport reports a failed sync, first refreshing the cache when +// the run still archived meetings. +func finishMuesliImport(identifier string, summary *muesli.ImportSummary, importErr error, refresh func() error) error { + if importErr == nil { + return nil + } + var refreshErr error + if summary != nil && summary.MeetingsAdded+summary.MeetingsUpdated > 0 && refresh != nil { + refreshErr = refresh() + } + return errors.Join(fmt.Errorf("muesli sync %s failed: %w", identifier, importErr), refreshErr) +} + +func writeMuesliSummary(out io.Writer, summary *muesli.ImportSummary) { + _, _ = fmt.Fprintln(out, "\nMuesli sync complete!") + _, _ = fmt.Fprintf(out, " Duration: %s\n", summary.Duration.Round(time.Second)) + _, _ = fmt.Fprintf(out, " Meetings processed: %d\n", summary.MeetingsProcessed) + _, _ = fmt.Fprintf(out, " Meetings added: %d\n", summary.MeetingsAdded) + _, _ = fmt.Fprintf(out, " Meetings updated: %d\n", summary.MeetingsUpdated) + if summary.SkippedInProgress > 0 { + _, _ = fmt.Fprintf(out, " Still in progress: %d (archived by a later sync)\n", summary.SkippedInProgress) + } + if summary.SkippedDeleted > 0 { + _, _ = fmt.Fprintf(out, " Deleted in Muesli: %d (kept archived)\n", summary.SkippedDeleted) + } + if summary.SkippedEmpty > 0 { + _, _ = fmt.Fprintf(out, " Empty: %d (no notes or transcript)\n", summary.SkippedEmpty) + } + if summary.ContactsState != "" { + _, _ = fmt.Fprintf(out, " Contacts: %s\n", summary.ContactsState) + } + if summary.ContactsState == muesli.ContactsUnavailable || summary.ContactsState == muesli.ContactsPartial { + _, _ = fmt.Fprintln(out, " Grant the msgvault daemon Full Disk Access so attendees picked from Contacts link to people.") + } +} + +// muesliImportOptions carries a configured source's paths and Contacts +// settings into an import. +func muesliImportOptions(source config.MuesliSource) muesli.ImportOptions { + return muesli.ImportOptions{ + Identifier: source.Identifier, AccountEmail: source.AccountEmail, + DBPath: source.EffectiveDBPath(), + ContactsEnabled: source.ContactsEnabled(), ContactsPath: source.EffectiveContactsPath(), + PhoneCountryCode: source.PhoneCountryCode, + } +} + +// runConfiguredMuesliSync is the daemon-scheduler entry point for one +// [[muesli]] source. +func runConfiguredMuesliSync(ctx context.Context, st *store.Store, source config.MuesliSource) error { + if _, err := st.GetSourceByTypeAndIdentifier(muesli.SourceType, source.Identifier); err != nil { + if errors.Is(err, store.ErrSourceNotFound) { + return fmt.Errorf("muesli source %q is not registered; run msgvault add-muesli %s first", + source.Identifier, source.Identifier) + } + return err + } + accountEmail, err := source.EffectiveAccountEmail() + if err != nil { + return err + } + options := muesliImportOptions(source) + options.AccountEmail = accountEmail + summary, importErr := muesli.NewImporter(st).Import(ctx, options) + refreshCtx := context.WithoutCancel(ctx) + refresh := func() error { + return rebuildMuesliCacheAfterScheduledSync(refreshCtx, "muesli:"+source.Identifier) + } + if err := finishMuesliImport(source.Identifier, summary, importErr, refresh); err != nil { + return err + } + return refresh() +} + +func init() { + syncMuesliCmd.Flags().IntVar(&syncMuesliLimit, "limit", 0, "max meetings processed per run (0 = no limit)") + syncMuesliCmd.Flags().StringVar(&syncMuesliAfter, "after", "", "only meetings that start on or after this UTC date (YYYY-MM-DD)") + syncMuesliCmd.Flags().BoolVar(&syncMuesliFull, "full", false, "rewrite every archived meeting, even unchanged ones (refreshes attribution)") + rootCmd.AddCommand(addMuesliCmd) + rootCmd.AddCommand(syncMuesliCmd) +} diff --git a/cmd/msgvault/cmd/muesli_test.go b/cmd/msgvault/cmd/muesli_test.go new file mode 100644 index 000000000..aa55839a7 --- /dev/null +++ b/cmd/msgvault/cmd/muesli_test.go @@ -0,0 +1,203 @@ +package cmd + +import ( + "bytes" + "context" + "database/sql" + "encoding/json/v2" + "errors" + "fmt" + "net/http" + "os" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/muesli" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil" +) + +func TestResolveMuesliSources(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + cfg := &config.Config{} + _, err := resolveMuesliSources(nil, cfg) + require.Error(err) + assert.Contains(err.Error(), "[[muesli]]") + + cfg = &config.Config{Muesli: []config.MuesliSource{ + {Identifier: "mac", AccountEmail: "you@example.com"}, + {Identifier: "studio", AccountEmail: "you@example.com"}, + }} + all, err := resolveMuesliSources(nil, cfg) + require.NoError(err) + assert.Len(all, 2) + + one, err := resolveMuesliSources([]string{"STUDIO"}, cfg) + require.NoError(err) + require.Len(one, 1) + assert.Equal("studio", one[0].Identifier) + + _, err = resolveMuesliSources([]string{"laptop"}, cfg) + require.Error(err) + assert.Contains(err.Error(), "configured: mac, studio") +} + +func TestProbeMuesliDatabaseRejectsForeignFiles(t *testing.T) { + path := filepath.Join(t.TempDir(), "notes.txt") + require.NoError(t, os.WriteFile(path, []byte("not sqlite"), 0o600)) + + err := probeMuesliDatabase(context.Background(), path) + + require.Error(t, err) + assert.Contains(t, err.Error(), "db_path") +} + +func TestRunConfiguredMuesliSyncRefusesUnregisteredSource(t *testing.T) { + st := testutil.NewTestStore(t) + + err := runConfiguredMuesliSync(context.Background(), st, config.MuesliSource{ + Identifier: "removed", AccountEmail: "you@example.com", + }) + + require.Error(t, err) + assert.Contains(t, err.Error(), "add-muesli removed") +} + +func TestServeScheduledMuesliSyncCompletes(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + cfg := lifecycleTestConfig(t.TempDir()) + cfg.Server.APIPort = freeTCPPort(t) + cfg.Analytics.Engine = config.AnalyticsEngineSQL + cfg.Analytics.AutoBuildCache = false + cfg.Vector.Enabled = false + path := filepath.Join(t.TempDir(), "muesli.db") + db, err := sql.Open("sqlite3", path) + require.NoError(err) + t.Cleanup(func() { _ = db.Close() }) + _, err = db.Exec(`CREATE TABLE meetings (id INTEGER PRIMARY KEY, title TEXT, start_time TEXT, created_at TEXT, raw_transcript TEXT); + INSERT INTO meetings VALUES (1, 'Planning', '2026-09-01T14:00:00Z', '2026-09-01 14:00:03', 'Synthetic meeting notes')`) + require.NoError(err) + require.NoError(db.Close()) + contacts := false + cfg.Muesli = []config.MuesliSource{{ + Identifier: "mac", AccountEmail: "you@example.com", DBPath: path, + Contacts: &contacts, Enabled: true, Schedule: "0 0 1 1 *", + }} + st, err := store.Open(cfg.DatabaseDSN()) + require.NoError(err) + t.Cleanup(func() { _ = st.Close() }) + require.NoError(st.InitSchema()) + _, err = st.GetOrCreateSource(muesli.SourceType, "mac") + require.NoError(err) + + ctx, cancel := context.WithCancel(t.Context()) + cmd := &cobra.Command{Use: serveCmd.Use} + cmd.SetContext(testInvocationContext(ctx, cfg, invocationOptions{})) + errCh := make(chan error, 1) + go func() { errCh <- runServe(cmd, nil) }() + t.Cleanup(func() { + cancel() + select { + case err := <-errCh: + require.NoError(err) + case <-time.After(serveLifecycleTestTimeout): + require.FailNow("daemon did not stop") + } + }) + waitForServeHealth(t, cfg.Server.APIPort, errCh) + baseURL := fmt.Sprintf("http://127.0.0.1:%d", cfg.Server.APIPort) + client := &http.Client{Timeout: time.Second} + response, err := client.Post(baseURL+"/api/v1/sync/mac?source_type=muesli", "application/json", nil) + require.NoError(err) + require.NoError(response.Body.Close()) + require.Equal(http.StatusAccepted, response.StatusCode) + var status api.SourceStatusResponse + require.Eventually(func() bool { + response, err := client.Get(baseURL + "/api/v1/sources/status?source_type=muesli") + if err != nil { + return false + } + defer func() { _ = response.Body.Close() }() + return json.UnmarshalRead(response.Body, &status) == nil && len(status.Sources) == 1 && + status.Sources[0].LastSuccessfulSync != nil && status.Sources[0].CanSync + }, serveLifecycleTestTimeout, 20*time.Millisecond, "scheduled import did not finish") + assert.Equal(int64(1), status.Sources[0].LastSuccessfulSync.MessagesAdded) + assert.Empty(status.Sources[0].SchedulerLastError, "post-import cache refresh must receive the daemon configuration") +} + +func TestFinishMuesliImportRefreshesCacheAfterPartialWrites(t *testing.T) { + refreshed := 0 + err := finishMuesliImport("mac", &muesli.ImportSummary{MeetingsAdded: 1}, + errors.New("meeting 3 failed"), func() error { refreshed++; return nil }) + + require.Error(t, err) + assert.Equal(t, 1, refreshed) + assert.Contains(t, err.Error(), "muesli sync mac failed") +} + +func TestWriteMuesliSummaryReportsSkippedMeetings(t *testing.T) { + var out bytes.Buffer + + writeMuesliSummary(&out, &muesli.ImportSummary{ + MeetingsProcessed: 3, MeetingsAdded: 2, SkippedDeleted: 4, SkippedEmpty: 1, SkippedInProgress: 1, + }) + + assert.Contains(t, out.String(), "Deleted in Muesli: 4 (kept archived)") + assert.Contains(t, out.String(), "Empty: 1 (no notes or transcript)") + assert.Contains(t, out.String(), "Still in progress: 1") +} + +func TestWriteMuesliSummaryOmitsZeroSkipCounts(t *testing.T) { + var out bytes.Buffer + + writeMuesliSummary(&out, &muesli.ImportSummary{MeetingsProcessed: 1, MeetingsAdded: 1}) + + assert.NotContains(t, out.String(), "Deleted in Muesli") + assert.NotContains(t, out.String(), "Empty:") +} + +func TestWriteMuesliSummaryReportsContactsState(t *testing.T) { + var out bytes.Buffer + + writeMuesliSummary(&out, &muesli.ImportSummary{ContactsState: muesli.ContactsUnavailable}) + + assert.Contains(t, out.String(), "Contacts: unavailable") + assert.Contains(t, out.String(), "Full Disk Access") +} + +func TestMuesliImportOptionsCarryContactsSettings(t *testing.T) { + enabled := false + opts := muesliImportOptions(config.MuesliSource{ + Identifier: "mac", AccountEmail: "you@example.com", DBPath: "/tmp/muesli.db", + Contacts: &enabled, ContactsPath: "/tmp/AddressBook", PhoneCountryCode: "44", + }) + + assert.Equal(t, muesli.ImportOptions{ + Identifier: "mac", AccountEmail: "you@example.com", DBPath: "/tmp/muesli.db", + ContactsEnabled: false, ContactsPath: "/tmp/AddressBook", PhoneCountryCode: "44", + }, opts) +} + +func TestMuesliCommandsUseInvocationConfiguration(t *testing.T) { + t.Setenv(daemonCLISubprocessEnv, strconv.Itoa(os.Getppid())) + cfg := &config.Config{Muesli: []config.MuesliSource{{Identifier: "mac"}}} + for _, command := range []*cobra.Command{addMuesliCmd, syncMuesliCmd} { + t.Run(command.Name(), func(t *testing.T) { + cmd := &cobra.Command{} + cmd.SetContext(testInvocationContext(t.Context(), cfg, invocationOptions{})) + + err := command.RunE(cmd, []string{"missing"}) + + assert.ErrorContains(t, err, `no [[muesli]] entry with identifier "missing" (configured: mac)`) + }) + } +} diff --git a/cmd/msgvault/cmd/multimodal_probe.go b/cmd/msgvault/cmd/multimodal_probe.go index edadfaecb..b5e3ea725 100644 --- a/cmd/msgvault/cmd/multimodal_probe.go +++ b/cmd/msgvault/cmd/multimodal_probe.go @@ -43,6 +43,11 @@ then run 'msgvault multimodal build --yes' to consent to exactly that capability profile.`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg if strings.TrimSpace(multimodalProbeSeeds) == "" { return usageErr(cmd, errors.New("--seeds is required: a private directory holding synthetic image_webp.webp, image_webp_alt.webp, video_mp4.mp4, and video_mp4_alt.mp4 seeds")) } diff --git a/cmd/msgvault/cmd/notion_meetings.go b/cmd/msgvault/cmd/notion_meetings.go index 92fdc4461..7726c5cc3 100644 --- a/cmd/msgvault/cmd/notion_meetings.go +++ b/cmd/msgvault/cmd/notion_meetings.go @@ -25,7 +25,7 @@ var ( newNotionMeetingsClient = func(baseURL, token string) notionmeetings.Source { return notionmeetings.NewClient(baseURL, token) } - rebuildNotionMeetingsCacheAfterWrite = rebuildCacheAfterWrite + rebuildNotionMeetingsCacheAfterWrite = rebuildCacheAfterManualSync rebuildNotionMeetingsCacheAfterScheduledSync = rebuildCacheAfterScheduledSync ) @@ -38,7 +38,10 @@ const notionMeetingsConfigHint = `Add to your config.toml: enabled = true # schedule = "15 */6 * * *" # optional daemon schedule` -func resolveNotionMeetingsSource(args []string) (*config.NotionMeetingsSource, error) { +func resolveNotionMeetingsSource(args []string, cfg *config.Config) (*config.NotionMeetingsSource, error) { + if cfg == nil { + return nil, errors.New("configuration is unavailable") + } if len(cfg.NotionMeetings) == 0 { return nil, errors.New("no [[notion_meetings]] sources configured\n\n" + notionMeetingsConfigHint) } @@ -61,9 +64,12 @@ func resolveNotionMeetingsSource(args []string) (*config.NotionMeetingsSource, e return &source, nil } -func resolveNotionMeetingsSources(args []string, probe bool) ([]config.NotionMeetingsSource, error) { +func resolveNotionMeetingsSources(args []string, probe bool, cfg *config.Config) ([]config.NotionMeetingsSource, error) { + if cfg == nil { + return nil, errors.New("configuration is unavailable") + } if probe || len(args) > 0 || len(cfg.NotionMeetings) == 1 { - source, err := resolveNotionMeetingsSource(args) + source, err := resolveNotionMeetingsSource(args, cfg) if err != nil { return nil, err } @@ -133,10 +139,15 @@ var addNotionMeetingsCmd = &cobra.Command{ Short: "Register and validate a Notion AI Meeting Notes source", Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg if !isDaemonCLISubprocess() { return runDaemonCLICommandHTTPFromCobra(cmd, args) } - source, err := resolveNotionMeetingsSource(args) + source, err := resolveNotionMeetingsSource(args, cfg) if err != nil { return err } @@ -151,7 +162,7 @@ var addNotionMeetingsCmd = &cobra.Command{ if err := runNotionMeetingsProbe(cmd.Context(), cmd.OutOrStdout(), client); err != nil { return err } - st, cleanup, err := openWritableStoreAndInitForIngest() + st, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) if err != nil { return err } @@ -160,7 +171,7 @@ var addNotionMeetingsCmd = &cobra.Command{ source.Identifier, accountEmail); err != nil { return err } - if err := runPostSourceCreateMigrations(st); err != nil { + if err := runPostSourceCreateMigrationsForInvocation(st, state); err != nil { return fmt.Errorf("post-source-create migrations: %w", err) } _, _ = fmt.Fprintf(cmd.OutOrStdout(), "\nNotion meeting source %s registered.\n", source.Identifier) @@ -180,10 +191,15 @@ a local visible-set filter. --limit caps discovery work but not due transcript maintenance. --probe validates access without printing meeting content.`, Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg if !isDaemonCLISubprocess() { return runDaemonCLICommandHTTPFromCobra(cmd, args) } - sources, err := resolveNotionMeetingsSources(args, syncNotionMeetingsProbe) + sources, err := resolveNotionMeetingsSources(args, syncNotionMeetingsProbe, cfg) if err != nil { return err } @@ -210,7 +226,7 @@ maintenance. --probe validates access without printing meeting content.`, newNotionMeetingsClient(notionmeetings.DefaultBaseURL, source.Token)) } - st, cleanup, err := openWritableStoreAndInitForIngest() + st, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) if err != nil { return err } @@ -230,12 +246,12 @@ maintenance. --probe validates access without printing meeting content.`, }) accumulateNotionMeetingsWrites(pendingWrites, summary) if err := finishNotionMeetingsImport(source.Identifier, pendingWrites, importErr, - func() error { return rebuildNotionMeetingsCacheAfterWrite(dbPath) }); err != nil { + func() error { return rebuildNotionMeetingsCacheAfterWrite(dbPath, state) }); err != nil { return err } writeNotionMeetingsSummary(cmd.OutOrStdout(), summary) } - return rebuildNotionMeetingsCacheAfterWrite(dbPath) + return rebuildNotionMeetingsCacheAfterWrite(dbPath, state) }, } @@ -311,5 +327,5 @@ func init() { syncNotionMeetingsCmd.Flags().BoolVar(&syncNotionMeetingsProbe, "probe", false, "validate capabilities and result shape without printing meeting content") rootCmd.AddCommand(addNotionMeetingsCmd) - rootCmd.AddCommand(syncNotionMeetingsCmd) + rootCmd.AddCommand(addManualSyncCacheFlags(syncNotionMeetingsCmd)) } diff --git a/cmd/msgvault/cmd/notion_meetings_test.go b/cmd/msgvault/cmd/notion_meetings_test.go index d96feb529..e630cc006 100644 --- a/cmd/msgvault/cmd/notion_meetings_test.go +++ b/cmd/msgvault/cmd/notion_meetings_test.go @@ -38,6 +38,8 @@ func (f fakeNotionProbe) ListUsers(context.Context, string) (*notionmeetings.Use } func TestResolveNotionMeetingsSource(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) previous := cfg @@ -46,17 +48,21 @@ func TestResolveNotionMeetingsSource(t *testing.T) { {Identifier: "personal", Token: "secret-1"}, {Identifier: "work", Token: "secret-2"}, }} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx - _, err := resolveNotionMeetingsSource(nil) + _, err := resolveNotionMeetingsSource(nil, cfg) require.Error(err) assert.Contains(err.Error(), "multiple [[notion_meetings]]") - source, err := resolveNotionMeetingsSource([]string{"work"}) + source, err := resolveNotionMeetingsSource([]string{"work"}, cfg) require.NoError(err) assert.Equal("work", source.Identifier) } func TestResolveNotionMeetingsSourcesRequiresProbeIdentifierForMultipleSources(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) previous := cfg @@ -65,12 +71,14 @@ func TestResolveNotionMeetingsSourcesRequiresProbeIdentifierForMultipleSources(t {Identifier: "personal", Token: "secret-1"}, {Identifier: "work", Token: "secret-2"}, }} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx - _, err := resolveNotionMeetingsSources(nil, true) + _, err := resolveNotionMeetingsSources(nil, true, cfg) require.Error(err) assert.Contains(err.Error(), "multiple [[notion_meetings]]") - sources, err := resolveNotionMeetingsSources([]string{"work"}, true) + sources, err := resolveNotionMeetingsSources([]string{"work"}, true, cfg) require.NoError(err) require.Len(sources, 1) assert.Equal("work", sources[0].Identifier) diff --git a/cmd/msgvault/cmd/organization_test.go b/cmd/msgvault/cmd/organization_test.go index 87d4c3cf0..0beff641c 100644 --- a/cmd/msgvault/cmd/organization_test.go +++ b/cmd/msgvault/cmd/organization_test.go @@ -2,6 +2,7 @@ package cmd import ( "bytes" + "context" "encoding/json" "fmt" "net/http" @@ -36,8 +37,8 @@ func TestOrganizationCreateSendsNormalizedBodyAndPrintsResult(t *testing.T) { assert.NoError(err) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) - output := runOrganizationCommand(t, organizationCreateCmd, []string{"Example Org", "--kind", "company", "--domain", "Example.com"}) + testCtx := withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) + output := runOrganizationCommand(testCtx, t, organizationCreateCmd, []string{"Example Org", "--kind", "company", "--domain", "Example.com"}) require.NoError(decodeErr) assert.Equal("Example Org", received.Name) assert.Equal("company", received.Kind) @@ -74,8 +75,8 @@ func TestOrganizationSetReadsCurrentRevisionAndSendsIfMatch(t *testing.T) { } })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) - output := runOrganizationCommand(t, organizationSetCmd, []string{"4", "--name", "Example Group"}) + testCtx := withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) + output := runOrganizationCommand(testCtx, t, organizationSetCmd, []string{"4", "--name", "Example Group"}) require.NoError(decodeErr) assert.Equal(int32(2), requests.Load()) assert.Equal(`"organization-4-r3"`, ifMatch) @@ -122,11 +123,11 @@ func TestOrganizationLifecycleCommandsPreserveRootFields(t *testing.T) { } })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) - runOrganizationCommand(t, test.command, []string{"4"}) + runOrganizationCommand(testCtx, t, test.command, []string{"4"}) require.NoError(decodeErr) assert.Equal("example.com", body["primary_domain"]) assert.Equal("Synthetic description.", body["description"]) @@ -150,8 +151,9 @@ func TestOrganizationDeleteReportsEmploymentConflict(t *testing.T) { assert.NoError(err) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) + testCtx := withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) command := cloneOrganizationCommand(organizationDeleteCmd) + command.SetContext(testCtx) var output bytes.Buffer command.SetOut(&output) command.SetErr(&output) @@ -173,8 +175,8 @@ func TestOrganizationListSendsQueryParametersAndRendersTable(t *testing.T) { assert.NoError(err) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) - output := runOrganizationCommand(t, organizationListCmd, []string{"--limit", "25", "--query", "example", "--include-retired"}) + testCtx := withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) + output := runOrganizationCommand(testCtx, t, organizationListCmd, []string{"--limit", "25", "--query", "example", "--include-retired"}) assert.Contains(rawQuery, "limit=25") assert.Contains(rawQuery, "q=example") assert.Contains(rawQuery, "include_retired=true") @@ -193,8 +195,8 @@ func TestOrganizationShowHistoryPrintsSupersededRows(t *testing.T) { assert.NoError(err) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) - output := runOrganizationCommand(t, organizationShowCmd, []string{"4", "--history"}) + testCtx := withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) + output := runOrganizationCommand(testCtx, t, organizationShowCmd, []string{"4", "--history"}) assert.Contains(output, "Earlier Org") assert.Contains(output, "active until 2026-07-30T12:01:00Z") } @@ -216,11 +218,10 @@ func TestOrganizationAttributeSetTransmitsOptionalExpectedValueID(t *testing.T) assert.NoError(err) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) - - output := runOrganizationCommand(t, organizationAttributeSetCmd, []string{ + output := runOrganizationCommand(testCtx, t, organizationAttributeSetCmd, []string{ "4", "--definition", "industry_focus", "--text", "information retrieval", "--expected-value-id", "7", "--ordinal", "2", }) @@ -252,11 +253,10 @@ func TestOrganizationAttributeClearForwardsOrdinalExpectedValueIDAndDryRun(t *te assert.NoError(err) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) - - output := runOrganizationCommand(t, organizationAttributeClearCmd, []string{ + output := runOrganizationCommand(testCtx, t, organizationAttributeClearCmd, []string{ "4", "industry_focus", "--ordinal", "2", "--expected-value-id", "8", "--dry-run", }) require.NotEmpty(output) @@ -272,7 +272,7 @@ func cloneOrganizationCommand(template *cobra.Command) *cobra.Command { }) return command } -func runOrganizationCommand(t *testing.T, template *cobra.Command, args []string) string { +func runOrganizationCommand(ctx context.Context, t *testing.T, template *cobra.Command, args []string) string { t.Helper() saved := organizationJSON organizationJSON = false @@ -282,6 +282,7 @@ func runOrganizationCommand(t *testing.T, template *cobra.Command, args []string command.SetOut(&output) command.SetErr(&output) command.SetArgs(args) + command.SetContext(ctx) require.NoError(t, command.Execute()) return output.String() } diff --git a/cmd/msgvault/cmd/output.go b/cmd/msgvault/cmd/output.go index 58973d438..072d86a70 100644 --- a/cmd/msgvault/cmd/output.go +++ b/cmd/msgvault/cmd/output.go @@ -4,6 +4,7 @@ import ( "encoding/json/jsontext" "encoding/json/v2" "fmt" + "io" "os" "strings" "text/tabwriter" @@ -138,7 +139,10 @@ func formatSize(bytes int64) string { } func printJSON(v any) error { - enc := jsontext.NewEncoder(os.Stdout, jsontext.WithIndentPrefix(""), jsontext.WithIndent(" ")) + return printJSONTo(os.Stdout, v) +} - return json.MarshalEncode(enc, v, json.Deterministic(true)) +func printJSONTo(w io.Writer, value any) error { + encoder := jsontext.NewEncoder(w, jsontext.WithIndentPrefix(""), jsontext.WithIndent(" ")) + return json.MarshalEncode(encoder, value, json.Deterministic(true)) } diff --git a/cmd/msgvault/cmd/owner_attribution_parity_test.go b/cmd/msgvault/cmd/owner_attribution_parity_test.go new file mode 100644 index 000000000..bb517866c --- /dev/null +++ b/cmd/msgvault/cmd/owner_attribution_parity_test.go @@ -0,0 +1,158 @@ +package cmd + +import ( + "database/sql" + "fmt" + "path/filepath" + "slices" + "testing" + "time" + + _ "github.com/duckdb/duckdb-go/v2" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/store" +) + +// TestOwnerAttributionParityStoreVersusCache pins the DuckDB owner mirror in +// build_cache.go to the store's attribution SQL: every message's is_from_me +// and every source's owner participants must agree row for row. +func TestOwnerAttributionParityStoreVersusCache(t *testing.T) { + require := require.New(t) + tmp := t.TempDir() + dbPath := filepath.Join(tmp, "msgvault.db") + analyticsDir := filepath.Join(tmp, "analytics") + st, err := store.Open(dbPath) + require.NoError(err) + t.Cleanup(func() { _ = st.Close() }) + require.NoError(st.InitSchema()) + + sourceA, err := st.GetOrCreateSource("gmail", "alice@example.com") + require.NoError(err) + sourceB, err := st.GetOrCreateSource("whatsapp", "+15550100") + require.NoError(err) + identities := map[int64][]string{ + sourceA.ID: {"alice@example.com", "alias@example.com", "carol@example.com"}, + sourceB.ID: {"+15550100", "alice@example.com"}, + } + for source, addresses := range identities { + for _, address := range addresses { + require.NoError(st.AddAccountIdentity(source, address, "manual")) + } + } + + caseSender, err := st.EnsureParticipant("Alice@Example.COM", "Alice", "example.com") + require.NoError(err) + aliasOnly, err := st.EnsureParticipantByPhone("+15550111", "Alias", "whatsapp") + require.NoError(err) + require.NoError(st.SetParticipantIdentifier(aliasOnly, "email", "Alias@Example.com")) + guarded, err := st.EnsureParticipant("bob@example.com", "Bob", "example.com") + require.NoError(err) + require.NoError(st.SetParticipantIdentifier(guarded, "email", "carol@example.com")) + phone, err := st.EnsureParticipantByPhone("+15550100", "Phone", "whatsapp") + require.NoError(err) + + sentAt := time.Date(2025, 3, 1, 12, 0, 0, 0, time.UTC) + messageIDs := map[string]int64{} + addMessage := func(source int64, id string, sender int64, envelope ...store.RecipientSet) { + conv, err := st.EnsureConversation(source, "thread-"+id, "Thread") + require.NoError(err) + messageID, err := st.PersistMessage(&store.MessagePersistData{ + Message: &store.Message{ + ConversationID: conv, SourceID: source, SourceMessageID: id, MessageType: "email", + SenderID: sql.NullInt64{Int64: sender, Valid: true}, + SentAt: sql.NullTime{Time: sentAt, Valid: true}, + }, + Recipients: envelope, + }) + require.NoError(err) + messageIDs[id] = messageID + } + fromEnvelope := func(participant int64, address string) store.RecipientSet { + return store.RecipientSet{Type: "from", ParticipantIDs: []int64{participant}, + DisplayNames: []string{""}, EmailAddresses: []string{address}} + } + for _, sender := range []int64{caseSender, aliasOnly, guarded, phone} { + addMessage(sourceA.ID, fmt.Sprintf("a-fallback-%d", sender), sender) + addMessage(sourceB.ID, fmt.Sprintf("b-fallback-%d", sender), sender) + } + // Each envelope disagrees with its sender, so only the envelope branch explains the result. + addMessage(sourceA.ID, "a-envelope-disagrees", caseSender, fromEnvelope(guarded, "bob@example.com")) + addMessage(sourceA.ID, "a-envelope-owner", guarded, fromEnvelope(caseSender, "Alice@Example.COM")) + + _, err = buildCache(dbPath, analyticsDir, false) + require.NoError(err) + duckdb, err := sql.Open("duckdb", "") + require.NoError(err) + t.Cleanup(func() { _ = duckdb.Close() }) + + type attribution struct { + ID int64 + IsFromMe bool + } + readAttribution := func(db *sql.DB, query string, args ...any) []attribution { + rows, err := db.Query(query, args...) + require.NoError(err) + defer func() { _ = rows.Close() }() + var got []attribution + for rows.Next() { + var row attribution + require.NoError(rows.Scan(&row.ID, &row.IsFromMe)) + got = append(got, row) + } + require.NoError(rows.Err()) + return got + } + storeRows := readAttribution(st.DB(), `SELECT id, is_from_me FROM messages ORDER BY id`) + require.Len(storeRows, 10) + fromMe := 0 + for _, row := range storeRows { + if row.IsFromMe { + fromMe++ + } + } + envelopeFromMe := map[int64]bool{} + for _, row := range storeRows { + envelopeFromMe[row.ID] = row.IsFromMe + } + require.False(envelopeFromMe[messageIDs["a-envelope-disagrees"]], "a non-owner envelope overrides an owner sender") + require.True(envelopeFromMe[messageIDs["a-envelope-owner"]], "an owner envelope overrides a non-owner sender") + require.NotZero(fromMe, "the fixture must attribute some messages to the owner") + require.Less(fromMe, len(storeRows), "the fixture must leave some messages unattributed") + require.Equal(storeRows, readAttribution(duckdb, + `SELECT id, is_from_me FROM read_parquet(?, hive_partitioning=true) ORDER BY id`, + filepath.Join(analyticsDir, "messages", "**", "*.parquet"))) + + type owner struct{ Source, Participant int64 } + var storeOwners []owner + for source, addresses := range identities { + for _, address := range addresses { + resolved, err := st.ResolveAccountIdentityContext(t.Context(), source, address) + require.NoError(err) + for _, participant := range resolved.ParticipantIDs { + if !slices.Contains(storeOwners, owner{source, participant}) { + storeOwners = append(storeOwners, owner{source, participant}) + } + } + } + } + slices.SortFunc(storeOwners, func(a, b owner) int { + if a.Source != b.Source { + return int(a.Source - b.Source) + } + return int(a.Participant - b.Participant) + }) + rows, err := duckdb.Query(`SELECT source_id, participant_id FROM read_parquet(?) + GROUP BY source_id, participant_id ORDER BY source_id, participant_id`, + filepath.Join(analyticsDir, "owner_participants", "*.parquet")) + require.NoError(err) + defer func() { _ = rows.Close() }() + var cacheOwners []owner + for rows.Next() { + var row owner + require.NoError(rows.Scan(&row.Source, &row.Participant)) + cacheOwners = append(cacheOwners, row) + } + require.NoError(rows.Err()) + require.NotEmpty(storeOwners) + require.Equal(storeOwners, cacheOwners) +} diff --git a/cmd/msgvault/cmd/pack_attachments.go b/cmd/msgvault/cmd/pack_attachments.go index ee1aac75a..7ec921082 100644 --- a/cmd/msgvault/cmd/pack_attachments.go +++ b/cmd/msgvault/cmd/pack_attachments.go @@ -1,6 +1,7 @@ package cmd import ( + "errors" "fmt" "io" @@ -33,7 +34,12 @@ To go back to loose files (e.g., before downgrading msgvault), run } func runPackAttachmentsLocal(cmd *cobra.Command) error { - s, cleanup, err := openWritableStoreAndInit() + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + s, cleanup, err := openWritableStoreAndInitForInvocation(state) if err != nil { return err } diff --git a/cmd/msgvault/cmd/pack_attachments_test.go b/cmd/msgvault/cmd/pack_attachments_test.go index 7dc38a759..d965d45fa 100644 --- a/cmd/msgvault/cmd/pack_attachments_test.go +++ b/cmd/msgvault/cmd/pack_attachments_test.go @@ -21,7 +21,8 @@ func TestPackAttachmentsProxiesThroughDaemonCLIRunner(t *testing.T) { `{"type":"stdout","data":"Packed 0 blob(s) (0B) into 0 pack(s).\n"}`, `{"type":"complete"}`, ) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx var stdout bytes.Buffer cmd := &cobra.Command{ @@ -29,6 +30,7 @@ func TestPackAttachmentsProxiesThroughDaemonCLIRunner(t *testing.T) { Args: packAttachmentsCmd.Args, RunE: packAttachmentsCmd.RunE, } + cmd.SetContext(testCtx) cmd.SetOut(&stdout) require.NoError(cmd.Execute(), "pack-attachments") diff --git a/cmd/msgvault/cmd/packed_repack_integration_test.go b/cmd/msgvault/cmd/packed_repack_integration_test.go index b1e3ffb33..6dbb4ccd4 100644 --- a/cmd/msgvault/cmd/packed_repack_integration_test.go +++ b/cmd/msgvault/cmd/packed_repack_integration_test.go @@ -157,11 +157,11 @@ func assertBlobReadSurfaces( URL: httpServer.URL, AllowInsecure: true, }) require.NoError(err) - configureRemoteDaemonForTest(t, httpServer.URL) - mcpOpts, err := daemonMCPServeOptions(context.Background(), client) - require.NoError(err) + testCtx := configureRemoteDaemonForTest(t, httpServer.URL) + _ = testCtx + mcpOpts := daemonMCPServeOptions(testCtx, client, invocationFromContext(testCtx)) require.NotNil(mcpOpts.AttachmentReader) - mcpData, err := mcpOpts.AttachmentReader.ReadAttachment(context.Background(), hash) + mcpData, err := mcpOpts.AttachmentReader.ReadAttachment(testCtx, hash) require.NoError(err) assert.Equal(content, mcpData, "daemon-backed MCP attachment reader follows packed authority") @@ -177,14 +177,15 @@ func assertBlobReadSurfaces( exportAttachmentBase64 = false exportAttachmentOutput = filepath.Join(t.TempDir(), "single-export.bin") cmd := &cobra.Command{Use: "export-attachment"} - cmd.SetContext(context.Background()) + cmd.SetContext(testCtx) + cmd.SetContext(testCtx) require.NoError(runExportAttachmentHTTP(cmd, hash)) single, err := os.ReadFile(exportAttachmentOutput) require.NoError(err) assert.Equal(content, single) directory := t.TempDir() - dirResult := exportAttachmentsFromHTTP(context.Background(), client, directory, + dirResult := exportAttachmentsFromHTTP(testCtx, client, directory, []query.AttachmentInfo{{ Filename: filename, ContentHash: hash, Size: int64(len(content)), }}) diff --git a/cmd/msgvault/cmd/person.go b/cmd/msgvault/cmd/person.go index af6022dc9..af2359611 100644 --- a/cmd/msgvault/cmd/person.go +++ b/cmd/msgvault/cmd/person.go @@ -680,6 +680,8 @@ func init() { personCmd.AddCommand(newPersonSweepCommand(defaultPersonSweepCommandDeps())) personCmd.AddCommand(newPersonEnrichmentCommand(defaultPersonEnrichmentCommandDeps())) personCmd.AddCommand(newPersonBriefCommand()) + personCmd.AddCommand(newPersonAgendaCommand()) + personCmd.AddCommand(newPersonScoringCommand()) personCmd.AddCommand(personPromoteCmd, personGetCmd, personListCmd, personSetDisplayNameCmd, personDeleteCmd, personTrackCmd, personUntrackCmd, personMergeCmd, personSplitCmd, personMergeHistoryCmd, personMergeShowCmd, diff --git a/cmd/msgvault/cmd/person_agenda.go b/cmd/msgvault/cmd/person_agenda.go new file mode 100644 index 000000000..1193a3cf0 --- /dev/null +++ b/cmd/msgvault/cmd/person_agenda.go @@ -0,0 +1,251 @@ +package cmd + +import ( + "encoding/json/jsontext" + "encoding/json/v2" + "errors" + "fmt" + "strconv" + "strings" + "text/tabwriter" + + "github.com/google/uuid" + "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/daemonclient" + "go.kenn.io/msgvault/internal/textutil" + "go.kenn.io/msgvault/pkg/client/generated" +) + +func newPersonAgendaCommand() *cobra.Command { + var jsonOutput bool + agenda := &cobra.Command{ + Use: "agenda", + Short: "Manage a person's live Kata agenda", + } + agenda.PersistentFlags().BoolVar(&jsonOutput, flagJSON, false, "Output as JSON") + + list := &cobra.Command{ + Use: "list ", + Short: "List open Kata items linked to a person", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + personID, err := positivePersonCLIArg(cmd, args[0], personValue) + if err != nil { + return err + } + client, closeClient, err := openPersonAgendaClient(cmd) + if err != nil { + return err + } + defer closeClient() + result, err := client.ListPersonAgenda(cmd.Context(), personID) + if err != nil { + return err + } + if jsonOutput { + return writePersonAgendaJSON(cmd, result) + } + writer := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 2, ' ', 0) + _, _ = fmt.Fprintln(writer, "LIST\tREF\tSTATE\tPRIORITY\tTITLE") + for _, item := range result.Items { + priority := "-" + if item.Priority != nil { + priority = strconv.FormatInt(*item.Priority, 10) + } + _, _ = fmt.Fprintf(writer, "%s\t%s\t%s\t%s\t%s\n", + textutil.SanitizeTerminal(item.List), textutil.SanitizeTerminal(item.Ref), + textutil.SanitizeTerminal(item.State), priority, textutil.SanitizeTerminal(item.Title)) + } + if err := writer.Flush(); err != nil { + return fmt.Errorf("write agenda: %w", err) + } + if result.Truncated { + _, err = fmt.Fprintln(cmd.ErrOrStderr(), "More open tasks are linked to this person. View the full list in Kata.") + } + if err != nil { + return fmt.Errorf("write agenda notice: %w", err) + } + return nil + }, + } + + var createTitle, createBody, createList, idempotencyKey string + var createPriority int64 + var createLabels []string + create := &cobra.Command{ + Use: "create ", + Short: "Create and link a Kata item", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + personID, err := positivePersonCLIArg(cmd, args[0], personValue) + if err != nil { + return err + } + if strings.TrimSpace(createTitle) == "" { + return usageErr(cmd, errors.New("--title is required")) + } + request := generated.PersonAgendaCreateRequest{Title: createTitle, Labels: createLabels} + if cmd.Flags().Changed("body") { + request.Body = &createBody + } + if cmd.Flags().Changed("list") { + request.List = &createList + } + if cmd.Flags().Changed("priority") { + if err := validPersonAgendaPriority(createPriority); err != nil { + return usageErr(cmd, err) + } + request.Priority = &createPriority + } + client, closeClient, err := openPersonAgendaClient(cmd) + if err != nil { + return err + } + defer closeClient() + key := strings.TrimSpace(idempotencyKey) + if key == "" { + id, err := uuid.NewRandom() + if err != nil { + return fmt.Errorf("generate retry key: %w", err) + } + key = id.String() + if _, err := fmt.Fprintf(cmd.ErrOrStderr(), "Idempotency key for retries: %s\n", key); err != nil { + return fmt.Errorf("write retry key: %w", err) + } + } + item, err := client.CreatePersonAgendaItem(cmd.Context(), personID, key, request) + if err != nil { + return err + } + return writePersonAgendaItem(cmd, item, jsonOutput) + }, + } + create.Flags().StringVar(&createTitle, "title", "", "Task title") + create.Flags().StringVar(&createBody, "body", "", "Task body") + create.Flags().StringVar(&createList, "list", "", "Virtual list") + create.Flags().Int64Var(&createPriority, "priority", 0, "Kata priority from 0 through 4") + create.Flags().StringSliceVar(&createLabels, "label", nil, "Kata label (repeatable)") + create.Flags().StringVar(&idempotencyKey, "idempotency-key", "", "Retry key (generated when omitted; reuse for retries)") + + var linkList string + link := &cobra.Command{ + Use: "link ", + Short: "Link an existing Kata item to a person", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + personID, ref, err := personAgendaTarget(cmd, args) + if err != nil { + return err + } + request := generated.PersonAgendaLinkRequest{Ref: ref} + if cmd.Flags().Changed("list") { + request.List = &linkList + } + client, closeClient, err := openPersonAgendaClient(cmd) + if err != nil { + return err + } + defer closeClient() + item, err := client.LinkPersonAgendaItem(cmd.Context(), personID, request) + if err != nil { + return err + } + return writePersonAgendaItem(cmd, item, jsonOutput) + }, + } + link.Flags().StringVar(&linkList, "list", "", "Virtual list") + + var editList string + edit := &cobra.Command{ + Use: "edit ", + Short: "Move a linked Kata item to another list", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + personID, ref, err := personAgendaTarget(cmd, args) + if err != nil { + return err + } + if !cmd.Flags().Changed("list") { + return usageErr(cmd, errors.New("--list is required; edit task content in Kata")) + } + client, closeClient, err := openPersonAgendaClient(cmd) + if err != nil { + return err + } + defer closeClient() + item, err := client.UpdatePersonAgendaItem(cmd.Context(), personID, ref, generated.PersonAgendaUpdateRequest{List: editList}) + if err != nil { + return err + } + return writePersonAgendaItem(cmd, item, jsonOutput) + }, + } + edit.Flags().StringVar(&editList, "list", "", "Replacement virtual list") + + unlink := &cobra.Command{ + Use: "unlink ", + Short: "Unlink a Kata item without deleting it", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + personID, ref, err := personAgendaTarget(cmd, args) + if err != nil { + return err + } + client, closeClient, err := openPersonAgendaClient(cmd) + if err != nil { + return err + } + defer closeClient() + item, err := client.UnlinkPersonAgendaItem(cmd.Context(), personID, ref) + if err != nil { + return err + } + return writePersonAgendaItem(cmd, item, jsonOutput) + }, + } + + agenda.AddCommand(list, create, link, edit, unlink) + return agenda +} + +func personAgendaTarget(cmd *cobra.Command, args []string) (int64, string, error) { + personID, err := positivePersonCLIArg(cmd, args[0], personValue) + if err != nil { + return 0, "", err + } + ref := strings.TrimSpace(args[1]) + if ref == "" { + return 0, "", usageErr(cmd, errors.New("kata ref must not be blank")) + } + return personID, ref, nil +} + +func validPersonAgendaPriority(priority int64) error { + if priority < 0 || priority > 4 { + return errors.New("priority must be from 0 through 4") + } + return nil +} + +func openPersonAgendaClient(cmd *cobra.Command) (*daemonclient.Client, func(), error) { + client, _, err := OpenHTTPStore(cmd.Context()) + if err != nil { + return nil, nil, err + } + return client, func() { _ = client.Close() }, nil +} + +func writePersonAgendaJSON(cmd *cobra.Command, value any) error { + return json.MarshalEncode(jsontext.NewEncoder(cmd.OutOrStdout()), value, json.Deterministic(true)) +} + +func writePersonAgendaItem(cmd *cobra.Command, item generated.PersonAgendaItem, jsonOutput bool) error { + if jsonOutput { + return writePersonAgendaJSON(cmd, item) + } + _, err := fmt.Fprintf(cmd.OutOrStdout(), "%s\t%s\n", textutil.SanitizeTerminal(item.Ref), textutil.SanitizeTerminal(item.Title)) + if err != nil { + return fmt.Errorf("write person agenda item: %w", err) + } + return nil +} diff --git a/cmd/msgvault/cmd/person_agenda_api_e2e_test.go b/cmd/msgvault/cmd/person_agenda_api_e2e_test.go new file mode 100644 index 000000000..38d620727 --- /dev/null +++ b/cmd/msgvault/cmd/person_agenda_api_e2e_test.go @@ -0,0 +1,96 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/testutil" +) + +// nativeKataStub serves the endpoints taskclient.ConnectKata probes for a native Kata task service, and answers person-filtered agenda +// listing with an empty issue page. +func nativeKataStub(t *testing.T) *httptest.Server { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/api/v1/capabilities": + http.NotFound(w, r) + case "/api/v1/health": + assert.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "ok": true, "schema_version": 28, "version": "v0.18.0", "uptime": "1s", + "started_at": "2026-09-22T00:00:00Z", "api_schema_version": "0.21.0", + })) + case "/api/v1/instance": + assert.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "instance_uid": "instance-1", "version": "0.18.0", + "auth": map[string]any{}, "web_ui_capabilities": map[string]any{}, + })) + case "/api/v1/projects": + assert.NoError(t, json.NewEncoder(w).Encode(map[string]any{"projects": []map[string]any{{ + "id": 1, "uid": "project-uid", "name": "msgvault", "active": true, "revision": 4, + "created_at": "2026-09-22T00:00:00Z", "metadata": map[string]any{}, + }}})) + case "/api/v1/projects/1/issues": + assert.True(t, strings.HasPrefix(r.URL.Query().Get("meta"), "msgvault.person=")) + assert.Equal(t, "open", r.URL.Query().Get("status")) + assert.NoError(t, json.NewEncoder(w).Encode(map[string]any{"issues": []any{}})) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(server.Close) + return server +} + +// TestPersonAgendaEndpointServesThroughTheProductionAdapter pins that the live +// agenda route is reachable in the daemon. It builds the server exactly as +// serve.go does -- Store: &storeAPIAdapter{...} -- and asserts the agenda list +// answers over HTTP. When the adapter lacked ListPersonUIDsContext, the backend +// initialized as nil and every agenda endpoint answered 503 +// task_integration_unavailable no matter how the task service was configured. +func TestPersonAgendaEndpointServesThroughTheProductionAdapter(t *testing.T) { + require := require.New(t) + + st := testutil.NewTestStore(t) + participantID, err := st.EnsureParticipantByIdentifier( + "email", "agenda-adapter@example.test", "Agenda Adapter") + require.NoError(err, "create participant") + person, _, err := st.CreatePersonFromParticipant(participantID) + require.NoError(err, "create person") + + kata := nativeKataStub(t) + srv := api.NewServerWithOptions(api.ServerOptions{ + Config: &config.Config{Integrations: config.IntegrationsConfig{Kata: config.TaskIntegrationConfig{ + Enabled: true, Endpoint: kata.URL, APIKey: "secret", DefaultProject: "msgvault", + }}}, + Store: &storeAPIAdapter{store: st}, + Logger: slog.New(slog.DiscardHandler), + }) + httpSrv := httptest.NewServer(srv.Router()) + t.Cleanup(httpSrv.Close) + + response, err := http.Get( + fmt.Sprintf("%s/api/v1/people/%d/agenda", httpSrv.URL, person.ID)) + require.NoError(err, "GET /people/{id}/agenda") + defer func() { _ = response.Body.Close() }() + body, err := io.ReadAll(response.Body) + require.NoError(err, "read agenda response") + require.Equalf(http.StatusOK, response.StatusCode, + "the daemon's own adapter must back the agenda route, not report it unavailable: %s %s", + response.Status, body) + var result api.PersonAgendaResult + require.NoError(json.Unmarshal(body, &result), "decode agenda result") + assert.Equal(t, "msgvault", result.Project) + assert.Empty(t, result.Items) +} diff --git a/cmd/msgvault/cmd/person_agenda_test.go b/cmd/msgvault/cmd/person_agenda_test.go new file mode 100644 index 000000000..2efbb9b1a --- /dev/null +++ b/cmd/msgvault/cmd/person_agenda_test.go @@ -0,0 +1,120 @@ +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/config" +) + +func TestPersonAgendaCommandsUseDaemonKataContract(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + type request struct { + method string + path string + idempotencyKey string + body map[string]any + } + requests := make([]request, 0, 5) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + entry := request{method: r.Method, path: r.URL.Path, idempotencyKey: r.Header.Get("Idempotency-Key")} + if r.Body != nil && r.Method != http.MethodGet && r.Method != http.MethodDelete { + assert.NoError(json.NewDecoder(r.Body).Decode(&entry.body)) + } + requests = append(requests, entry) + w.Header().Set("Content-Type", "application/json") + if r.Method == http.MethodGet { + assert.NoError(json.NewEncoder(w).Encode(map[string]any{"project": "msgvault", "items": []any{agendaCLIItem("task-1", "Ask")}})) + return + } + if r.Method == http.MethodPost { + w.WriteHeader(http.StatusCreated) + } + assert.NoError(json.NewEncoder(w).Encode(map[string]any{"item": agendaCLIItem("task-1", "Ask")})) + })) + t.Cleanup(server.Close) + testCtx := withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) + + output, err := executePersonAgendaCommand(testCtx, t, "list", "7", "--json") + require.NoError(err) + assert.Contains(output, `"project":"msgvault"`) + + output, err = executePersonAgendaCommand(testCtx, t, "create", "7", "--title", "Ask", "--body", "Context", "--priority", "0", "--idempotency-key", "retry-1", "--json") + require.NoError(err) + assert.Contains(output, `"ref":"task-1"`) + + _, err = executePersonAgendaCommand(testCtx, t, "link", "7", "task-1", "--list", "gift ideas") + require.NoError(err) + _, err = executePersonAgendaCommand(testCtx, t, "edit", "7", "task-1", "--list", "gift ideas") + require.NoError(err) + _, err = executePersonAgendaCommand(testCtx, t, "unlink", "7", "task-1") + require.NoError(err) + + require.Len(requests, 5) + assert.Equal(request{method: http.MethodGet, path: "/api/v1/people/7/agenda"}, requests[0]) + assert.Equal("retry-1", requests[1].idempotencyKey) + assert.Equal(map[string]any{"title": "Ask", "body": "Context", "priority": float64(0)}, requests[1].body) + assert.Equal("/api/v1/people/7/agenda/links", requests[2].path) + assert.Equal(map[string]any{"ref": "task-1", "list": "gift ideas"}, requests[2].body) + assert.Equal(http.MethodPatch, requests[3].method) + assert.Equal(map[string]any{"list": "gift ideas"}, requests[3].body) + assert.Equal(http.MethodDelete, requests[4].method) +} + +func TestPersonAgendaCreateGeneratesRetryKey(t *testing.T) { + var key string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + key = r.Header.Get("Idempotency-Key") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + assert.NoError(t, json.NewEncoder(w).Encode(map[string]any{"item": agendaCLIItem("task-1", "Ask")})) + })) + t.Cleanup(server.Close) + testCtx := withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) + output, err := executePersonAgendaCommand(testCtx, t, "create", "7", "--title", "Ask") + require.NoError(t, err) + require.NotEmpty(t, key) + assert.Contains(t, output, key, "the generated key must be available for a retry") +} + +func executePersonAgendaCommand(ctx context.Context, t *testing.T, args ...string) (string, error) { + t.Helper() + command := newPersonAgendaCommand() + var output bytes.Buffer + command.SetOut(&output) + command.SetErr(&output) + command.SetArgs(args) + command.SetContext(ctx) + err := command.Execute() + return output.String(), err +} + +func agendaCLIItem(ref, title string) map[string]any { + return map[string]any{ + "uid": "01TASK", "ref": ref, "qualified_ref": "msgvault#" + ref, "project": "msgvault", + "title": title, "revision": "1", "list": "agenda", "status": "open", "state": "open", + } +} + +func TestPersonAgendaListTextShowsVirtualHierarchy(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, err := w.Write([]byte(`{"project":"msgvault","truncated":true,"items":[{"uid":"01TASK","ref":"task-1","qualified_ref":"msgvault#task-1","project":"msgvault","title":"Ask","revision":"1","list":"gift ideas","status":"open","state":"open"}]}`)) + assert.NoError(t, err) + })) + t.Cleanup(server.Close) + testCtx := withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) + + output, err := executePersonAgendaCommand(testCtx, t, "list", "7") + require.NoError(t, err) + assert.True(t, strings.Contains(output, "gift ideas") && strings.Contains(output, "task-1") && strings.Contains(output, "Ask"), output) + assert.Contains(t, output, "More open tasks are linked to this person.") +} diff --git a/cmd/msgvault/cmd/person_attributes_test.go b/cmd/msgvault/cmd/person_attributes_test.go index d202aa96c..cb03fd1c9 100644 --- a/cmd/msgvault/cmd/person_attributes_test.go +++ b/cmd/msgvault/cmd/person_attributes_test.go @@ -110,11 +110,12 @@ func TestPersonAttributesSetCoercesScalarAndForwardsMetadata(t *testing.T) { } })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) + _ = testCtx - output, err := runAttributeCommand(t, personAttributesSetCmd, + output, err := runAttributeCommand(testCtx, t, personAttributesSetCmd, "7", "primary_channel", "--value", "chat", "--source", "extraction", "--source-ref", "message:1234", "--confidence", "0.62", "--actor", "extractor", @@ -142,13 +143,14 @@ func TestPersonAttributesSetRejectsExplicitNonPositiveExpectedValueID(t *testing w.WriteHeader(http.StatusInternalServerError) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) + _ = testCtx for _, expectedID := range []string{"0", "-1"} { t.Run(expectedID, func(t *testing.T) { - _, err := runAttributeCommand(t, personAttributesSetCmd, + _, err := runAttributeCommand(testCtx, t, personAttributesSetCmd, "7", "primary_channel", "--value", "chat", "--expected-value-id", expectedID) require.Error(err) @@ -179,11 +181,12 @@ func TestPersonAttributesClearForwardsOrdinalAndExpectedValueID(t *testing.T) { assert.NoError(err) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) + _ = testCtx - output, err := runAttributeCommand(t, personAttributesClearCmd, + output, err := runAttributeCommand(testCtx, t, personAttributesClearCmd, "7", "ask_me_about", "--ordinal", "1", "--expected-value-id", "11") require.NoError(err) assert.Contains(query, "ordinal=1") diff --git a/cmd/msgvault/cmd/person_brief_test.go b/cmd/msgvault/cmd/person_brief_test.go index d58fa7ee3..bde358efd 100644 --- a/cmd/msgvault/cmd/person_brief_test.go +++ b/cmd/msgvault/cmd/person_brief_test.go @@ -2,6 +2,7 @@ package cmd import ( "bytes" + "context" "encoding/json" "io" "net/http" @@ -20,6 +21,7 @@ import ( // wire shapes the daemon serves. type personBriefTestDaemon struct { t *testing.T + ctx context.Context requests atomic.Int32 method string path string @@ -46,13 +48,14 @@ func newPersonBriefTestDaemon(t *testing.T, response string) *personBriefTestDae _, _ = w.Write([]byte(daemon.response)) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) + daemon.ctx = testCtx return daemon } -func runPersonBriefCommand(t *testing.T, args ...string) (string, error) { +func runPersonBriefCommand(ctx context.Context, t *testing.T, args ...string) (string, error) { t.Helper() root := &cobra.Command{Use: "msgvault"} person := &cobra.Command{Use: personValue} @@ -62,7 +65,7 @@ func runPersonBriefCommand(t *testing.T, args ...string) (string, error) { root.SetOut(&output) root.SetErr(&output) root.SetArgs(append([]string{personValue, "brief"}, args...)) - err := root.Execute() + err := root.ExecuteContext(ctx) return output.String(), err } @@ -101,7 +104,7 @@ func TestPersonBriefShowPrintsParagraphVersionLineAndEvidenceDates(t *testing.T) requirements := require.New(t) daemon := newPersonBriefTestDaemon(t, personBriefCLIPayload) - output, err := runPersonBriefCommand(t, "show", "7") + output, err := runPersonBriefCommand(daemon.ctx, t, "show", "7") requirements.NoError(err) assertions.Equal(http.MethodGet, daemon.method) assertions.Equal("/api/v1/people/7/brief", daemon.path) @@ -117,9 +120,9 @@ func TestPersonBriefShowPrintsParagraphVersionLineAndEvidenceDates(t *testing.T) func TestPersonBriefShowJSONPassesTheDaemonBodyThrough(t *testing.T) { assertions := assert.New(t) requirements := require.New(t) - newPersonBriefTestDaemon(t, personBriefCLIPayload) + daemon := newPersonBriefTestDaemon(t, personBriefCLIPayload) - output, err := runPersonBriefCommand(t, "show", "7", "--json") + output, err := runPersonBriefCommand(daemon.ctx, t, "show", "7", "--json") requirements.NoError(err) var decoded map[string]any requirements.NoError(json.Unmarshal([]byte(output), &decoded)) @@ -134,7 +137,7 @@ func TestPersonBriefShowReportsAMissingVersion(t *testing.T) { `{"error":"person_brief_not_found","message":"Person brief not found"}`) daemon.status = http.StatusNotFound - _, err := runPersonBriefCommand(t, "show", "7") + _, err := runPersonBriefCommand(daemon.ctx, t, "show", "7") require.Error(t, err) assert.ErrorContains(t, err, "Person brief not found") } @@ -154,7 +157,7 @@ func TestPersonBriefHistoryPrintsOneLinePerVersion(t *testing.T) { "program_version":"v1","provider":"openai_chat","model":"gpt-test", "rejected_at":null,"rejected_reason":"","superseded_at":"2026-08-29T18:42:10Z"}]}`) - output, err := runPersonBriefCommand(t, "history", "7") + output, err := runPersonBriefCommand(daemon.ctx, t, "history", "7") requirements.NoError(err) assertions.Equal("/api/v1/people/7/brief/versions", daemon.path) lines := strings.Split(strings.TrimSpace(output), "\n") @@ -169,7 +172,7 @@ func TestPersonBriefHistoryPrintsOneLinePerVersion(t *testing.T) { func TestPersonBriefHistoryHonorsLimit(t *testing.T) { daemon := newPersonBriefTestDaemon(t, `{"versions":[]}`) - output, err := runPersonBriefCommand(t, "history", "7", "--limit", "5") + output, err := runPersonBriefCommand(daemon.ctx, t, "history", "7", "--limit", "5") require.NoError(t, err) assert.Equal(t, "/api/v1/people/7/brief/versions?limit=5", daemon.path) assert.Contains(t, output, "VERSION") @@ -186,7 +189,7 @@ func TestPersonBriefRejectSendsTheReason(t *testing.T) { "rejected_at":"2026-08-30T09:00:00Z","rejected_reason":"wrong thread", "superseded_at":null}`) - output, err := runPersonBriefCommand(t, "reject", "7", "--reason", "wrong thread") + output, err := runPersonBriefCommand(daemon.ctx, t, "reject", "7", "--reason", "wrong thread") requirements.NoError(err) assertions.Equal(http.MethodPost, daemon.method) assertions.Equal("/api/v1/people/7/brief/reject", daemon.path) @@ -202,7 +205,7 @@ func TestPersonBriefGenerateReportsTheRunAndWarnsAboutSpend(t *testing.T) { `{"run_id":"run-1","attempt_id":"attempt-1","brief_version":3, "brief_failure_class":""}`) - output, err := runPersonBriefCommand(t, "generate", "7") + output, err := runPersonBriefCommand(daemon.ctx, t, "generate", "7") requirements.NoError(err) assertions.Equal(http.MethodPost, daemon.method) assertions.Equal("/api/v1/people/7/brief/generate", daemon.path) @@ -210,18 +213,18 @@ func TestPersonBriefGenerateReportsTheRunAndWarnsAboutSpend(t *testing.T) { assertions.Contains(output, "attempt-1") assertions.Contains(output, "version 3") - help, err := runPersonBriefCommand(t, "generate", "--help") + help, err := runPersonBriefCommand(daemon.ctx, t, "generate", "--help") requirements.NoError(err) assertions.Contains(help, "extraction page") assertions.Contains(help, "budget") } func TestPersonBriefGenerateReportsADeferredBrief(t *testing.T) { - newPersonBriefTestDaemon(t, + daemon := newPersonBriefTestDaemon(t, `{"run_id":"run-2","attempt_id":"attempt-2","brief_version":0, "brief_failure_class":"budget"}`) - output, err := runPersonBriefCommand(t, "generate", "7") + output, err := runPersonBriefCommand(daemon.ctx, t, "generate", "7") require.NoError(t, err) assert.Contains(t, output, "no new version") assert.Contains(t, output, "budget") @@ -233,7 +236,7 @@ func TestPersonBriefEnrollAndUnenrollReplaceState(t *testing.T) { daemon := newPersonBriefTestDaemon(t, `{"person_id":7,"enrolled":true,"enabled_at":"2026-08-29T18:42:10Z","actor":"api"}`) - output, err := runPersonBriefCommand(t, "enroll", "7", "--track") + output, err := runPersonBriefCommand(daemon.ctx, t, "enroll", "7", "--track") requirements.NoError(err) assertions.Equal(http.MethodPut, daemon.method) assertions.Equal("/api/v1/people/7/brief-enrollment", daemon.path) @@ -241,17 +244,17 @@ func TestPersonBriefEnrollAndUnenrollReplaceState(t *testing.T) { assertions.Contains(output, "Person 7 brief: enrolled") daemon.response = `{"person_id":7,"enrolled":true,"enabled_at":"2026-08-29T18:42:10Z","actor":"api"}` - _, err = runPersonBriefCommand(t, "enroll", "7") + _, err = runPersonBriefCommand(daemon.ctx, t, "enroll", "7") requirements.NoError(err) assertions.JSONEq(`{"enrolled":true,"track":false}`, daemon.body) daemon.response = `{"person_id":7,"enrolled":false,"enabled_at":null,"actor":""}` - output, err = runPersonBriefCommand(t, "unenroll", "7") + output, err = runPersonBriefCommand(daemon.ctx, t, "unenroll", "7") requirements.NoError(err) assertions.JSONEq(`{"enrolled":false,"track":false}`, daemon.body) assertions.Contains(output, "Person 7 brief: not enrolled") - jsonOutput, err := runPersonBriefCommand(t, "unenroll", "7", "--json") + jsonOutput, err := runPersonBriefCommand(daemon.ctx, t, "unenroll", "7", "--json") requirements.NoError(err) assertions.JSONEq( `{"person_id":7,"enrolled":false,"enabled_at":null,"actor":""}`, jsonOutput) @@ -263,7 +266,7 @@ func TestPersonBriefEnrollReportsAnUntrackedPerson(t *testing.T) { "`msgvault person track 7`"+` first, or enroll with --track"}`) daemon.status = http.StatusConflict - _, err := runPersonBriefCommand(t, "enroll", "7") + _, err := runPersonBriefCommand(daemon.ctx, t, "enroll", "7") require.Error(t, err) assert.ErrorContains(t, err, "msgvault person track") } @@ -272,7 +275,7 @@ func TestPersonBriefRejectsInvalidPersonIDBeforeNetwork(t *testing.T) { daemon := newPersonBriefTestDaemon(t, `{}`) for _, name := range []string{"show", "history", "generate", "reject", "enroll", "unenroll"} { - _, err := runPersonBriefCommand(t, name, "0") + _, err := runPersonBriefCommand(daemon.ctx, t, name, "0") require.Error(t, err, name) require.ErrorContains(t, err, "positive integer", name) } diff --git a/cmd/msgvault/cmd/person_directory_test.go b/cmd/msgvault/cmd/person_directory_test.go index 206b5e1f1..b770019f5 100644 --- a/cmd/msgvault/cmd/person_directory_test.go +++ b/cmd/msgvault/cmd/person_directory_test.go @@ -41,7 +41,7 @@ func TestPersonDirectoryProductionCommandRegistrationAndFlags(t *testing.T) { require.Error(command.Args(command, []string{"unexpected"})) } -func personDirectoryTestResponse(t *testing.T, status int, payload string) <-chan *http.Request { +func personDirectoryTestResponse(t *testing.T, status int, payload string) (<-chan *http.Request, context.Context) { t.Helper() requests := make(chan *http.Request, 8) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -51,16 +51,26 @@ func personDirectoryTestResponse(t *testing.T, status int, payload string) <-cha _, _ = io.WriteString(w, payload) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) - return requests + testCtx := withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) + return requests, testCtx } func runPersonDirectoryCommand(ctx context.Context, t *testing.T, args ...string) (string, error) { t.Helper() root := &cobra.Command{Use: "msgvault", SilenceErrors: true, SilenceUsage: true} + root.PersistentPreRunE = func(cmd *cobra.Command, _ []string) error { + prepareInvocation(cmd) + return nil + } localFlag := rootCmd.PersistentFlags().Lookup("local") + savedValue := localFlag.Value.String() savedChanged := localFlag.Changed - t.Cleanup(func() { localFlag.Changed = savedChanged }) + _ = localFlag.Value.Set(localFlag.DefValue) + localFlag.Changed = false + t.Cleanup(func() { + _ = localFlag.Value.Set(savedValue) + localFlag.Changed = savedChanged + }) root.PersistentFlags().AddFlag(localFlag) person := &cobra.Command{Use: personValue} person.AddCommand(newPersonDirectoryCommand()) @@ -74,6 +84,9 @@ func runPersonDirectoryCommand(ctx context.Context, t *testing.T, args ...string } func TestPersonDirectoryCommandMapsDirectoryQueryParameters(t *testing.T) { + cfg := testConfigValue() + _ = cfg + for _, tc := range []struct { name string args []string @@ -111,8 +124,8 @@ func TestPersonDirectoryCommandMapsDirectoryQueryParameters(t *testing.T) { t.Run(tc.name, func(t *testing.T) { assert := assert.New(t) require := require.New(t) - requests := personDirectoryTestResponse(t, http.StatusOK, `{"people":[]}`) - output, err := runPersonDirectoryCommand(t.Context(), t, append(tc.args, "--json")...) + requests, testCtx := personDirectoryTestResponse(t, http.StatusOK, `{"people":[]}`) + output, err := runPersonDirectoryCommand(testCtx, t, append(tc.args, "--json")...) require.NoError(err) require.Len(requests, 1) request := <-requests @@ -128,13 +141,13 @@ func TestPersonDirectoryCommandMapsDirectoryQueryParameters(t *testing.T) { t.Run(flag+"="+invalid, func(t *testing.T) { assert := assert.New(t) require := require.New(t) - requests := personDirectoryTestResponse(t, http.StatusOK, `{"people":[]}`) - output, err := runPersonDirectoryCommand(t.Context(), t, flag, invalid, "--json") + requests, testCtx := personDirectoryTestResponse(t, http.StatusOK, `{"people":[]}`) + output, err := runPersonDirectoryCommand(testCtx, t, flag, invalid, "--json") require.ErrorContains(err, flag+": must be YYYY-MM-DD or RFC3339") assert.Empty(requests) assert.Empty(output) cfg = nil - _, err = runPersonDirectoryCommand(t.Context(), t, flag, invalid) + _, err = runPersonDirectoryCommand(testCtx, t, flag, invalid) assert.ErrorContains(err, flag+": must be YYYY-MM-DD or RFC3339") }) } @@ -157,11 +170,11 @@ func TestPersonDirectoryCommandForwardsCursorAndPrintsNextCursor(t *testing.T) { } })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) - first, err := runPersonDirectoryCommand(t.Context(), t) + testCtx := withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) + first, err := runPersonDirectoryCommand(testCtx, t) require.NoError(err) assert.Contains(first, "Next cursor: "+cursor+"\n") - second, err := runPersonDirectoryCommand(t.Context(), t, "--cursor", cursor) + second, err := runPersonDirectoryCommand(testCtx, t, "--cursor", cursor) require.NoError(err) assert.NotContains(second, "Next cursor") assert.Equal([]string{"", cursor}, cursors) @@ -192,8 +205,8 @@ func TestPersonDirectoryCommandJSONPreservesAbsentLastContact(t *testing.T) { t.Run(tc.name, func(t *testing.T) { assert := assert.New(t) require := require.New(t) - personDirectoryTestResponse(t, http.StatusOK, tc.payload) - output, err := runPersonDirectoryCommand(t.Context(), t, "--json") + _, testCtx := personDirectoryTestResponse(t, http.StatusOK, tc.payload) + output, err := runPersonDirectoryCommand(testCtx, t, "--json") require.NoError(err) decode := func(raw string) any { decoder := json.NewDecoder(strings.NewReader(raw)) @@ -210,11 +223,11 @@ func TestPersonDirectoryCommandJSONPreservesAbsentLastContact(t *testing.T) { func TestPersonDirectoryCommandHumanOutputShowsRecentContactFields(t *testing.T) { assert := assert.New(t) require := require.New(t) - personDirectoryTestResponse(t, http.StatusOK, personDirectoryCLIPayload) + _, testCtx := personDirectoryTestResponse(t, http.StatusOK, personDirectoryCLIPayload) savedJSON := personJSON personJSON = true t.Cleanup(func() { personJSON = savedJSON }) - output, err := runPersonDirectoryCommand(t.Context(), t) + output, err := runPersonDirectoryCommand(testCtx, t) require.NoError(err) assert.True(personJSON) lines := strings.Split(strings.TrimSpace(output), "\n") @@ -236,8 +249,8 @@ func TestPersonDirectoryCommandSanitizesDaemonSuppliedText(t *testing.T) { }], "next_cursor": "\u001b]8;;https://example.test\u0007opaque\u001b]8;;\u0007\u001b[31m-cursor\u001b[0m\r\n" }` - personDirectoryTestResponse(t, http.StatusOK, payload) - output, err := runPersonDirectoryCommand(t.Context(), t) + _, testCtx := personDirectoryTestResponse(t, http.StatusOK, payload) + output, err := runPersonDirectoryCommand(testCtx, t) require.NoError(err) for _, control := range []string{"\x1b", "\r", "\a", "\u009b"} { assert.NotContains(output, control) @@ -246,7 +259,7 @@ func TestPersonDirectoryCommandSanitizesDaemonSuppliedText(t *testing.T) { assert.Contains(strings.Join(strings.Fields(output), " "), "Alice Example") assert.True(strings.HasSuffix(strings.TrimSpace(output), "Next cursor: opaque-cursor")) assert.Len(strings.Split(strings.TrimSpace(output), "\n"), 3) - jsonOutput, err := runPersonDirectoryCommand(t.Context(), t, "--json") + jsonOutput, err := runPersonDirectoryCommand(testCtx, t, "--json") require.NoError(err) assert.JSONEq(payload, jsonOutput) } @@ -296,7 +309,7 @@ func TestPersonDirectoryCLIDaemonRouting(t *testing.T) { if mode != "local default" { localCfg.Remote = config.RemoteConfig{URL: remote.URL, APIKey: "remote-directory-secret", AllowInsecure: true} } - withStoreResolverConfig(t, localCfg) + testCtx := withStoreResolverConfig(t, localCfg) stubStartServeBackgroundProcess(t, func(*config.Config, backgroundServeStartOptions) (*backgroundServeProcess, error) { starts.Add(1) return nil, errors.New("unexpected daemon start") @@ -308,7 +321,7 @@ func TestPersonDirectoryCLIDaemonRouting(t *testing.T) { if mode == "unreachable remote" { remote.Close() } - output, err := runPersonDirectoryCommand(t.Context(), t, args...) + output, err := runPersonDirectoryCommand(testCtx, t, args...) switch mode { case "unreachable remote": require.Error(err) @@ -356,8 +369,8 @@ func TestPersonDirectoryCLICancellation(t *testing.T) { t.Cleanup(remote.Close) localCfg.Remote = config.RemoteConfig{URL: remote.URL, AllowInsecure: true} } - withStoreResolverConfig(t, localCfg) - ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + testCtx := withStoreResolverConfig(t, localCfg) + ctx, cancel := context.WithTimeout(testCtx, 30*time.Second) defer cancel() go func() { select { @@ -381,14 +394,16 @@ func TestPersonDirectoryCLICancellation(t *testing.T) { func TestPersonDirectoryCommandRejectsInvalidSortBeforeRequest(t *testing.T) { assert := assert.New(t) - requests := personDirectoryTestResponse(t, http.StatusOK, `{"people":[]}`) - output, err := runPersonDirectoryCommand(t.Context(), t, "--sort", "oldest", "--json") + requests, testCtx := personDirectoryTestResponse(t, http.StatusOK, `{"people":[]}`) + output, err := runPersonDirectoryCommand(testCtx, t, "--sort", "oldest", "--json") require.ErrorContains(t, err, "--sort: must be name, last_contact_desc, or last_contact_asc") assert.Empty(requests) assert.Empty(output) - cfg = nil - _, err = runPersonDirectoryCommand(t.Context(), t, "--sort", "oldest") + state := invocationFromContext(testCtx) + require.NotNil(t, state) + state.cfg = nil + _, err = runPersonDirectoryCommand(testCtx, t, "--sort", "oldest") assert.ErrorContains(err, "--sort:") } @@ -422,8 +437,8 @@ func TestPersonDirectoryCommandReturnsDaemonQueryErrors(t *testing.T) { t.Run(tc.name, func(t *testing.T) { assert := assert.New(t) require := require.New(t) - requests := personDirectoryTestResponse(t, tc.status, `{"error":"`+tc.code+`","message":"Daemon rejected Directory query"}`) - output, err := runPersonDirectoryCommand(t.Context(), t, append(tc.args, "--json")...) + requests, testCtx := personDirectoryTestResponse(t, tc.status, `{"error":"`+tc.code+`","message":"Daemon rejected Directory query"}`) + output, err := runPersonDirectoryCommand(testCtx, t, append(tc.args, "--json")...) var apiErr *daemonclient.APIError require.ErrorAs(err, &apiErr) assert.Equal(tc.status, apiErr.Status) @@ -447,8 +462,8 @@ func TestPersonDirectoryCommandReturnsResponseErrors(t *testing.T) { {name: "invalid timestamp", status: http.StatusOK, payload: `{"people":[{"last_contact_at":"\u001b[31mnow"}]}`}, } { t.Run(tc.name, func(t *testing.T) { - requests := personDirectoryTestResponse(t, tc.status, tc.payload) - output, err := runPersonDirectoryCommand(t.Context(), t, "--json") + requests, testCtx := personDirectoryTestResponse(t, tc.status, tc.payload) + output, err := runPersonDirectoryCommand(testCtx, t, "--json") require.Error(t, err) assert.Empty(t, output) assert.Len(t, requests, 1) diff --git a/cmd/msgvault/cmd/person_enrichment.go b/cmd/msgvault/cmd/person_enrichment.go index cee9ddd55..b653a5835 100644 --- a/cmd/msgvault/cmd/person_enrichment.go +++ b/cmd/msgvault/cmd/person_enrichment.go @@ -16,6 +16,7 @@ import ( "time" "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/personenrichment" "go.kenn.io/msgvault/internal/providercredentials" "go.kenn.io/msgvault/internal/store" @@ -30,6 +31,7 @@ func (f personEnrichmentManualWorkerFunc) RunOnce(ctx context.Context, runID int } type personEnrichmentCommandDeps struct { + bind func(context.Context) personEnrichmentCommandDeps config func() personenrichment.Config openStore func() (*store.Store, func(), error) lookupEnv personenrichment.CredentialLookup @@ -40,17 +42,53 @@ type personEnrichmentCommandDeps struct { clock func() time.Time } -func defaultPersonEnrichmentCommandDeps() personEnrichmentCommandDeps { +func defaultPersonEnrichmentCommandDeps(contexts ...context.Context) personEnrichmentCommandDeps { + if len(contexts) > 0 { + state := invocationFromContext(contexts[0]) + deps := defaultPersonEnrichmentCommandDeps() + if state != nil { + return deps.bind(contexts[0]) + } + } return personEnrichmentCommandDeps{ - config: func() personenrichment.Config { - if cfg == nil { - return personenrichment.Config{} + bind: func(ctx context.Context) personEnrichmentCommandDeps { + deps := defaultPersonEnrichmentCommandDeps() + state := invocationFromContext(ctx) + var currentCfg *config.Config + if state != nil && state.cfg != nil { + currentCfg = state.cfg + } + deps.config = func() personenrichment.Config { + if currentCfg == nil { + return personenrichment.Config{} + } + return currentCfg.People.Enrichment } - return cfg.People.Enrichment + deps.lookupEnv = func(name string) (string, bool) { + return personEnrichmentEnvironmentLookup(currentCfg)(name) + } + deps.newManualWorker = func( + workerCtx context.Context, st *store.Store, enrichmentConfig personenrichment.Config, + ) (personEnrichmentScheduleWorker, error) { + return newPersonEnrichmentCLIWorker( + workerCtx, st, enrichmentConfig, + personEnrichmentEnvironmentLookup(currentCfg), + personEnrichmentProviderCredentialLookup(currentCfg), + ) + } + deps.openStore = func() (*store.Store, func(), error) { + return openWritableStoreAndInitForInvocation(state) + } + return deps + }, + config: func() personenrichment.Config { + return personenrichment.Config{} + }, + openStore: func() (*store.Store, func(), error) { + return nil, nil, errors.New("configuration is unavailable") }, - openStore: openWritableStoreAndInit, lookupEnv: func(name string) (string, bool) { - return personEnrichmentEnvironmentLookup(cfg)(name) + return personEnrichmentEnvironmentLookup(nil)(name) }, proxyLookupEnv: os.LookupEnv, isDaemonSubprocess: isDaemonCLISubprocess, @@ -62,8 +100,8 @@ func defaultPersonEnrichmentCommandDeps() personEnrichmentCommandDeps { ) (personEnrichmentScheduleWorker, error) { return newPersonEnrichmentCLIWorker( ctx, st, enrichmentConfig, - personEnrichmentEnvironmentLookup(cfg), - personEnrichmentProviderCredentialLookup(cfg), + personEnrichmentEnvironmentLookup(nil), + personEnrichmentProviderCredentialLookup(nil), ) }, clock: time.Now, @@ -129,6 +167,9 @@ func newPersonEnrichmentStatusCommand(deps personEnrichmentCommandDeps) *cobra.C command := &cobra.Command{ Use: "status", Args: cobra.NoArgs, Short: "Show bounded enrichment policy and privacy status", RunE: func(command *cobra.Command, args []string) error { + if invocationFromContext(command.Context()) != nil && deps.bind != nil { + deps = deps.bind(command.Context()) + } if !deps.isDaemonSubprocess() { return proxyPersonEnrichmentCommand(command, args, deps) } @@ -178,6 +219,9 @@ func newPersonEnrichmentProfilesCommand(deps personEnrichmentCommandDeps) *cobra command := &cobra.Command{ Use: "profiles", Args: cobra.NoArgs, Short: "List immutable enrichment provider profiles", RunE: func(command *cobra.Command, args []string) error { + if invocationFromContext(command.Context()) != nil && deps.bind != nil { + deps = deps.bind(command.Context()) + } if !deps.isDaemonSubprocess() { return proxyPersonEnrichmentCommand(command, args, deps) } @@ -211,6 +255,9 @@ func newPersonEnrichmentConsentCommand(deps personEnrichmentCommandDeps) *cobra. command := &cobra.Command{ Use: "consent ", Args: cobra.ExactArgs(1), Short: "Grant exact enrichment policy consent", RunE: func(command *cobra.Command, args []string) error { + if invocationFromContext(command.Context()) != nil && deps.bind != nil { + deps = deps.bind(command.Context()) + } if !deps.isDaemonSubprocess() { return proxyPersonEnrichmentCommand(command, args, deps) } @@ -247,6 +294,9 @@ func newPersonEnrichmentRevokeCommand(deps personEnrichmentCommandDeps) *cobra.C command := &cobra.Command{ Use: "revoke [fingerprint]", Args: cobra.MaximumNArgs(1), Short: "Revoke exact enrichment policy consent", RunE: func(command *cobra.Command, args []string) error { + if invocationFromContext(command.Context()) != nil && deps.bind != nil { + deps = deps.bind(command.Context()) + } if all == (len(args) == 1) { return errors.New("revoke requires exactly one fingerprint or --all") } @@ -294,6 +344,9 @@ func newPersonEnrichmentRunCommand(deps personEnrichmentCommandDeps) *cobra.Comm command := &cobra.Command{ Use: "run", Args: cobra.NoArgs, Short: "Run durable enrichment work for one person and provider", RunE: func(command *cobra.Command, args []string) error { + if invocationFromContext(command.Context()) != nil && deps.bind != nil { + deps = deps.bind(command.Context()) + } if personID <= 0 || strings.TrimSpace(providerName) == "" || strings.TrimSpace(idempotencyKey) == "" { return errors.New("run requires --person, --provider, and --idempotency-key") } @@ -404,6 +457,9 @@ func newPersonEnrichmentSuppressCommand(deps personEnrichmentCommandDeps) *cobra command := &cobra.Command{ Use: "suppress", Args: cobra.NoArgs, Short: "Suppress enrichment by person or stdin identifier", RunE: func(command *cobra.Command, _ []string) error { + if invocationFromContext(command.Context()) != nil && deps.bind != nil { + deps = deps.bind(command.Context()) + } personMode := personID > 0 providerMode := strings.TrimSpace(providerName) != "" || strings.TrimSpace(providerNamespace) != "" if personMode == providerMode { @@ -712,6 +768,7 @@ func newPersonEnrichmentCLIWorker( } factories := make(map[string]personenrichment.ProviderFactory) providerConfigs := make(map[string]personenrichment.ProviderConfig) + providerFingerprints := make(map[string]string) for _, configured := range config.Providers { provider := configured if !provider.Enabled { @@ -725,6 +782,7 @@ func newPersonEnrichmentCLIWorker( return nil, err } providerConfigs[provider.Name] = provider + providerFingerprints[provider.Name] = profile.Fingerprint switch provider.Kind { case personenrichment.ProviderExa: factories[provider.Name] = func(config personenrichment.ProviderConfig, credential string) (personenrichment.Provider, error) { @@ -743,6 +801,8 @@ func newPersonEnrichmentCLIWorker( return personenrichment.NewWorker(st, st, *gate, factories, personenrichment.WorkerOptions{ Owner: "daemon-person-enrichment-manual", LeaseDuration: config.LeaseDuration, RenewEvery: config.LeaseDuration / 4, Clock: time.Now, - Jitter: func(delay time.Duration) time.Duration { return delay }, ProviderConfigs: providerConfigs, + Jitter: func(delay time.Duration) time.Duration { return delay }, + ProviderConfigs: providerConfigs, + ProviderFingerprints: providerFingerprints, }) } diff --git a/cmd/msgvault/cmd/person_enrichment_schedule_test.go b/cmd/msgvault/cmd/person_enrichment_schedule_test.go index 87dee8d75..2ed9ede29 100644 --- a/cmd/msgvault/cmd/person_enrichment_schedule_test.go +++ b/cmd/msgvault/cmd/person_enrichment_schedule_test.go @@ -155,7 +155,7 @@ func TestPersonEnrichmentScheduleResumesRunningRunAfterCrash(t *testing.T) { worker := scheduleEnrichmentWorkerFunc(func(ctx context.Context, runID int64) (bool, error) { calls = append(calls, runID) lease, claimErr := f.Store.ClaimWork(ctx, personenrichment.ClaimOptions{ - RunID: runID, Owner: "recovery-worker", ProviderName: profile.Name, + RunID: runID, Owner: "recovery-worker", ProviderName: profile.Name, ProfileFingerprint: profile.Fingerprint, Now: now, LeaseDuration: time.Minute, }) if claimErr != nil || lease == nil { @@ -481,12 +481,15 @@ func TestRegisterPersonEnrichmentJobCancelsWorkForUnavailableProfiles(t *testing requirements.NoError(err) checks.Equal("terminal", stored.State) requirements.NotNil(stored.FailureClass) - checks.Equal(string(personenrichment.FailurePolicy), *stored.FailureClass) + checks.Equal(string(personenrichment.FailureProfileUnavailable), *stored.FailureClass) work, err := f.Store.ListPersonEnrichmentWorkContext(t.Context(), store.PersonEnrichmentWorkFilter{ PersonID: person.ID, ProfileFingerprint: staleProfile.Fingerprint, Limit: 10, }) requirements.NoError(err) - checks.Empty(work) + requirements.Len(work, 1) + checks.Nil(work[0].RunID) + checks.Nil(work[0].ActiveAttemptID) + checks.Nil(work[0].LeaseOwner) requirements.NoError(f.Store.CompleteRun(t.Context(), run.ID, personenrichment.RunCompletion{ CompletedAt: now.Add(time.Second), })) @@ -556,10 +559,18 @@ func scheduleWorker( require.NoError(t, err) gate, err := personenrichment.NewEgressGate(st, st, hasher, credential) require.NoError(t, err) + catalog, err := st.BuildPersonFactCatalogContext(t.Context(), true) + require.NoError(t, err) + fingerprints := make(map[string]string, len(configs)) + for name, config := range configs { + profile, err := config.Profile(catalog) + require.NoError(t, err) + fingerprints[name] = profile.Fingerprint + } worker, err := personenrichment.NewWorker(st, st, *gate, factories, personenrichment.WorkerOptions{ Owner: "schedule-real-worker", LeaseDuration: time.Minute, RenewEvery: 10 * time.Second, Clock: func() time.Time { return now }, Jitter: func(delay time.Duration) time.Duration { return delay }, - ProviderConfigs: configs, + ProviderConfigs: configs, ProviderFingerprints: fingerprints, }) require.NoError(t, err) return worker @@ -661,7 +672,7 @@ func scheduleTestAttempt( ) *personenrichment.DurableAttempt { t.Helper() lease, err := st.ClaimWork(t.Context(), personenrichment.ClaimOptions{ - RunID: runID, Owner: owner, ProviderName: profile.Name, + RunID: runID, Owner: owner, ProviderName: profile.Name, ProfileFingerprint: profile.Fingerprint, Now: now, LeaseDuration: time.Minute, }) require.NoError(t, err) diff --git a/cmd/msgvault/cmd/person_enrichment_test.go b/cmd/msgvault/cmd/person_enrichment_test.go index d0b35ef37..93b9c6d8f 100644 --- a/cmd/msgvault/cmd/person_enrichment_test.go +++ b/cmd/msgvault/cmd/person_enrichment_test.go @@ -474,7 +474,7 @@ func TestPersonEnrichmentManualRunPersistsAndReusesRunIDBeforeWork(t *testing.T) require.NoError(t, workErr) require.NotEmpty(t, work) lease, claimErr := st.ClaimWork(ctx, personenrichment.ClaimOptions{ - RunID: runID, Owner: "manual-test-worker", ProviderName: profile.Name, + RunID: runID, Owner: "manual-test-worker", ProviderName: profile.Name, ProfileFingerprint: profile.Fingerprint, Now: now, LeaseDuration: time.Minute, }) require.NoError(t, claimErr) @@ -596,7 +596,7 @@ func TestPersonEnrichmentManualRunKeepsRunIDOnLeaseAndAttemptAndReportsFinalCoun return false, nil } lease, claimErr := st.ClaimWork(ctx, personenrichment.ClaimOptions{ - RunID: runID, Owner: "manual-final-worker", ProviderName: profile.Name, + RunID: runID, Owner: "manual-final-worker", ProviderName: profile.Name, ProfileFingerprint: profile.Fingerprint, Now: now, LeaseDuration: time.Minute, }) require.NoError(t, claimErr) @@ -696,6 +696,7 @@ func testPersonEnrichmentCommandDeps( t *testing.T, enrichment personenrichment.Config, st *store.Store, ) personEnrichmentCommandDeps { t.Helper() + cfg := testConfigValue() saved := cfg t.Cleanup(func() { cfg = saved }) cfg = providerCredentialTestConfig(t) diff --git a/cmd/msgvault/cmd/person_facts_test.go b/cmd/msgvault/cmd/person_facts_test.go index fb2af7aff..e1599f0ff 100644 --- a/cmd/msgvault/cmd/person_facts_test.go +++ b/cmd/msgvault/cmd/person_facts_test.go @@ -2,6 +2,7 @@ package cmd import ( "bytes" + "context" "encoding/json" "fmt" "log/slog" @@ -68,16 +69,16 @@ func TestPersonFactsGeneratedPinsResponseDecodesBadPersonID(t *testing.T) { assertions.NoError(err) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) - client, _, err := OpenHTTPStore(t.Context()) + client, _, err := OpenHTTPStore(testCtx) requirements.NoError(err) t.Cleanup(func() { _ = client.Close() }) generatedClient, err := client.GeneratedClient() requirements.NoError(err) - response, err := generatedClient.ListPersonFactPinsWithResponse(t.Context(), + response, err := generatedClient.ListPersonFactPinsWithResponse(testCtx, &generated.ListPersonFactPinsRequestOptions{ PathParams: &generated.ListPersonFactPinsPath{ID: 7}, }) @@ -100,16 +101,15 @@ func TestPersonFactsCatalogUsesGeneratedClientPathAndRendersJSONAndTable(t *test assertions.NoError(err) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) - - table, err := runPersonFactsCommand(t, "catalog") + table, err := runPersonFactsCommand(testCtx, t, "catalog") requirements.NoError(err) assertions.Contains(table, "KIND") assertions.Contains(table, "primary_channel") - jsonOutput, err := runPersonFactsCommand(t, "catalog", "--include-sensitive", "--json") + jsonOutput, err := runPersonFactsCommand(testCtx, t, "catalog", "--include-sensitive", "--json") requirements.NoError(err) assertions.JSONEq(personFactCatalogResponse, jsonOutput) requirements.Len(queries, 2) @@ -166,10 +166,9 @@ func TestPersonFactsHistoryCommandsUseExactGeneratedPathsQueriesAndTables(t *tes } })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) - for _, test := range []struct { command string header string @@ -179,7 +178,7 @@ func TestPersonFactsHistoryCommandsUseExactGeneratedPathsQueriesAndTables(t *tes {command: "claims", header: "CLAIM KEY", path: "/api/v1/people/7/fact-claims"}, {command: "decisions", header: "ACTION", path: "/api/v1/people/7/fact-decisions"}, } { - output, err := runPersonFactsCommand(t, test.command, "7", + output, err := runPersonFactsCommand(testCtx, t, test.command, "7", "--target", personFactEmploymentTarget, "--limit", "4", "--offset", "2") requirements.NoError(err) assertions.Contains(output, test.header) @@ -189,13 +188,13 @@ func TestPersonFactsHistoryCommandsUseExactGeneratedPathsQueriesAndTables(t *tes assertions.Equal("2", query.Get("offset")) } - pins, err := runPersonFactsCommand(t, "pins", "7") + pins, err := runPersonFactsCommand(testCtx, t, "pins", "7") requirements.NoError(err) assertions.Contains(pins, "PINNED") assertions.Contains(pins, "true") assertions.Contains(pins, personFactEmploymentTarget) - jsonOutput, err := runPersonFactsCommand(t, "claims", "7", "--json") + jsonOutput, err := runPersonFactsCommand(testCtx, t, "claims", "7", "--json") requirements.NoError(err) assertions.JSONEq(`[{ "id":8,"generation_id":6,"claim_key":"claim-key", @@ -238,11 +237,10 @@ func TestPersonFactsClaimsTableRetainsMalformedPersistedTargets(t *testing.T) { assertions.NoError(err) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) - - output, err := runPersonFactsCommand(t, "claims", "7") + output, err := runPersonFactsCommand(testCtx, t, "claims", "7") requirements.NoError(err) assertions.Contains(output, "candidate:legacy:key:legacy-revision") assertions.Contains(output, "legacy-claim") @@ -266,11 +264,10 @@ func TestPersonFactsEvidenceStatusForwardsFalseFilterAndRendersNewestFirst(t *te ]}`)) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) - - output, err := runPersonFactsCommand(t, "evidence-status", "7", + output, err := runPersonFactsCommand(testCtx, t, "evidence-status", "7", "--evidence-key", "evidence-key", "--supported=false", "--limit", "5", "--offset", "1") requirements.NoError(err) assertions.Contains(output, "SUPPORTED") @@ -305,10 +302,9 @@ func TestPersonFactsPinAndUnpinSendExactBodyWithoutActorAndRenderProjection(t *t }`)) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) - for _, test := range []struct { command string pinned bool @@ -316,7 +312,7 @@ func TestPersonFactsPinAndUnpinSendExactBodyWithoutActorAndRenderProjection(t *t {command: "pin", pinned: true}, {command: "unpin", pinned: false}, } { - output, err := runPersonFactsCommand(t, test.command, "7", "attribute", "target key") + output, err := runPersonFactsCommand(testCtx, t, test.command, "7", "attribute", "target key") requirements.NoError(err) assertions.Contains(output, "person_attribute_value:11") } @@ -335,10 +331,9 @@ func TestPersonFactsCommandsRejectInvalidInputsBeforeNetwork(t *testing.T) { requests.Add(1) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) - for _, args := range [][]string{ {"evidence", "0"}, {"claims", "7", "--limit", "201"}, @@ -348,19 +343,20 @@ func TestPersonFactsCommandsRejectInvalidInputsBeforeNetwork(t *testing.T) { {"pin", "7", "candidate", "target-key"}, {"unpin", "7", "attribute", " "}, } { - _, err := runPersonFactsCommand(t, args...) + _, err := runPersonFactsCommand(testCtx, t, args...) require.Error(t, err, args) } assert.Zero(t, requests.Load()) } -func runPersonFactsCommand(t *testing.T, args ...string) (string, error) { +func runPersonFactsCommand(ctx context.Context, t *testing.T, args ...string) (string, error) { t.Helper() command := newPersonFactsCommand() var output bytes.Buffer command.SetOut(&output) command.SetErr(&output) command.SetArgs(args) + command.SetContext(ctx) err := command.Execute() return output.String(), err } diff --git a/cmd/msgvault/cmd/person_notes_test.go b/cmd/msgvault/cmd/person_notes_test.go index fd01dffef..3031e203c 100644 --- a/cmd/msgvault/cmd/person_notes_test.go +++ b/cmd/msgvault/cmd/person_notes_test.go @@ -2,6 +2,7 @@ package cmd import ( "bytes" + "context" "encoding/json" "net/http" "net/http/httptest" @@ -50,7 +51,7 @@ const testPersonNotesWriteJSON = `{ }` func executePersonNotesCommand( - t *testing.T, input string, args ...string, + ctx context.Context, t *testing.T, input string, args ...string, ) (string, error) { t.Helper() root := &cobra.Command{Use: "person"} @@ -77,6 +78,7 @@ func executePersonNotesCommand( root.SetOut(&output) root.SetErr(&output) root.SetArgs(append([]string{"notes"}, args...)) + root.SetContext(ctx) err := root.Execute() return output.String(), err } @@ -93,11 +95,10 @@ func TestPersonNotesGetRoutesThroughDaemonAndPrintsOnlyText(t *testing.T) { assert.NoError(t, err) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) - - output, err := executePersonNotesCommand(t, "", "get", "7") + output, err := executePersonNotesCommand(testCtx, t, "", "get", "7") require.NoError(t, err) assert.Equal(t, "Private\ncontext\n", output) } @@ -111,11 +112,10 @@ func TestPersonNotesGetJSONEmitsFullCurrentValue(t *testing.T) { assert.NoError(err) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) - - output, err := executePersonNotesCommand(t, "", "get", "7", "--json") + output, err := executePersonNotesCommand(testCtx, t, "", "get", "7", "--json") require.NoError(err) var value map[string]any require.NoError(json.Unmarshal([]byte(output), &value)) @@ -149,11 +149,10 @@ func TestPersonNotesSetForwardsTypedTextAndOptionalCAS(t *testing.T) { assert.NoError(err) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) - - output, err := executePersonNotesCommand(t, "", "set", "7", + output, err := executePersonNotesCommand(testCtx, t, "", "set", "7", "--text", "Replacement", "--expected-value-id", "71", "--json") require.NoError(err) assert.Equal(map[string]any{"text": "Replacement", "type": "text"}, body["value"]) @@ -189,11 +188,10 @@ func TestPersonNotesSetReadsFileWithoutDiscardingMultilineText(t *testing.T) { assert.NoError(err) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) - - _, err := executePersonNotesCommand(t, "", "set", "7", "--text", "@"+path) + _, err := executePersonNotesCommand(testCtx, t, "", "set", "7", "--text", "@"+path) require.NoError(err) value, ok := body["value"].(map[string]any) require.True(ok) @@ -214,11 +212,10 @@ func TestPersonNotesAppendPreservesMultilineStdinAndUsesAtomicRoute(t *testing.T assert.NoError(t, err) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) - - _, err := executePersonNotesCommand(t, "First line\nSeñor 🌍\n", "append", "7", "--text", "-") + _, err := executePersonNotesCommand(testCtx, t, "First line\nSeñor 🌍\n", "append", "7", "--text", "-") require.NoError(t, err) assert.Equal(t, "First line\nSeñor 🌍\n", body["text"]) assert.Equal(t, "user", body["source"]) @@ -230,10 +227,9 @@ func TestPersonNotesRejectsBlankTextBeforeDaemonRequest(t *testing.T) { requests.Add(1) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) - for _, test := range []struct { name string input string @@ -243,7 +239,7 @@ func TestPersonNotesRejectsBlankTextBeforeDaemonRequest(t *testing.T) { {name: "stdin", input: " \n\t\n", args: []string{"append", "7", "--text", "-"}}, } { t.Run(test.name, func(t *testing.T) { - _, err := executePersonNotesCommand(t, test.input, test.args...) + _, err := executePersonNotesCommand(testCtx, t, test.input, test.args...) require.Error(t, err) assert.ErrorContains(t, err, "notes text must not be blank") }) diff --git a/cmd/msgvault/cmd/person_provider.go b/cmd/msgvault/cmd/person_provider.go index a98b04074..f3aa064a0 100644 --- a/cmd/msgvault/cmd/person_provider.go +++ b/cmd/msgvault/cmd/person_provider.go @@ -10,6 +10,7 @@ import ( "io" "net/http" "os" + "path/filepath" "reflect" "slices" "strings" @@ -17,9 +18,11 @@ import ( "github.com/spf13/cobra" "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/daemonclient" "go.kenn.io/msgvault/internal/peoplesweep" "go.kenn.io/msgvault/internal/store" "go.kenn.io/msgvault/internal/vector" + "go.kenn.io/msgvault/pkg/client/generated" ) const ( @@ -36,6 +39,7 @@ type personProviderStore interface { RevokeAllPersonInferenceConsents(ctx context.Context, actor string) (int64, error) GetPersonInferenceConsentStatus(ctx context.Context, fingerprint string) (*store.PersonInferenceConsentStatus, error) HasSuccessfulPersonInferenceCheck(ctx context.Context, fingerprint string) (bool, error) + InvalidatePersonInferenceCheck(ctx context.Context, fingerprint string) (bool, error) RecordPersonInferenceCheck(ctx context.Context, check store.PersonInferenceCheck) error GetPersonInferenceCheck(ctx context.Context, fingerprint string) (*store.PersonInferenceCheck, error) HasActivePersonInferenceConsent(ctx context.Context, fingerprint string) (bool, error) @@ -60,23 +64,25 @@ type personProviderCodexClient interface { } type personProviderCommandDeps struct { + bind func(personProviderCommandDeps, context.Context) personProviderCommandDeps config func() peoplesweep.Config vectorConfig func() vector.Config openStore func() (personProviderStore, func(), error) openReadStore func() (personProviderStore, func(), error) - newChecker func(peoplesweep.Config, personProviderStore) (personProviderChecker, error) - newCodexClient func(peoplesweep.Config) (personProviderCodexClient, error) + newChecker func(peoplesweep.Config, personProviderStore, personProviderSetupDeps) (personProviderChecker, error) + newCodexClient func(peoplesweep.Config, personProviderSetupDeps) (personProviderCodexClient, error) isDaemonSubprocess func() bool providerStoreOwnedByDaemon func(context.Context) (bool, error) // daemonAliveForRestartNotice reports whether any daemon process in // this machine's data dir is alive, regardless of API compatibility. - // It backs only the restart guidance for local config mutations: a - // daemon left running across a CLI upgrade fails the compatibility - // check yet its scheduled sweeps still keep the startup config. + // Local config mutations use it for restart guidance; removal refuses a + // daemon left running across a CLI upgrade that fails the compatibility + // check because its scheduled sweeps still keep the startup config. daemonAliveForRestartNotice func(context.Context) (bool, error) remoteConfigured func() bool lookupEnv peoplesweep.CredentialLookup proxy func(*cobra.Command, []string, map[string]string) error + removeWithDaemon func(context.Context, string, string) error readConfigFile func() (config.ConfigFile, error) configHomeDir func() string editConfigTables func(string, []config.TableEdit) (config.ConfigFile, error) @@ -174,45 +180,98 @@ type personSemanticProviderRevokeAllOutput struct { Profiles []personSemanticProviderStatusOutput `json:"profiles"` } -func defaultPersonProviderCommandDeps() personProviderCommandDeps { +func defaultPersonProviderCommandDeps(contexts ...context.Context) personProviderCommandDeps { setup := defaultPersonProviderSetupDeps() setup.openCredentialStore = func() (peoplesweep.CredentialStore, error) { - if cfg == nil { - return nil, errors.New("configuration is unavailable") - } - return peoplesweep.NewFileCredentialStore(cfg.TokensDir()), nil - } - return personProviderCommandDeps{ - config: func() peoplesweep.Config { - if cfg == nil { - return peoplesweep.Config{} + return nil, errors.New("configuration is unavailable") + } + deps := personProviderCommandDeps{ + bind: func(deps personProviderCommandDeps, ctx context.Context) personProviderCommandDeps { + state := invocationFromContext(ctx) + var currentCfg *config.Config + if state != nil && state.cfg != nil { + currentCfg = state.cfg + } + deps.setup.codexAuthHome = "" + if currentCfg != nil { + deps.setup.codexAuthHome = filepath.Join(currentCfg.TokensDir(), "people-codex") + } + deps.config = func() peoplesweep.Config { + if currentCfg == nil { + return peoplesweep.Config{} + } + return currentCfg.People.Sweep + } + deps.vectorConfig = func() vector.Config { + if currentCfg == nil { + return vector.Config{} + } + return currentCfg.Vector + } + deps.openReadStore = func() (personProviderStore, func(), error) { + if currentCfg == nil { + return nil, nil, errors.New("configuration is unavailable") + } + st, err := store.OpenReadOnly(currentCfg.DatabaseDSN()) + if err != nil { + return nil, nil, err + } + return st, func() { _ = st.Close() }, nil + } + deps.openStore = func() (personProviderStore, func(), error) { + return openWritableStoreAndInitForInvocation(state) + } + deps.setup.openCredentialStore = func() (peoplesweep.CredentialStore, error) { + if currentCfg == nil { + return nil, errors.New("configuration is unavailable") + } + return peoplesweep.NewFileCredentialStore(currentCfg.TokensDir()), nil + } + deps.remoteConfigured = func() bool { return IsRemoteMode(state) } + deps.readConfigFile = func() (config.ConfigFile, error) { + if currentCfg == nil { + return config.ConfigFile{}, errors.New("configuration is unavailable") + } + return config.ReadConfigFile(currentCfg.ConfigFilePath()) + } + deps.configHomeDir = func() string { + if currentCfg == nil { + return "" + } + return currentCfg.HomeDir + } + deps.editConfigTables = func(ifMatch string, edits []config.TableEdit) (config.ConfigFile, error) { + if currentCfg == nil { + return config.ConfigFile{}, errors.New("configuration is unavailable") + } + return config.EditConfigTables(currentCfg.ConfigFilePath(), ifMatch, edits) + } + deps.restoreConfigFile = func(published, before config.ConfigFile) (config.ConfigFile, error) { + if currentCfg == nil { + return config.ConfigFile{}, errors.New("configuration is unavailable") + } + return config.RestoreConfigFile(currentCfg.ConfigFilePath(), published, before) } - return cfg.People.Sweep + return deps + }, + config: func() peoplesweep.Config { + return peoplesweep.Config{} }, vectorConfig: func() vector.Config { - if cfg == nil { - return vector.Config{} - } - return cfg.Vector + return vector.Config{} }, openStore: func() (personProviderStore, func(), error) { - return openWritableStoreAndInit() + return nil, nil, errors.New("configuration is unavailable") }, openReadStore: func() (personProviderStore, func(), error) { - if cfg == nil { - return nil, nil, errors.New("configuration is unavailable") - } - st, err := store.OpenReadOnly(cfg.DatabaseDSN()) - if err != nil { - return nil, nil, err - } - return st, func() { _ = st.Close() }, nil + return nil, nil, errors.New("configuration is unavailable") }, - newChecker: func(config peoplesweep.Config, st personProviderStore) (personProviderChecker, error) { - registry, err := peoplesweep.NewDriverRegistry( + newChecker: func(config peoplesweep.Config, st personProviderStore, setup personProviderSetupDeps) (personProviderChecker, error) { + registry, err := peoplesweep.NewDriverRegistryWithCodexAuthHome( http.DefaultClient, peoplesweep.NewCodexCommandStarter(), peoplesweep.NewReleasedCodexIsolationGate(), + setup.codexAuthHome, ) if err != nil { return nil, err @@ -235,15 +294,19 @@ func defaultPersonProviderCommandDeps() personProviderCommandDeps { peoplesweep.NewCredentialResolver(credentialStore, os.LookupEnv), ) }, - newCodexClient: func(config peoplesweep.Config) (personProviderCodexClient, error) { + newCodexClient: func(config peoplesweep.Config, setup personProviderSetupDeps) (personProviderCodexClient, error) { + if !peoplesweep.CodexReleaseAvailable() { + return nil, peoplesweep.ErrCodexIsolationUnreleased + } _, provider, err := config.ActiveProviderConfig() if err != nil { return nil, err } - registry, err := peoplesweep.NewDriverRegistry( + registry, err := peoplesweep.NewDriverRegistryWithCodexAuthHome( http.DefaultClient, peoplesweep.NewCodexCommandStarter(), peoplesweep.NewReleasedCodexIsolationGate(), + setup.codexAuthHome, ) if err != nil { return nil, err @@ -260,23 +323,26 @@ func defaultPersonProviderCommandDeps() personProviderCommandDeps { }, isDaemonSubprocess: isDaemonCLISubprocess, providerStoreOwnedByDaemon: func(ctx context.Context) (bool, error) { - if IsRemoteMode() { + state := invocationFromContext(ctx) + if IsRemoteMode(state) { return true, nil } - if cfg == nil { + if state == nil || state.cfg == nil { return false, errors.New("configuration is unavailable") } - runtime, err := findCompatibleDaemonRuntimeContext(ctx, cfg.Data.DataDir) + runtime, err := findCompatibleDaemonRuntimeContext(ctx, state.cfg.Data.DataDir) return runtime != nil, err }, daemonAliveForRestartNotice: func(ctx context.Context) (bool, error) { - if cfg == nil { + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil { return false, errors.New("configuration is unavailable") } - return findAnyDaemonRuntimeContext(ctx, cfg.Data.DataDir) != nil, nil + return findAnyDaemonRuntimeContext(ctx, state.cfg.Data.DataDir) != nil, nil }, + removeWithDaemon: removePersonProviderWithDaemon, lookupEnv: os.LookupEnv, - remoteConfigured: IsRemoteMode, + remoteConfigured: func() bool { return false }, proxy: func(command *cobra.Command, args []string, env map[string]string) error { if len(env) == 0 { return runDaemonCLICommandHTTPFromCobra(command, args) @@ -284,31 +350,35 @@ func defaultPersonProviderCommandDeps() personProviderCommandDeps { return runDaemonCLICommandHTTPFromCobraWithEnv(command, args, env) }, readConfigFile: func() (config.ConfigFile, error) { - if cfg == nil { - return config.ConfigFile{}, errors.New("configuration is unavailable") - } - return config.ReadConfigFile(cfg.ConfigFilePath()) + return config.ConfigFile{}, errors.New("configuration is unavailable") }, configHomeDir: func() string { - if cfg == nil { - return "" - } - return cfg.HomeDir + return "" }, editConfigTables: func(ifMatch string, edits []config.TableEdit) (config.ConfigFile, error) { - if cfg == nil { - return config.ConfigFile{}, errors.New("configuration is unavailable") - } - return config.EditConfigTables(cfg.ConfigFilePath(), ifMatch, edits) + return config.ConfigFile{}, errors.New("configuration is unavailable") }, restoreConfigFile: func(published, before config.ConfigFile) (config.ConfigFile, error) { - if cfg == nil { - return config.ConfigFile{}, errors.New("configuration is unavailable") - } - return config.RestoreConfigFile(cfg.ConfigFilePath(), published, before) + return config.ConfigFile{}, errors.New("configuration is unavailable") }, setup: setup, } + if len(contexts) > 0 { + return personProviderDepsForContext(contexts[0], deps) + } + return deps +} + +func defaultPersonProviderCommandDepsForContext(ctx context.Context) personProviderCommandDeps { + deps := defaultPersonProviderCommandDeps() + return personProviderDepsForContext(ctx, deps) +} + +func personProviderDepsForContext(ctx context.Context, deps personProviderCommandDeps) personProviderCommandDeps { + if invocationFromContext(ctx) == nil || deps.bind == nil { + return deps + } + return deps.bind(deps, ctx) } func newPersonProviderCommand(deps personProviderCommandDeps) *cobra.Command { @@ -318,6 +388,7 @@ func newPersonProviderCommand(deps personProviderCommandDeps) *cobra.Command { } provider.AddCommand( newPersonProviderAddCommand(deps), + newPersonProviderCodexEnrollCommand(defaultCodexEnrollDeps()), newPersonProviderSetCommand(deps), newPersonProviderRemoveCommand(deps), newPersonProviderListCommand(deps), @@ -342,6 +413,7 @@ func newPersonProviderReverifyCommand(deps personProviderCommandDeps) *cobra.Com Short: "Re-run the exact provider check and consent", Args: optionalPersonProviderNameArgs, RunE: func(command *cobra.Command, args []string) error { + deps := personProviderDepsForContext(command.Context(), deps) if !deps.isDaemonSubprocess() { return deps.proxy(command, args, nil) } @@ -387,6 +459,7 @@ func newPersonProviderStatusCommand(deps personProviderCommandDeps) *cobra.Comma Short: "Show the exact people inference policy and consent state", Args: optionalPersonProviderNameArgs, RunE: func(command *cobra.Command, args []string) error { + deps := personProviderDepsForContext(command.Context(), deps) if !deps.isDaemonSubprocess() { return deps.proxy(command, args, nil) } @@ -418,6 +491,7 @@ func newPersonProviderListCommand(deps personProviderCommandDeps) *cobra.Command Short: "List named people inference provider profiles", Args: cobra.NoArgs, RunE: func(command *cobra.Command, args []string) error { + deps := personProviderDepsForContext(command.Context(), deps) if !deps.isDaemonSubprocess() { return deps.proxy(command, args, nil) } @@ -435,6 +509,7 @@ func newPersonProviderUseCommand(deps personProviderCommandDeps) *cobra.Command Short: "Select an exactly checked people inference provider profile", Args: exactPersonProviderNameArgs, RunE: func(command *cobra.Command, args []string) error { + deps := personProviderDepsForContext(command.Context(), deps) return runPersonProviderUse(command, deps, args[0], jsonOutput) }, } @@ -449,6 +524,7 @@ func newPersonProviderRemoveCommand(deps personProviderCommandDeps) *cobra.Comma Short: "Remove a named people inference provider profile", Args: exactPersonProviderNameArgs, RunE: func(command *cobra.Command, args []string) error { + deps := personProviderDepsForContext(command.Context(), deps) return runPersonProviderRemove(command, deps, args[0], jsonOutput) }, } @@ -516,7 +592,8 @@ func rejectRemotePersonProviderMutation(deps personProviderCommandDeps, operatio // compatibility-sensitive because it routes store operations (nothing may // be proxied to an API-incompatible daemon), while a daemon left live across // a CLI upgrade fails that compatibility check yet still serves stale -// startup config, so the restart notice falls back to the compatibility- +// startup config. Removal must reject directStore && daemonRunning; other +// mutations use restart guidance from the compatibility- // agnostic liveness probe. With no ownership signal at all, remove's // existing convention is preserved: assume the store is daemon-owned. func personProviderMutationScope( @@ -834,13 +911,32 @@ func runPersonProviderRemove( if err := peoplesweep.ValidateProviderProfileName(name); err != nil { return err } - if deps.readConfigFile == nil || deps.editConfigTables == nil || deps.restoreConfigFile == nil { + if deps.readConfigFile == nil { return errors.New("people provider config editing is unavailable") } before, err := deps.readConfigFile() if err != nil { return err } + directStore, daemonRunning, err := personProviderMutationScope(command.Context(), deps) + if err != nil { + return err + } + if !directStore { + if deps.removeWithDaemon == nil { + return errors.New("people provider daemon removal is unavailable") + } + if err := deps.removeWithDaemon(command.Context(), name, before.ETag); err != nil { + return err + } + return writePersonProviderRemoved(command, name, true, jsonOutput) + } + if daemonRunning { + return errors.New("cannot identify the running people provider policy; stop the daemon before removing a profile") + } + if deps.editConfigTables == nil || deps.restoreConfigFile == nil { + return errors.New("people provider config editing is unavailable") + } configured, err := personProviderConfigFromSnapshot(deps, before) if err != nil { return err @@ -900,22 +996,10 @@ func runPersonProviderRemove( } }() } - directStore, daemonRunning, scopeErr := personProviderMutationScope(command.Context(), deps) - if scopeErr != nil { - return scopeErr - } - if !directStore { - if err := proxySavedPersonProviderRevoke(command, deps, name, profile.Fingerprint); err != nil { - return err - } - } after, err := deps.editConfigTables(before.ETag, edits) if err != nil { if errors.Is(err, config.ErrConfigChanged) { - return rollbackUncertainPersonProviderRemove(err, deps, before, after, !directStore) - } - if !directStore { - return errors.Join(err, errors.New("exact people provider consent remains revoked after config conflict")) + return errors.Join(err, restoreRemovedPersonProviderConfig(deps, after, before)) } return err } @@ -923,17 +1007,18 @@ func runPersonProviderRemove( return errors.Join(cause, restoreRemovedPersonProviderConfig(deps, after, before)) } - if directStore { - st, cleanup, openErr := deps.openStore() - if openErr != nil { - return rollback(openErr) - } - defer cleanup() - if _, err := st.RevokePersonInferenceConsent( - command.Context(), profile.Fingerprint, personProviderConsentActor, - ); err != nil { - return rollback(err) - } + st, cleanup, openErr := deps.openStore() + if openErr != nil { + return rollback(openErr) + } + defer cleanup() + if _, err := st.RevokePersonInferenceConsent( + command.Context(), profile.Fingerprint, personProviderConsentActor, + ); err != nil { + return rollback(err) + } + if _, err := st.InvalidatePersonInferenceCheck(command.Context(), profile.Fingerprint); err != nil { + return rollback(err) } if provider.Credential == peoplesweep.CredentialStored { if err := credentials.Delete(name, deletionGuard); err != nil { @@ -942,6 +1027,10 @@ func runPersonProviderRemove( errors.New("exact people provider consent remains revoked")) } } + return writePersonProviderRemoved(command, name, false, jsonOutput) +} + +func writePersonProviderRemoved(command *cobra.Command, name string, daemonRunning, jsonOutput bool) error { if jsonOutput { return json.MarshalEncode(jsontext.NewEncoder(command.OutOrStdout()), personProviderRemoveOutput{ Name: name, Removed: true, DaemonRestartRequired: daemonRunning, @@ -964,17 +1053,38 @@ func restoreRemovedPersonProviderConfig( return nil } -func rollbackUncertainPersonProviderRemove( - cause error, - deps personProviderCommandDeps, - before, expected config.ConfigFile, - consentRevoked bool, -) error { - cause = errors.Join(cause, restoreRemovedPersonProviderConfig(deps, expected, before)) - if consentRevoked { - cause = errors.Join(cause, errors.New("exact people provider consent remains revoked")) +// removePersonProviderWithDaemon uses the Settings operation so revocation can +// include the policy captured by the daemon at startup. Never auto-start or +// fall back to a local write if the owning daemon cannot perform the removal. +func removePersonProviderWithDaemon(ctx context.Context, name, ifMatch string) error { + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + currentCfg := state.cfg + runtime, err := findCompatibleDaemonRuntimeContext(ctx, currentCfg.Data.DataDir) + if err != nil { + return err + } + if runtime == nil { + return errors.New("people provider daemon is unavailable; retry after stopping or restarting it") } - return cause + if err := probeLocalDaemonAuth(ctx, runtime, currentCfg); err != nil { + return err + } + client, err := localDaemonAPIClient(urlFromDaemonRuntime(runtime), currentCfg.Server.AuthenticationKey()) + if err != nil { + return err + } + response, err := client.DeleteSettingsPeopleInferenceProviderWithResponse(ctx, + &generated.DeleteSettingsPeopleInferenceProviderRequestOptions{ + PathParams: &generated.DeleteSettingsPeopleInferenceProviderPath{Name: name}, + Header: &generated.DeleteSettingsPeopleInferenceProviderHeaders{IfMatch: ifMatch}, + }) + if err != nil || response == nil || response.StatusCode != http.StatusOK { + return fmt.Errorf("remove people provider: %w", daemonclient.APIResponseError(response, err)) + } + return nil } func newPersonProviderConsentCommand(deps personProviderCommandDeps) *cobra.Command { @@ -986,6 +1096,7 @@ func newPersonProviderConsentCommand(deps personProviderCommandDeps) *cobra.Comm Short: "Consent to the exact people inference policy", Args: optionalPersonProviderNameArgs, RunE: func(command *cobra.Command, args []string) error { + deps := personProviderDepsForContext(command.Context(), deps) if !deps.isDaemonSubprocess() { return deps.proxy(command, args, nil) } @@ -1021,6 +1132,7 @@ func newPersonProviderRevokeCommand(deps personProviderCommandDeps) *cobra.Comma Short: "Revoke consent for the exact people inference policy", Args: optionalPersonProviderNameArgs, RunE: func(command *cobra.Command, args []string) error { + deps := personProviderDepsForContext(command.Context(), deps) if fingerprint != "" { if err := validatePersonProviderFingerprint(fingerprint); err != nil { return err @@ -1068,7 +1180,7 @@ func newPersonProviderRevokeCommand(deps personProviderCommandDeps) *cobra.Comma if fingerprint != "" { return errors.New("--fingerprint requires one named people provider revoke") } - return runPersonProviderRevoke(command, runDeps, all, jsonOutput, semanticEmbeddings) + return runPersonProviderRevoke(command, runDeps, all, jsonOutput, semanticEmbeddings, ifFingerprint != "") }, } command.Flags().BoolVar(&all, "all", false, "Revoke consent for every stored provider policy") @@ -1103,6 +1215,7 @@ func newPersonProviderHistoryCommand(deps personProviderCommandDeps) *cobra.Comm Short: "Show redacted sweep history for an optional provider profile", Args: optionalPersonProviderNameArgs, RunE: func(command *cobra.Command, args []string) error { + deps := personProviderDepsForContext(command.Context(), deps) if !deps.isDaemonSubprocess() { return deps.proxy(command, args, nil) } @@ -1156,6 +1269,7 @@ func newPersonProviderCheckCommand(deps personProviderCommandDeps) *cobra.Comman Short: "Run a fixed synthetic request through the people inference provider", Args: optionalPersonProviderNameArgs, RunE: func(command *cobra.Command, args []string) error { + deps := personProviderDepsForContext(command.Context(), deps) if !deps.isDaemonSubprocess() { if deps.providerStoreOwnedByDaemon != nil { owned, err := deps.providerStoreOwnedByDaemon(command.Context()) @@ -1193,6 +1307,7 @@ func newPersonProviderLoginCommand(deps personProviderCommandDeps) *cobra.Comman Short: "Start Codex ChatGPT device-code login", Args: cobra.NoArgs, RunE: func(command *cobra.Command, args []string) error { + deps := personProviderDepsForContext(command.Context(), deps) return runPersonProviderLogin(command, deps, jsonOutput) }, } @@ -1207,6 +1322,7 @@ func newPersonProviderModelsCommand(deps personProviderCommandDeps) *cobra.Comma Short: "List Codex models and reasoning efforts", Args: cobra.NoArgs, RunE: func(command *cobra.Command, args []string) error { + deps := personProviderDepsForContext(command.Context(), deps) return runPersonProviderModels(command, deps, jsonOutput) }, } @@ -1335,6 +1451,7 @@ func runPersonProviderRevoke( all bool, jsonOutput bool, semanticEmbeddings bool, + invalidateCheck bool, ) error { if semanticEmbeddings { return runPersonSemanticProviderRevoke(command, deps, all, jsonOutput) @@ -1378,6 +1495,13 @@ func runPersonProviderRevoke( ); err != nil { return err } + // Fingerprint-guarded revocation precedes profile replacement. + // Its capability proof must not survive publication of a new credential. + if invalidateCheck { + if _, err := st.InvalidatePersonInferenceCheck(command.Context(), profile.Fingerprint); err != nil { + return err + } + } if jsonOutput { output, err := personProviderStatusFor(command.Context(), st, profile) if err != nil { @@ -1631,7 +1755,7 @@ func checkPersonProvider( return personProviderCheckOutput{}, err } defer cleanup() - checker, err := deps.newChecker(config, st) + checker, err := deps.newChecker(config, st, deps.setup) if err != nil { return personProviderCheckOutput{}, err } @@ -1763,7 +1887,7 @@ func currentPersonProviderCodexClient( if deps.newCodexClient == nil { return nil, errors.New("codex app-server operations are unavailable") } - return deps.newCodexClient(config) + return deps.newCodexClient(config, deps.setup) } func openPersonProviderProfile( diff --git a/cmd/msgvault/cmd/person_provider_codex_enroll.go b/cmd/msgvault/cmd/person_provider_codex_enroll.go new file mode 100644 index 000000000..fdb30c544 --- /dev/null +++ b/cmd/msgvault/cmd/person_provider_codex_enroll.go @@ -0,0 +1,256 @@ +package cmd + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "slices" + "strings" + "time" + + "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/peoplesweep" + "go.kenn.io/msgvault/internal/tui" +) + +type codexEnrollDeps struct { + openBackend func(context.Context) (tui.PeopleInferenceBackend, func(), error) + isTerminal func(*cobra.Command) bool + pollInterval time.Duration +} + +type codexEnrollOptions struct { + model string + reasoningEffort string + retentionPosture string + trainingPosture string + allowedSources []string + sourceSince string + sourceUntil string + allowSensitive bool + yes bool +} + +func defaultCodexEnrollDeps() codexEnrollDeps { + return codexEnrollDeps{ + openBackend: func(ctx context.Context) (tui.PeopleInferenceBackend, func(), error) { + if !peoplesweep.CodexReleaseAvailable() { + return nil, nil, peoplesweep.ErrCodexIsolationUnreleased + } + client, _, err := OpenHTTPStore(ctx) + if err != nil { + return nil, nil, err + } + return newTUISettingsBackend(client), func() { _ = client.Close() }, nil + }, + isTerminal: commandStdinIsTerminal, + pollInterval: time.Second, + } +} + +func newPersonProviderCodexEnrollCommand(deps codexEnrollDeps) *cobra.Command { + var options codexEnrollOptions + command := &cobra.Command{ + Use: "enroll-codex ", + Short: "Create and select a Codex people provider through the daemon", + Args: exactPersonProviderNameArgs, + RunE: func(command *cobra.Command, args []string) error { + return runPersonProviderCodexEnroll(command, deps, args[0], options) + }, + } + flags := command.Flags() + flags.StringVar(&options.model, "model", "", "Codex model ID (prompt when omitted)") + flags.StringVar(&options.reasoningEffort, "reasoning-effort", "", "Supported reasoning effort (prompt when omitted)") + flags.StringVar(&options.retentionPosture, "retention-posture", "", "Operator retention assertion") + flags.StringVar(&options.trainingPosture, "training-posture", "", "Operator training assertion") + flags.StringSliceVar(&options.allowedSources, "source", nil, "Archive source class (repeatable)") + flags.StringVar(&options.sourceSince, "source-since", "", "Earliest disclosed source date (YYYY-MM-DD)") + flags.StringVar(&options.sourceUntil, "source-until", "", "Latest disclosed source date (YYYY-MM-DD)") + flags.BoolVar(&options.allowSensitive, "allow-sensitive", false, "Explicitly allow or exclude sensitive archive content") + flags.BoolVar(&options.yes, "yes", false, "Confirm the displayed check disclosure and select the profile") + return command +} + +func runPersonProviderCodexEnroll( + command *cobra.Command, deps codexEnrollDeps, name string, options codexEnrollOptions, +) error { + if deps.isTerminal == nil || !deps.isTerminal(command) { + return errors.New("codex enrollment requires a terminal; noninteractive commands never start device login") + } + if !command.Flags().Changed("allow-sensitive") || strings.TrimSpace(options.retentionPosture) == "" || + strings.TrimSpace(options.trainingPosture) == "" || len(options.allowedSources) == 0 { + return errors.New("codex enrollment requires --retention-posture, --training-posture, --source, and --allow-sensitive=true|false") + } + for _, source := range options.allowedSources { + if !slices.Contains([]string{"conversation_text", "meeting_text", "document_text"}, source) { + return fmt.Errorf("unsupported source class %q", source) + } + } + since, err := time.Parse("2006-01-02", options.sourceSince) + if err != nil { + return errors.New("--source-since must be a valid YYYY-MM-DD date") + } + if options.sourceUntil != "" { + until, err := time.Parse("2006-01-02", options.sourceUntil) + if err != nil || until.Before(since) { + return errors.New("--source-until must be a valid date on or after --source-since") + } + } + if deps.openBackend == nil { + return errors.New("daemon people enrollment client is unavailable") + } + backend, closeBackend, err := deps.openBackend(command.Context()) + if err != nil { + return err + } + if closeBackend != nil { + defer closeBackend() + } + if backend == nil { + return errors.New("daemon people enrollment client is unavailable") + } + ctx := command.Context() + login, err := backend.StartCodexLogin(ctx, name) + if err != nil { + return err + } + sessionActive := true + defer func() { + if sessionActive { + cancelCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = backend.CancelCodexLogin(cancelCtx, login.SessionID) + } + }() + out := command.OutOrStdout() + _, _ = fmt.Fprintf(out, "Verification URL: %s\nUser code: %s\nLocal deadline: %s\n", + login.URL, login.Code, login.Deadline.UTC().Format(time.RFC3339)) + interval := deps.pollInterval + if interval <= 0 { + interval = time.Second + } + for { + if !login.Deadline.IsZero() && time.Now().After(login.Deadline) { + return errors.New("codex device login reached its local deadline") + } + poll, err := backend.PollCodexLogin(ctx, login.SessionID) + if err != nil { + return err + } + if poll.Complete { + break + } + timer := time.NewTimer(interval) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + } + models, err := backend.ListCodexModels(ctx, login.SessionID) + if err != nil { + return err + } + reader := bufio.NewReader(command.InOrStdin()) + model, effort, err := chooseCodexEnrollmentModel(out, reader, models, options.model, options.reasoningEffort) + if err != nil { + return err + } + profile, err := backend.SaveCodexProfile(ctx, login.SessionID, tui.CodexProfileRequest{ + Name: name, Model: model, ReasoningEffort: effort, + RetentionPosture: strings.TrimSpace(options.retentionPosture), + TrainingPosture: strings.TrimSpace(options.trainingPosture), + AllowedSources: append([]string(nil), options.allowedSources...), + SourceSince: options.sourceSince, SourceUntil: options.sourceUntil, + AllowSensitive: options.allowSensitive, + }) + if err != nil { + return err + } + sessionActive = false // Profile creation consumes the daemon login draft. + disclosure, err := backend.CheckCodexProfile(ctx, profile) + if err != nil { + return err + } + _, _ = fmt.Fprintf(out, "Synthetic check passed for %s (fingerprint %s).\n%s\n", + profile, disclosure.Fingerprint, disclosure.Text) + if !options.yes { + confirmed, err := promptCodexEnrollmentYes(reader, out, "Grant consent and select this profile? [y/N]: ") + if err != nil { + return err + } + if !confirmed { + return errors.New("consent declined; profile was saved but not selected") + } + } + if err := backend.ConsentCodexProfile(ctx, disclosure.Profile, disclosure.Fingerprint); err != nil { + return err + } + if err := backend.SelectCodexProfile(ctx, profile); err != nil { + return err + } + status, err := backend.LoadPeopleInferenceStatus(ctx) + if err != nil { + return err + } + _, _ = fmt.Fprintf(out, "Selected profile: %s\nConfigured: %s Running: %s\n", + profile, status.Configured, status.Running) + if status.PendingRestart { + _, _ = fmt.Fprintln(out, "Restart the daemon to use the selected profile.") + } + return nil +} + +func chooseCodexEnrollmentModel( + out io.Writer, reader *bufio.Reader, models []tui.CodexModelChoice, modelID, effort string, +) (string, string, error) { + if len(models) == 0 { + return "", "", errors.New("signed-in Codex account returned no models") + } + if modelID == "" { + _, _ = fmt.Fprintln(out, "Available Codex models:") + for _, model := range models { + _, _ = fmt.Fprintf(out, " %s (reasoning: %s)\n", model.ID, strings.Join(model.ReasoningEfforts, ", ")) + } + _, _ = fmt.Fprint(out, "Model ID: ") + answer, err := reader.ReadString('\n') + if err != nil && !errors.Is(err, io.EOF) { + return "", "", fmt.Errorf("read model choice: %w", err) + } + modelID = strings.TrimSpace(answer) + } + for _, model := range models { + if model.ID != modelID { + continue + } + if effort == "" { + _, _ = fmt.Fprintf(out, "Reasoning effort for %s (%s; default %s): ", + model.ID, strings.Join(model.ReasoningEfforts, ", "), model.DefaultReasoningEffort) + answer, err := reader.ReadString('\n') + if err != nil && !errors.Is(err, io.EOF) { + return "", "", fmt.Errorf("read reasoning effort: %w", err) + } + effort = strings.TrimSpace(answer) + if effort == "" { + effort = model.DefaultReasoningEffort + } + } + if !slices.Contains(model.ReasoningEfforts, effort) { + return "", "", fmt.Errorf("reasoning effort %q is unavailable for model %q", effort, modelID) + } + return modelID, effort, nil + } + return "", "", fmt.Errorf("model %q is unavailable for the signed-in Codex account", modelID) +} + +func promptCodexEnrollmentYes(reader *bufio.Reader, out io.Writer, prompt string) (bool, error) { + _, _ = fmt.Fprint(out, prompt) + answer, err := reader.ReadString('\n') + if err != nil && !errors.Is(err, io.EOF) { + return false, fmt.Errorf("read consent answer: %w", err) + } + return strings.EqualFold(strings.TrimSpace(answer), "y") || strings.EqualFold(strings.TrimSpace(answer), "yes"), nil +} diff --git a/cmd/msgvault/cmd/person_provider_codex_enroll_test.go b/cmd/msgvault/cmd/person_provider_codex_enroll_test.go new file mode 100644 index 000000000..9b9f3a515 --- /dev/null +++ b/cmd/msgvault/cmd/person_provider_codex_enroll_test.go @@ -0,0 +1,226 @@ +package cmd + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/peoplesweep" + "go.kenn.io/msgvault/internal/tui" +) + +func TestPersonProviderCodexEnrollRejectsNoninteractiveBeforeLogin(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + opened := false + command := newPersonProviderCodexEnrollCommand(codexEnrollDeps{ + isTerminal: func(*cobra.Command) bool { return false }, + openBackend: func(context.Context) (tui.PeopleInferenceBackend, func(), error) { + opened = true + return nil, nil, nil + }, + }) + command.SetArgs([]string{ + "codex-profile", "--source", "conversation_text", "--source-since", "2025-01-01", + "--retention-posture", "operator assertion", "--training-posture", "operator assertion", + "--allow-sensitive=false", "--yes", + }) + command.SetIn(strings.NewReader("")) + command.SetOut(&bytes.Buffer{}) + command.SetErr(&bytes.Buffer{}) + err := command.Execute() + require.Error(err) + assert.Contains(err.Error(), "terminal") + assert.False(opened) +} + +func TestPersonProviderCodexEnrollRejectsBlankPolicyBeforeLogin(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + opened := false + command := newPersonProviderCodexEnrollCommand(codexEnrollDeps{ + isTerminal: func(*cobra.Command) bool { return true }, + openBackend: func(context.Context) (tui.PeopleInferenceBackend, func(), error) { + opened = true + return nil, nil, nil + }, + }) + command.SetArgs([]string{ + "codex-profile", "--source", "conversation_text", "--source-since", "2025-01-01", + "--retention-posture", " ", "--training-posture", "operator assertion", "--allow-sensitive=false", + }) + command.SetIn(strings.NewReader("")) + command.SetOut(&bytes.Buffer{}) + command.SetErr(&bytes.Buffer{}) + err := command.Execute() + require.Error(err) + assert.Contains(err.Error(), "retention-posture") + assert.False(opened) +} + +func TestPersonProviderCodexEnrollCompletesDaemonJourney(t *testing.T) { + testPersonProviderCodexEnrollDaemonJourney(t, "y\n", true) +} + +func TestPersonProviderCodexEnrollDeclinedConsentLeavesSavedProfileUnselected(t *testing.T) { + testPersonProviderCodexEnrollDaemonJourney(t, "n\n", false) +} + +func testPersonProviderCodexEnrollDaemonJourney(t *testing.T, consentAnswer string, wantSelected bool) { + t.Helper() + assert := assert.New(t) + require := require.New(t) + var calls []string + saved, checked, consented, selected := false, false, false, false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls = append(calls, r.Method+" "+r.URL.Path) + w.Header().Set("Content-Type", "application/json") + writeStatus := func() { + profiles := []map[string]any{} + if saved { + profiles = append(profiles, map[string]any{ + "name": "codex-profile", "protocol": "codex_app_server", "model": "gpt-test", + "output_mode": "native_json_schema", "credential_source": "none", + "allowed_sources": []string{"conversation_text"}, "source_since": "2025-01-01", + "retention_posture": "operator assertion: no retention", + "training_posture": "operator assertion: no training", + "fingerprint": "fp-1", "checked": checked, "consent_active": consented, + }) + } + status := map[string]any{"profiles": profiles, "running_name": "old-profile", "pending_restart": selected} + if selected { + status["configured_name"] = "codex-profile" + } + assert.NoError(json.NewEncoder(w).Encode(status)) + } + switch r.Method + " " + r.URL.Path { + case "POST /api/v1/settings/people-inference/codex/login": + var body map[string]any + assert.NoError(json.NewDecoder(r.Body).Decode(&body)) + assert.Equal("codex-profile", body["name"]) + _, _ = w.Write([]byte(`{"session_id":"session-1","verification_url":"https://example.test/device","user_code":"ABCD-EFGH","local_deadline":"2099-09-23T12:05:00Z"}`)) + case "GET /api/v1/settings/people-inference/codex/login/session-1": + _, _ = w.Write([]byte(`{"state":"complete"}`)) + case "GET /api/v1/settings/people-inference/codex/login/session-1/models": + _, _ = w.Write([]byte(`{"models":[{"id":"gpt-test","display_name":"Test model","default_reasoning_effort":"medium","supported_efforts":["low","medium"]}]}`)) + case "GET /api/v1/settings/people-inference": + w.Header().Set("ETag", `"config-1"`) + writeStatus() + case "PUT /api/v1/settings/people-inference/codex/login/session-1/profile": + assert.Equal(`"config-1"`, r.Header.Get("If-Match")) + var body map[string]any + assert.NoError(json.NewDecoder(r.Body).Decode(&body)) + assert.Equal("gpt-test", body["model"]) + assert.Equal("low", body["reasoning_effort"]) + assert.Equal(false, body["allow_sensitive"]) + saved = true + writeStatus() + case "POST /api/v1/settings/people-inference/providers/codex-profile/check": + checked = true + _, _ = w.Write([]byte(`{"ok":true,"fingerprint":"fp-1","model":"gpt-test","usage":{}}`)) + case "POST /api/v1/settings/people-inference/providers/codex-profile/consent": + var body map[string]any + assert.NoError(json.NewDecoder(r.Body).Decode(&body)) + assert.Equal("fp-1", body["fingerprint"]) + assert.Equal(true, body["confirmed"]) + consented = true + writeStatus() + case "POST /api/v1/settings/people-inference/select": + var body map[string]any + assert.NoError(json.NewDecoder(r.Body).Decode(&body)) + assert.Equal("codex-profile", body["name"]) + selected = true + writeStatus() + default: + http.NotFound(w, r) + } + })) + t.Cleanup(server.Close) + backend := newTUISettingsBackend(newTUISettingsDaemonClient(t, server)) + command := newPersonProviderCodexEnrollCommand(codexEnrollDeps{ + isTerminal: func(*cobra.Command) bool { return true }, + openBackend: func(context.Context) (tui.PeopleInferenceBackend, func(), error) { + return backend, func() {}, nil + }, + pollInterval: time.Millisecond, + }) + command.SetArgs([]string{ + "codex-profile", "--source", "conversation_text", "--source-since", "2025-01-01", + "--retention-posture", "operator assertion: no retention", "--training-posture", "operator assertion: no training", + "--allow-sensitive=false", "--model", "gpt-test", "--reasoning-effort", "low", + }) + command.SetIn(strings.NewReader(consentAnswer)) + var output bytes.Buffer + command.SetOut(&output) + command.SetErr(&bytes.Buffer{}) + err := command.Execute() + if wantSelected { + require.NoError(err) + } else { + require.ErrorContains(err, "consent declined; profile was saved but not selected") + } + assert.True(saved) + assert.True(checked) + assert.Equal(wantSelected, consented) + assert.Equal(wantSelected, selected) + assert.Contains(output.String(), "https://example.test/device") + assert.Contains(output.String(), "ABCD-EFGH") + assert.Contains(output.String(), "fp-1") + assert.Contains(output.String(), "codex-profile") + assert.NotContains(strings.Join(calls, "\n"), "DELETE /api/v1/settings/people-inference/codex/login/session-1") + wantCalls := []string{ + "POST /api/v1/settings/people-inference/codex/login", + "GET /api/v1/settings/people-inference/codex/login/session-1", + "GET /api/v1/settings/people-inference/codex/login/session-1/models", + "GET /api/v1/settings/people-inference", + "PUT /api/v1/settings/people-inference/codex/login/session-1/profile", + "GET /api/v1/settings/people-inference", + "POST /api/v1/settings/people-inference/providers/codex-profile/check", + "GET /api/v1/settings/people-inference", + } + if wantSelected { + wantCalls = append(wantCalls, + "GET /api/v1/settings/people-inference", + "POST /api/v1/settings/people-inference/providers/codex-profile/consent", + "GET /api/v1/settings/people-inference", + "POST /api/v1/settings/people-inference/select", + "GET /api/v1/settings/people-inference", + ) + } else { + assert.NotContains(strings.Join(calls, "\n"), "POST /api/v1/settings/people-inference/providers/codex-profile/consent") + assert.NotContains(strings.Join(calls, "\n"), "POST /api/v1/settings/people-inference/select") + } + assert.Equal(wantCalls, calls) +} + +func TestChooseCodexEnrollmentModelPromptsForAvailableChoice(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + models := []tui.CodexModelChoice{{ID: "gpt-test", DefaultReasoningEffort: "medium", ReasoningEfforts: []string{"low", "medium"}}} + var output bytes.Buffer + model, effort, err := chooseCodexEnrollmentModel(&output, bufio.NewReader(strings.NewReader("gpt-test\nlow\n")), models, "", "") + require.NoError(err) + assert.Equal("gpt-test", model) + assert.Equal("low", effort) + assert.Contains(output.String(), "gpt-test") + assert.Contains(output.String(), "low, medium") +} + +func TestCodexProductionClientsRefuseUnreleasedBuild(t *testing.T) { + if peoplesweep.CodexReleaseAvailable() { + t.Skip("Codex release has been approved") + } + _, err := defaultPersonProviderCommandDeps().newCodexClient(peoplesweep.Config{}, personProviderSetupDeps{}) + require.ErrorIs(t, err, peoplesweep.ErrCodexIsolationUnreleased) + _, _, err = defaultCodexEnrollDeps().openBackend(t.Context()) + require.ErrorIs(t, err, peoplesweep.ErrCodexIsolationUnreleased) +} diff --git a/cmd/msgvault/cmd/person_provider_daemon_test.go b/cmd/msgvault/cmd/person_provider_daemon_test.go index 6090e5c30..9ddc8f52b 100644 --- a/cmd/msgvault/cmd/person_provider_daemon_test.go +++ b/cmd/msgvault/cmd/person_provider_daemon_test.go @@ -7,12 +7,14 @@ import ( "fmt" "io" "log/slog" + "maps" "net/http" "net/http/httptest" "os" "strings" "sync/atomic" "testing" + "time" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" @@ -20,6 +22,7 @@ import ( "go.kenn.io/msgvault/internal/api" "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/peoplesweep" + "go.kenn.io/msgvault/internal/personenrollment" "go.kenn.io/msgvault/internal/store" "go.kenn.io/msgvault/internal/testutil" ) @@ -45,6 +48,7 @@ func (s *inProcessPersonProviderDaemonStore) RunCLICommand( deps.newChecker = func( config peoplesweep.Config, consent personProviderStore, + _ personProviderSetupDeps, ) (personProviderChecker, error) { registry, err := peoplesweep.NewDriverRegistry(s.httpClient, nil, nil) if err != nil { @@ -124,8 +128,8 @@ func TestSavedPersonProviderCheckForwardsExactCredentialThroughDaemon(t *testing frontend := *daemonConfig frontend.People.Sweep = peopleConfig frontend.Remote = config.RemoteConfig{URL: server.URL, AllowInsecure: true} - withStoreResolverConfig(t, &frontend) - deps := defaultPersonProviderCommandDeps() + testCtx := withStoreResolverConfig(t, &frontend) + deps := defaultPersonProviderCommandDeps(testCtx) callerHasKey := false deps.setup.lookupEnv = func(name string) (string, bool) { assert.Equal(keyName, name) @@ -133,7 +137,7 @@ func TestSavedPersonProviderCheckForwardsExactCredentialThroughDaemon(t *testing } var output bytes.Buffer command := &cobra.Command{Use: "setup"} - command.SetContext(t.Context()) + command.SetContext(testCtx) command.SetOut(&output) command.SetErr(&output) require.Error(executeSavedPersonProviderCheck(command, deps, "onboarded", "", &output)) @@ -236,20 +240,20 @@ func TestPersonProviderRealDaemonSyntheticCheckAndRevoke(t *testing.T) { frontendConfig := *daemonConfig frontendConfig.Remote = config.RemoteConfig{URL: daemonHTTP.URL, AllowInsecure: true} - withStoreResolverConfig(t, &frontendConfig) + testCtx := withStoreResolverConfig(t, &frontendConfig) const environmentSecretCanary = "caller-key-never-in-daemon-request" t.Setenv("TEST_PROVIDER_KEY", environmentSecretCanary) deps := defaultPersonProviderCommandDeps() - reverifyOutput, err := executePersonProviderCommand(t, deps, "reverify", "--yes") + reverifyOutput, err := executePersonProviderCommandContext(testCtx, t, deps, "reverify", "--yes") require.NoError(err) assert.Contains(reverifyOutput, "People inference provider disclosure") assert.Contains(reverifyOutput, provider.URL+"/v1") captured := <-requests - consentOutput, err := executePersonProviderCommand(t, deps, "consent", "--yes", "--json") + consentOutput, err := executePersonProviderCommandContext(testCtx, t, deps, "consent", "--yes", "--json") require.NoError(err) assert.Contains(consentOutput, `"active":true`) - output, err := executePersonProviderCommand(t, deps, "check", "--json") + output, err := executePersonProviderCommandContext(testCtx, t, deps, "check", "--json") require.NoError(err) assert.JSONEq(`{ "ok":true, @@ -279,9 +283,9 @@ func TestPersonProviderRealDaemonSyntheticCheckAndRevoke(t *testing.T) { assert.NotContains(daemonLogs.String(), environmentSecretCanary) <-requests - _, err = executePersonProviderCommand(t, deps, "revoke", "--json") + _, err = executePersonProviderCommandContext(testCtx, t, deps, "revoke", "--json") require.NoError(err) - output, err = executePersonProviderCommand(t, deps, "check", "--json") + output, err = executePersonProviderCommandContext(testCtx, t, deps, "check", "--json") require.NoError(err) assert.JSONEq(`{ "ok":true, @@ -338,9 +342,9 @@ func TestPersonProviderStoredCheckKeepsSecretOutOfDaemonMetadata(t *testing.T) { frontendConfig := *daemonConfig frontendConfig.Remote = config.RemoteConfig{URL: daemonHTTP.URL, AllowInsecure: true} - withStoreResolverConfig(t, &frontendConfig) + testCtx := withStoreResolverConfig(t, &frontendConfig) deps := defaultPersonProviderCommandDeps() - output, err := executePersonProviderCommand(t, deps, "check", "stored", "--json") + output, err := executePersonProviderCommandContext(testCtx, t, deps, "check", "stored", "--json") require.NoError(err) assert.NotContains(output, secretCanary) @@ -363,3 +367,173 @@ func mustJSON(t *testing.T, value any) []byte { var _ api.CLIRunner = (*inProcessPersonProviderDaemonStore)(nil) var _ api.MessageStore = (*inProcessPersonProviderDaemonStore)(nil) var _ personProviderStore = (*store.Store)(nil) + +// Exercise the adapter installed by serve, not just the underlying Store. +func TestPeopleInferenceSettingsWithDaemonStore(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + providerServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal("/v1/chat/completions", r.URL.Path) + _, _ = io.WriteString(w, `{"model":"test-model","choices":[{"message":{"content":"{\"ok\":true}"},"finish_reason":"stop"}]}`) + })) + defer providerServer.Close() + configured := config.NewDefaultConfig() + configured.HomeDir = t.TempDir() + configured.Data.DataDir = configured.HomeDir + require.NoError(configured.Save()) + st := testutil.NewSQLiteTestStore(t) + before, err := config.ReadConfigFile(configured.ConfigFilePath()) + require.NoError(err) + provider := configuredPersonProvider(personProviderTestConfig()) + provider.Endpoint = providerServer.URL + "/v1" + provider.Auth, provider.Credential, provider.CredentialEnv = peoplesweep.AuthNone, peoplesweep.CredentialNone, "" + created, err := personenrollment.NewService(configured.ConfigFilePath(), st).CreateProfile(before.ETag, "local", provider) + require.NoError(err) + srv := api.NewServerWithOptions(api.ServerOptions{ + Config: configured, Store: &storeAPIAdapter{store: st}, Logger: slog.New(slog.DiscardHandler), + OperationGate: api.NewSerialOperationGate(), + }) + const profilePath = "/api/v1/settings/people-inference/providers/local" + request := func(method, path, body string) *httptest.ResponseRecorder { + r := httptest.NewRequest(method, path, strings.NewReader(body)) + r.RemoteAddr = "127.0.0.1:12345" + r.Header.Set("If-Match", created.ETag) + r.Header.Set("Content-Type", "application/json") + response := httptest.NewRecorder() + srv.Router().ServeHTTP(response, r) + return response + } + checked := request(http.MethodPost, profilePath+"/check", "") + require.Equal(http.StatusOK, checked.Code, checked.Body.String()) + consented := request(http.MethodPost, profilePath+"/consent", fmt.Sprintf(`{"confirmed":true,"fingerprint":%q}`, created.Fingerprint)) + require.Equal(http.StatusOK, consented.Code, consented.Body.String()) + var status api.PeopleInferenceSettingsResponse + require.NoError(json.Unmarshal(consented.Body.Bytes(), &status)) + var local *api.PeopleInferenceProfileSetting + for i := range status.Profiles { + if status.Profiles[i].Name == "local" { + local = &status.Profiles[i] + } + } + require.NotNil(local) + assert.True(local.Checked) + assert.True(local.ConsentActive) + removed := request(http.MethodDelete, profilePath, "") + require.Equal(http.StatusOK, removed.Code, removed.Body.String()) + active, err := st.HasActivePersonInferenceConsent(t.Context(), created.Fingerprint) + require.NoError(err) + assert.False(active) + verified, err := st.HasSuccessfulPersonInferenceCheck(t.Context(), created.Fingerprint) + require.NoError(err) + assert.False(verified) +} + +func TestPersonProviderRemoveRevokesRunningPolicyThroughDaemon(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + var calls atomic.Int64 + provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal("/v1/chat/completions", r.URL.Path) + calls.Add(1) + _, _ = io.WriteString(w, `{"model":"running-model","choices":[{"message":{"content":"{\"ok\":true}"},"finish_reason":"stop"}]}`) + })) + defer provider.Close() + startup := config.NewDefaultConfig() + startup.HomeDir = t.TempDir() + startup.Data.DataDir = startup.HomeDir + startup.People.Sweep = personProviderTestConfig() + beta := configuredPersonProvider(startup.People.Sweep) + beta.Model, beta.Endpoint = "running-model", provider.URL+"/v1" + beta.Auth, beta.Credential, beta.CredentialEnv = peoplesweep.AuthNone, peoplesweep.CredentialNone, "" + startup.People.Sweep.Providers["beta"] = beta + startup.People.Sweep.Provider.Name = "beta" + runningProfile, err := startup.People.Sweep.Profile() + require.NoError(err) + saved := *startup + saved.People.Sweep.Providers = maps.Clone(startup.People.Sweep.Providers) + beta.Model = "saved-model" + saved.People.Sweep.Providers["beta"] = beta + savedProfile, err := saved.People.Sweep.Profile() + require.NoError(err) + require.NotEqual(runningProfile.Fingerprint, savedProfile.Fingerprint) + saved.People.Sweep.Provider.Name = "default" + require.NoError(saved.Save()) + st := testutil.NewSQLiteTestStore(t) + for _, profile := range []peoplesweep.ProviderProfile{runningProfile, savedProfile} { + _, err = st.EnsurePersonInferenceProfile(t.Context(), profile) + require.NoError(err) + require.NoError(st.RecordPersonInferenceCheck(t.Context(), store.PersonInferenceCheck{ + ProfileFingerprint: profile.Fingerprint, CheckedAt: time.Now(), + DriverVersion: profile.DriverVersion, OutputMode: profile.OutputMode, ModelVersion: profile.Model, + })) + _, _, err = st.GrantPersonInferenceConsent(t.Context(), profile.Fingerprint, "test") + require.NoError(err) + } + adapter := &storeAPIAdapter{store: st} + srv := api.NewServerWithOptions(api.ServerOptions{ + Config: startup, Store: adapter, Logger: slog.New(slog.DiscardHandler), + DaemonVersion: Version, OperationGate: api.NewSerialOperationGate(), + }) + daemon := httptest.NewServer(srv.Router()) + defer daemon.Close() + writeStatsHTTPDaemonRuntime(t, startup.Data.DataDir, daemon) + testCtx := withStoreResolverConfig(t, &saved) + deps := defaultPersonProviderCommandDeps() + // Execute the old subprocess route in-process if removal still uses it. + // Both paths use the real command/store; the test never launches a host daemon. + deps.proxy = func(command *cobra.Command, args []string, _ map[string]string) error { + argv, err := daemonCLIArgsFromCobra(command, args) + if err != nil { + return err + } + _, err = executePersonProviderCommand(t, localPersonProviderDeps(saved.People.Sweep, st, nil), argv[2:]...) + return err + } + runner, err := newProductionStructuredRunner(startup, st) + require.NoError(err) + request := peoplesweep.StructuredRequest{ + ProgramID: "removal-test", ProgramVersion: "1", InputText: "synthetic input", SchemaName: "removal_test", + Sources: []peoplesweep.SourceDescriptor{{Class: peoplesweep.SourceConversationText, ObservedOn: "2025-06-01"}}, + JSONSchema: []byte(`{"type":"object","properties":{"ok":{"type":"boolean"}},"required":["ok"],"additionalProperties":false}`), + MaxOutputTokens: 16, + } + _, err = runner.RunStructured(t.Context(), request) + require.NoError(err) + require.Equal(int64(1), calls.Load()) + // Reject a stale config revision without revoking either policy. + before, err := config.ReadConfigFile(saved.ConfigFilePath()) + require.NoError(err) + require.NoError(os.WriteFile(saved.ConfigFilePath(), append(before.Content, []byte("\n# concurrent edit\n")...), 0o600)) + deps.removeWithDaemon = func(ctx context.Context, name, _ string) error { + return removePersonProviderWithDaemon(ctx, name, before.ETag) + } + _, err = executePersonProviderCommandContext(testCtx, t, deps, "remove", "beta") + require.ErrorContains(err, "config file changed") + for _, profile := range []peoplesweep.ProviderProfile{runningProfile, savedProfile} { + active, err := st.HasActivePersonInferenceConsent(t.Context(), profile.Fingerprint) + require.NoError(err) + assert.True(active, profile.Model) + } + deps.removeWithDaemon = removePersonProviderWithDaemon + output, err := executePersonProviderCommandContext(testCtx, t, deps, "remove", "beta", "--json") + require.NoError(err) + var removed personProviderRemoveOutput + require.NoError(json.Unmarshal([]byte(output), &removed)) + assert.Equal(personProviderRemoveOutput{Name: "beta", Removed: true, DaemonRestartRequired: true}, removed) + snapshot, err := config.ReadConfigFile(saved.ConfigFilePath()) + require.NoError(err) + current, err := config.LoadConfigFile(snapshot, saved.HomeDir) + require.NoError(err) + assert.NotContains(current.People.Sweep.Providers, "beta") + for _, profile := range []peoplesweep.ProviderProfile{runningProfile, savedProfile} { + active, err := st.HasActivePersonInferenceConsent(t.Context(), profile.Fingerprint) + require.NoError(err) + assert.False(active, profile.Model) + checked, err := st.HasSuccessfulPersonInferenceCheck(t.Context(), profile.Fingerprint) + require.NoError(err) + assert.False(checked, profile.Model) + } + _, err = runner.RunStructured(t.Context(), request) + require.Error(err) + assert.Equal(int64(1), calls.Load(), "removal must prevent further provider requests") +} diff --git a/cmd/msgvault/cmd/person_provider_routing_test.go b/cmd/msgvault/cmd/person_provider_routing_test.go index e2104e5d2..853560249 100644 --- a/cmd/msgvault/cmd/person_provider_routing_test.go +++ b/cmd/msgvault/cmd/person_provider_routing_test.go @@ -154,7 +154,7 @@ func TestPersonProviderLoginAndModelsNeverProxy(t *testing.T) { proxied = true return nil }, - newCodexClient: func(peoplesweep.Config) (personProviderCodexClient, error) { + newCodexClient: func(peoplesweep.Config, personProviderSetupDeps) (personProviderCodexClient, error) { return nil, assert.AnError }, } @@ -166,57 +166,6 @@ func TestPersonProviderLoginAndModelsNeverProxy(t *testing.T) { } } -func TestPersonProviderFrontendRemoveProxiesOnlyNamedRevoke(t *testing.T) { - assertAnError := assert.AnError - newRequire := require.New - assert := assert.New(t) - require := require.New(t) - configured := personProviderTestConfig() - beta := configuredPersonProvider(configured) - beta.Model = "beta-model" - configured.Providers["beta"] = beta - path, _ := retainedPersonProviderTestConfig(t, configured) - selected := configured - selected.Provider = peoplesweep.ProviderSelection{Name: "beta"} - profile, err := selected.Profile() - require.NoError(err) - var gotArgs []string - var events []string - deps := personProviderCommandDeps{ - config: func() peoplesweep.Config { return configured }, - isDaemonSubprocess: func() bool { return false }, - proxy: func(command *cobra.Command, args []string, _ map[string]string) error { - events = append(events, "revoke") - var err error - gotArgs, err = daemonCLIArgsFromCobra(command, args) - return err - }, - openStore: func() (personProviderStore, func(), error) { - require := newRequire(t) - require.FailNow("frontend remove must revoke through the daemon owner") - return nil, nil, assertAnError - }, - readConfigFile: func() (config.ConfigFile, error) { - return config.ReadConfigFile(path) - }, - editConfigTables: func(etag string, edits []config.TableEdit) (config.ConfigFile, error) { - events = append(events, "edit") - return config.EditConfigTables(path, etag, edits) - }, - restoreConfigFile: func(published, before config.ConfigFile) (config.ConfigFile, error) { - return config.RestoreConfigFile(path, published, before) - }, - configHomeDir: func() string { return filepath.Dir(path) }, - } - - _, err = executePersonProviderCommand(t, deps, "remove", "beta") - require.NoError(err) - assert.Equal([]string{ - "person", "provider", "revoke", "--if-fingerprint=" + profile.Fingerprint, "beta", - }, gotArgs) - assert.Equal([]string{"revoke", "edit"}, events) -} - func TestPersonProviderRemoveCompletesLocalPreflightBeforeRevoke(t *testing.T) { for _, test := range []struct { name string @@ -277,17 +226,12 @@ func TestPersonProviderRemoveCompletesLocalPreflightBeforeRevoke(t *testing.T) { if test.mutateFile != nil { test.mutateFile(t, path) } - revokes := 0 edits := 0 deps := personProviderCommandDeps{ config: func() peoplesweep.Config { return test.configured }, isDaemonSubprocess: func() bool { return false }, - providerStoreOwnedByDaemon: func(context.Context) (bool, error) { return true, nil }, - proxy: func(*cobra.Command, []string, map[string]string) error { - revokes++ - return nil - }, - readConfigFile: func() (config.ConfigFile, error) { return config.ReadConfigFile(path) }, + providerStoreOwnedByDaemon: func(context.Context) (bool, error) { return false, nil }, + readConfigFile: func() (config.ConfigFile, error) { return config.ReadConfigFile(path) }, editConfigTables: func(etag string, planned []config.TableEdit) (config.ConfigFile, error) { edits++ return config.EditConfigTables(path, etag, planned) @@ -299,13 +243,12 @@ func TestPersonProviderRemoveCompletesLocalPreflightBeforeRevoke(t *testing.T) { _, err := executePersonProviderCommand(t, deps, "remove", test.configured.Provider.Name) require.ErrorContains(t, err, test.wantError) - assert.Zero(t, revokes) assert.Zero(t, edits) }) } } -func TestPersonProviderDaemonRemovePreflightsStoredCredentialBeforeRevoke(t *testing.T) { +func TestPersonProviderLocalRemovePreflightsStoredCredentialBeforeRevoke(t *testing.T) { assert := assert.New(t) require := require.New(t) requireStoredCredentialStorePlatform(t) @@ -328,18 +271,12 @@ func TestPersonProviderDaemonRemovePreflightsStoredCredentialBeforeRevoke(t *tes externalPath := filepath.Join(t.TempDir(), "external-credential") require.NoError(os.WriteFile(externalPath, []byte("must-remain"), 0o600)) require.NoError(os.Symlink(externalPath, credentialPath)) - - revokes := 0 edits := 0 deps := personProviderCommandDeps{ config: func() peoplesweep.Config { return configured }, isDaemonSubprocess: func() bool { return false }, - providerStoreOwnedByDaemon: func(context.Context) (bool, error) { return true, nil }, - proxy: func(*cobra.Command, []string, map[string]string) error { - revokes++ - return nil - }, - readConfigFile: func() (config.ConfigFile, error) { return config.ReadConfigFile(path) }, + providerStoreOwnedByDaemon: func(context.Context) (bool, error) { return false, nil }, + readConfigFile: func() (config.ConfigFile, error) { return config.ReadConfigFile(path) }, editConfigTables: func(etag string, planned []config.TableEdit) (config.ConfigFile, error) { edits++ return config.EditConfigTables(path, etag, planned) @@ -352,7 +289,6 @@ func TestPersonProviderDaemonRemovePreflightsStoredCredentialBeforeRevoke(t *tes output, err := executePersonProviderCommand(t, deps, "remove", "beta") require.Error(err) - assert.Zero(revokes) assert.Zero(edits) assert.NotContains(output, providerSetupSecretCanary) assert.NotContains(err.Error(), providerSetupSecretCanary) @@ -367,7 +303,7 @@ func TestPersonProviderDaemonRemovePreflightsStoredCredentialBeforeRevoke(t *tes assert.Contains(finalConfig.People.Sweep.Providers, "beta") } -func TestPersonProviderDaemonRemoveMissingCredentialRootHasZeroSideEffects(t *testing.T) { +func TestPersonProviderLocalRemoveMissingCredentialRootHasZeroSideEffects(t *testing.T) { assert := assert.New(t) require := require.New(t) requireStoredCredentialStorePlatform(t) @@ -388,18 +324,12 @@ func TestPersonProviderDaemonRemoveMissingCredentialRootHasZeroSideEffects(t *te require.NoError(statErr) beforeEntries, readErr := os.ReadDir(tokensParent) require.NoError(readErr) - - revokes := 0 edits := 0 deps := personProviderCommandDeps{ config: func() peoplesweep.Config { return configured }, isDaemonSubprocess: func() bool { return false }, - providerStoreOwnedByDaemon: func(context.Context) (bool, error) { return true, nil }, - proxy: func(*cobra.Command, []string, map[string]string) error { - revokes++ - return nil - }, - readConfigFile: func() (config.ConfigFile, error) { return config.ReadConfigFile(path) }, + providerStoreOwnedByDaemon: func(context.Context) (bool, error) { return false, nil }, + readConfigFile: func() (config.ConfigFile, error) { return config.ReadConfigFile(path) }, editConfigTables: func(etag string, planned []config.TableEdit) (config.ConfigFile, error) { edits++ return config.EditConfigTables(path, etag, planned) @@ -412,7 +342,6 @@ func TestPersonProviderDaemonRemoveMissingCredentialRootHasZeroSideEffects(t *te output, err := executePersonProviderCommand(t, deps, "remove", "beta") require.ErrorContains(err, "preflight stored people provider credential deletion") - assert.Zero(revokes) assert.Zero(edits) assert.Zero(credentials.deletes) assert.NotContains(output, providerSetupSecretCanary) @@ -430,7 +359,7 @@ func TestPersonProviderDaemonRemoveMissingCredentialRootHasZeroSideEffects(t *te assert.Contains(finalConfig.People.Sweep.Providers, "beta") } -func TestPersonProviderDaemonRemoveValidReplacementRaceRollsBackExactConfig(t *testing.T) { +func TestPersonProviderLocalRemoveValidReplacementRaceRollsBackExactConfig(t *testing.T) { assert := assert.New(t) require := require.New(t) requireStoredCredentialStorePlatform(t) @@ -463,19 +392,15 @@ func TestPersonProviderDaemonRemoveValidReplacementRaceRollsBackExactConfig(t *t return os.WriteFile(credentialPath, replacementContents, 0o600) }, } - - revokes := 0 edits := 0 restores := 0 + st := testutil.NewSQLiteTestStore(t) deps := personProviderCommandDeps{ config: func() peoplesweep.Config { return configured }, + openStore: func() (personProviderStore, func(), error) { return st, func() {}, nil }, isDaemonSubprocess: func() bool { return false }, - providerStoreOwnedByDaemon: func(context.Context) (bool, error) { return true, nil }, - proxy: func(*cobra.Command, []string, map[string]string) error { - revokes++ - return nil - }, - readConfigFile: func() (config.ConfigFile, error) { return config.ReadConfigFile(path) }, + providerStoreOwnedByDaemon: func(context.Context) (bool, error) { return false, nil }, + readConfigFile: func() (config.ConfigFile, error) { return config.ReadConfigFile(path) }, editConfigTables: func(etag string, planned []config.TableEdit) (config.ConfigFile, error) { edits++ return config.EditConfigTables(path, etag, planned) @@ -490,7 +415,6 @@ func TestPersonProviderDaemonRemoveValidReplacementRaceRollsBackExactConfig(t *t output, err := executePersonProviderCommand(t, deps, "remove", "beta") require.ErrorContains(err, "credential changed during guarded deletion") require.ErrorContains(err, "exact people provider consent remains revoked") - assert.Equal(1, revokes) assert.Equal(1, edits) assert.Equal(1, restores) assert.NotContains(output, providerSetupSecretCanary) diff --git a/cmd/msgvault/cmd/person_provider_setup.go b/cmd/msgvault/cmd/person_provider_setup.go index 400174ecd..38d864a3d 100644 --- a/cmd/msgvault/cmd/person_provider_setup.go +++ b/cmd/msgvault/cmd/person_provider_setup.go @@ -27,6 +27,7 @@ type personProviderSetupDeps struct { negotiate func(context.Context, peoplesweep.ProviderConfig, peoplesweep.Credential) (peoplesweep.NegotiatedCapabilities, error) credentials peoplesweep.CredentialStore openCredentialStore func() (peoplesweep.CredentialStore, error) + codexAuthHome string lookupEnv peoplesweep.CredentialLookup isTerminal func(uintptr) bool readMasked func(*os.File, int) ([]byte, error) @@ -42,6 +43,7 @@ type personProviderCreateCredentialStore interface { type personProviderAddOptions struct { custom bool + presetID string protocol string endpoint string model string @@ -92,7 +94,7 @@ func (o personProviderAddOptions) explicitTransport() bool { // needsCatalog reports whether add must fetch the models.dev catalog: only // when a transport field is missing or a catalog price hint was requested. func (o personProviderAddOptions) needsCatalog() bool { - return !o.custom && (!o.explicitTransport() || o.acceptCatalogPrices) + return o.presetID == "" && !o.custom && (!o.explicitTransport() || o.acceptCatalogPrices) } func defaultPersonProviderSetupDeps() personProviderSetupDeps { @@ -145,11 +147,13 @@ func newPersonProviderAddCommand(deps personProviderCommandDeps) *cobra.Command Short: "Add and check a named people inference provider profile", Args: exactPersonProviderNameArgs, RunE: func(command *cobra.Command, args []string) error { + deps := personProviderDepsForContext(command.Context(), deps) return runPersonProviderAdd(command, deps, args[0], options) }, } flags := command.Flags() flags.BoolVar(&options.custom, "custom", false, "Skip public catalog suggestions") + flags.StringVar(&options.presetID, "provider", "", "First-party provider preset: openai, openrouter, or venice") flags.StringVar(&options.protocol, "protocol", "", "Explicit protocol identifier") flags.StringVar(&options.endpoint, "endpoint", "", "Explicit provider endpoint") flags.StringVar(&options.model, "model", "", "Explicit provider model identifier") @@ -179,6 +183,7 @@ func newPersonProviderSetCommand(deps personProviderCommandDeps) *cobra.Command Short: "Update and check a named people inference provider profile", Args: exactPersonProviderNameArgs, RunE: func(command *cobra.Command, args []string) error { + deps := personProviderDepsForContext(command.Context(), deps) return runPersonProviderSet(command, deps, args[0], options) }, } @@ -386,19 +391,37 @@ func runPersonProviderAdd( } func personProviderCandidate(options personProviderAddOptions) (peoplesweep.ProviderConfig, error) { - if options.protocol == "" || options.endpoint == "" || options.model == "" || options.auth == "" || + if (options.presetID == "" && (options.protocol == "" || options.endpoint == "" || options.auth == "")) || options.model == "" || options.retentionPosture == "" || options.trainingPosture == "" || len(options.allowedSources) == 0 || options.sourceSince == "" { - return peoplesweep.ProviderConfig{}, errors.New("protocol, endpoint, model, auth, retention, training, source, and source-since are required") - } - candidate := peoplesweep.ProviderConfig{ - Protocol: peoplesweep.Protocol(options.protocol), Endpoint: options.endpoint, - Model: options.model, Auth: peoplesweep.AuthScheme(options.auth), - RetentionPosture: options.retentionPosture, TrainingPosture: options.trainingPosture, - SourceSince: options.sourceSince, SourceUntil: options.sourceUntil, - AllowSensitive: options.allowSensitive, ReasoningEffort: options.reasoningEffort, - ReasoningMode: options.reasoningMode, RequestTimeout: options.requestTimeout, + return peoplesweep.ProviderConfig{}, errors.New("provider or protocol, endpoint, auth, model, retention, training, source, and source-since are required") } + var candidate peoplesweep.ProviderConfig + if options.presetID != "" { + var err error + candidate, err = peoplesweep.PresetProviderConfig(options.presetID, options.model) + if err != nil { + return peoplesweep.ProviderConfig{}, err + } + if (options.protocol != "" && options.protocol != string(candidate.Protocol)) || + (options.endpoint != "" && options.endpoint != candidate.Endpoint) || + (options.auth != "" && options.auth != string(candidate.Auth)) { + return peoplesweep.ProviderConfig{}, fmt.Errorf("people provider preset %q cannot use a different protocol, endpoint, or auth", options.presetID) + } + } else { + candidate.Protocol = peoplesweep.Protocol(options.protocol) + candidate.Endpoint = options.endpoint + candidate.Model = options.model + candidate.Auth = peoplesweep.AuthScheme(options.auth) + } + candidate.RetentionPosture = options.retentionPosture + candidate.TrainingPosture = options.trainingPosture + candidate.SourceSince = options.sourceSince + candidate.SourceUntil = options.sourceUntil + candidate.AllowSensitive = options.allowSensitive + candidate.ReasoningEffort = options.reasoningEffort + candidate.ReasoningMode = options.reasoningMode + candidate.RequestTimeout = options.requestTimeout for _, source := range options.allowedSources { candidate.AllowedSources = append(candidate.AllowedSources, peoplesweep.SourceClass(source)) } @@ -420,6 +443,9 @@ func personProviderCandidate(options personProviderAddOptions) (peoplesweep.Prov } func validatePersonProviderAddOptions(options personProviderAddOptions) error { + if options.presetID != "" && (options.custom || options.acceptCatalogPrices) { + return errors.New("--provider cannot be combined with --custom or --accept-catalog-prices") + } if options.protocol == string(peoplesweep.ProtocolCodexAppServer) { return errors.New("codex_app_server profiles are not created by person provider add: " + "generic onboarding requires an HTTP endpoint that codex_app_server forbids, and capability negotiation " + @@ -454,7 +480,7 @@ func resolvePersonProviderAddCandidate( options personProviderAddOptions, suggestions []peoplesweep.ProviderSuggestion, ) (peoplesweep.ProviderConfig, error) { - if options.custom || options.explicitTransport() { + if options.presetID != "" || options.custom || options.explicitTransport() { return personProviderCandidate(options) } selection, err := selectPersonProviderCatalogSuggestion(options, suggestions) @@ -1025,15 +1051,6 @@ func verifyPersonProviderFingerprint( return nil } -func proxySavedPersonProviderRevoke( - command *cobra.Command, - deps personProviderCommandDeps, - name string, - fingerprint string, -) error { - return proxySavedPersonProviderOperation(command, deps, "revoke", name, fingerprint, command.OutOrStdout()) -} - func proxySavedPersonProviderRevokeFingerprint( command *cobra.Command, deps personProviderCommandDeps, diff --git a/cmd/msgvault/cmd/person_provider_setup_test.go b/cmd/msgvault/cmd/person_provider_setup_test.go index ce702ec69..93818a496 100644 --- a/cmd/msgvault/cmd/person_provider_setup_test.go +++ b/cmd/msgvault/cmd/person_provider_setup_test.go @@ -13,6 +13,7 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/charmbracelet/x/term" "github.com/spf13/cobra" @@ -20,6 +21,7 @@ import ( "github.com/stretchr/testify/require" "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/peoplesweep" + "go.kenn.io/msgvault/internal/store" "go.kenn.io/msgvault/internal/testutil" ) @@ -276,6 +278,45 @@ func TestPersonProviderAddValidatesPolicyBeforeReadingCredentialOrNegotiating(t assert.NotContains(output, providerSetupSecretCanary) } +func TestPersonProviderPresetRejectsEndpointSwapBeforeCredential(t *testing.T) { + path, loaded := providerSetupConfigFile(t) + deps := providerSetupCommandDeps(t, path, loaded, nil) + var lookups, negotiations int + deps.setup.lookupEnv = func(string) (string, bool) { + lookups++ + return providerSetupSecretCanary, true + } + deps.setup.negotiate = func(context.Context, peoplesweep.ProviderConfig, peoplesweep.Credential) (peoplesweep.NegotiatedCapabilities, error) { + negotiations++ + return peoplesweep.NegotiatedCapabilities{}, nil + } + + _, err := executePersonProviderCommand(t, deps, + "add", "venice-bound", "--provider", "venice", "--model", "venice/model", + "--endpoint", "https://elsewhere.example.test/v1", "--credential-env", "VENICE_KEY", + "--retention-posture", "operator_asserted", "--training-posture", "operator_asserted", + "--source", "conversation_text", "--source-since", "2025-01-01", "--yes") + require.ErrorContains(t, err, "preset") + assert.Zero(t, lookups) + assert.Zero(t, negotiations) +} + +func TestPersonProviderPresetCandidateKeepsAssertionsExplicit(t *testing.T) { + assert := assert.New(t) + candidate, err := personProviderCandidate(personProviderAddOptions{ + presetID: "openrouter", model: "explicit/model", credentialEnv: "EXACT_ROUTER_KEY", + retentionPosture: "operator_asserted", trainingPosture: "operator_asserted", + allowedSources: []string{"conversation_text"}, sourceSince: "2025-01-01", + }) + require.NoError(t, err) + assert.Equal("openrouter", candidate.PresetID) + assert.Equal("https://openrouter.ai/api/v1", candidate.Endpoint) + assert.Equal(peoplesweep.CredentialEnv, candidate.Credential) + assert.Equal("EXACT_ROUTER_KEY", candidate.CredentialEnv) + assert.Equal("operator_asserted", candidate.RetentionPosture) + assert.Equal("explicit/model", candidate.Model) +} + func TestPersonProviderAddRejectsLocalOptionConflictsBeforeCatalogOrState(t *testing.T) { tests := []struct { name string @@ -1233,36 +1274,100 @@ func providerSetupCommandDeps( return deps } -// TestPersonProviderDefaultDependenciesResolveCredentialsAfterConfigLoad -// reproduces the real command lifecycle: the provider command and its default -// dependencies are constructed during init, before PersistentPreRunE loads the -// selected config. Stored credentials must therefore resolve from the live -// config at execution time rather than from the init-time nil cfg. +func newPersonProviderInvocationTestRoot() *cobra.Command { + root := &cobra.Command{Use: "msgvault", PersistentPreRunE: func(command *cobra.Command, _ []string) error { + state := prepareInvocation(command) + loadedConfig, err := config.Load(state.options.cfgFile, state.options.homeDir) + if err != nil { + return err + } + state.cfg = loadedConfig + return nil + }} + registerRootFlags(root) + registerAgentFlags(root) + return root +} + +// Construct dependencies before config load to exercise invocation binding. func TestPersonProviderDefaultDependenciesResolveCredentialsAfterConfigLoad(t *testing.T) { newAssert := assert.New assert := assert.New(t) require := require.New(t) - requireStoredCredentialStorePlatform(t) - previousConfig := cfg - previousConfigFile := cfgFile - previousHomeDir := homeDir - previousLogger := logger - previousLogResult := logResult - t.Cleanup(func() { - if logResult != nil && logResult != previousLogResult { - logResult.Close() - } - cfg = previousConfig - cfgFile = previousConfigFile - homeDir = previousHomeDir - logger = previousLogger - logResult = previousLogResult + path, _ := providerSetupConfigFile(t) + t.Setenv("LIVE_PROVIDER_KEY", providerSetupSecretCanary) + deps := defaultPersonProviderCommandDeps() + + st := testutil.NewSQLiteTestStore(t) + checker := &fixedPersonProviderChecker{response: peoplesweep.StructuredResponse{ + Output: []byte(`{"ok":true}`), ProviderVersion: peoplesweep.OpenAIChatProviderVersion, + ModelVersion: "live-model-v1", + }} + deps.openStore = func() (personProviderStore, func(), error) { return st, func() {}, nil } + deps.openReadStore = func() (personProviderStore, func(), error) { return st, func() {}, nil } + deps.newChecker = func(peoplesweep.Config, personProviderStore, personProviderSetupDeps) (personProviderChecker, error) { + return checker, nil + } + deps.isDaemonSubprocess = func() bool { return true } + deps.setup.catalog = nil + deps.setup.negotiate = func( + _ context.Context, + _ peoplesweep.ProviderConfig, + credential peoplesweep.Credential, + ) (peoplesweep.NegotiatedCapabilities, error) { + assert := newAssert(t) + assert.Equal(providerSetupSecretCanary, credential.Value()) + return peoplesweep.NegotiatedCapabilities{ + OutputMode: peoplesweep.OutputModeJSONObject, TokenLimitParameter: "max_tokens", + DriverVersion: peoplesweep.OpenAIChatProviderVersion, + }, nil + } + + root := newPersonProviderInvocationTestRoot() + person := &cobra.Command{Use: "person"} + person.AddCommand(newPersonProviderCommand(deps)) + root.AddCommand(person) + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(&output) + root.SetArgs([]string{ + "--config", path, + "person", "provider", "add", "live-stored", "--custom", + "--protocol", "openai_chat", "--endpoint", "https://live.example.test/v1", + "--model", "live-model", "--auth", "bearer", "--credential-env", "LIVE_PROVIDER_KEY", + "--retention-posture", "zero_retention", "--training-posture", "no_training", + "--source", "conversation_text", "--source-since", "2025-01-01", "--yes", }) + require.NoError(root.ExecuteContext(t.Context())) + assert.NotContains(output.String(), providerSetupSecretCanary) + + configData, err := os.ReadFile(path) + require.NoError(err) + assert.NotContains(string(configData), providerSetupSecretCanary) + assert.Contains(string(configData), `credential = "env"`) + assert.Contains(string(configData), `credential_env = "LIVE_PROVIDER_KEY"`) + + deps.isDaemonSubprocess = func() bool { return false } + deps.providerStoreOwnedByDaemon = func(context.Context) (bool, error) { return false, nil } + removeRoot := newPersonProviderInvocationTestRoot() + removePerson := &cobra.Command{Use: "person"} + removePerson.AddCommand(newPersonProviderCommand(deps)) + removeRoot.AddCommand(removePerson) + removeRoot.SetOut(&output) + removeRoot.SetErr(&output) + removeRoot.SetArgs([]string{"--config", path, "person", "provider", "remove", "live-stored"}) + require.NoError(removeRoot.ExecuteContext(t.Context())) + assert.Contains(output.String(), `Removed people provider profile "live-stored"`) + configData, err = os.ReadFile(path) + require.NoError(err) + assert.NotContains(string(configData), "live-stored") +} + +func TestPersonProviderDefaultDependenciesResolveStoredCredentialsAfterConfigLoad(t *testing.T) { + requireStoredCredentialStorePlatform(t) + assert := assert.New(t) + require := require.New(t) path, loaded := providerSetupConfigFile(t) - cfg = nil - cfgFile = path - homeDir = "" - logResult = nil deps := defaultPersonProviderCommandDeps() st := testutil.NewSQLiteTestStore(t) @@ -1272,7 +1377,7 @@ func TestPersonProviderDefaultDependenciesResolveCredentialsAfterConfigLoad(t *t }} deps.openStore = func() (personProviderStore, func(), error) { return st, func() {}, nil } deps.openReadStore = func() (personProviderStore, func(), error) { return st, func() {}, nil } - deps.newChecker = func(peoplesweep.Config, personProviderStore) (personProviderChecker, error) { + deps.newChecker = func(peoplesweep.Config, personProviderStore, personProviderSetupDeps) (personProviderChecker, error) { return checker, nil } deps.isDaemonSubprocess = func() bool { return true } @@ -1282,7 +1387,6 @@ func TestPersonProviderDefaultDependenciesResolveCredentialsAfterConfigLoad(t *t _ peoplesweep.ProviderConfig, credential peoplesweep.Credential, ) (peoplesweep.NegotiatedCapabilities, error) { - assert := newAssert(t) assert.Equal(providerSetupSecretCanary, credential.Value()) return peoplesweep.NegotiatedCapabilities{ OutputMode: peoplesweep.OutputModeJSONObject, TokenLimitParameter: "max_tokens", @@ -1290,7 +1394,7 @@ func TestPersonProviderDefaultDependenciesResolveCredentialsAfterConfigLoad(t *t }, nil } - root := &cobra.Command{Use: "msgvault", PersistentPreRunE: rootCmd.PersistentPreRunE} + root := newPersonProviderInvocationTestRoot() person := &cobra.Command{Use: "person"} person.AddCommand(newPersonProviderCommand(deps)) root.AddCommand(person) @@ -1299,6 +1403,7 @@ func TestPersonProviderDefaultDependenciesResolveCredentialsAfterConfigLoad(t *t root.SetOut(&output) root.SetErr(&output) root.SetArgs([]string{ + "--config", path, "person", "provider", "add", "live-stored", "--custom", "--protocol", "openai_chat", "--endpoint", "https://live.example.test/v1", "--model", "live-model", "--auth", "bearer", "--api-key-stdin", @@ -1325,13 +1430,15 @@ func TestPersonProviderDefaultDependenciesResolveCredentialsAfterConfigLoad(t *t assert.NotContains(string(configData), providerSetupSecretCanary) assert.Contains(string(configData), `credential = "stored"`) - removeRoot := &cobra.Command{Use: "msgvault", PersistentPreRunE: rootCmd.PersistentPreRunE} + deps.isDaemonSubprocess = func() bool { return false } + deps.providerStoreOwnedByDaemon = func(context.Context) (bool, error) { return false, nil } + removeRoot := newPersonProviderInvocationTestRoot() removePerson := &cobra.Command{Use: "person"} removePerson.AddCommand(newPersonProviderCommand(deps)) removeRoot.AddCommand(removePerson) removeRoot.SetOut(&output) removeRoot.SetErr(&output) - removeRoot.SetArgs([]string{"person", "provider", "remove", "live-stored"}) + removeRoot.SetArgs([]string{"--config", path, "person", "provider", "remove", "live-stored"}) require.NoError(removeRoot.ExecuteContext(t.Context())) assert.Contains(output.String(), `Removed people provider profile "live-stored"`) _, err = peoplesweep.NewFileCredentialStore(loaded.TokensDir()).Load("live-stored") @@ -1342,6 +1449,27 @@ func TestPersonProviderDefaultDependenciesResolveCredentialsAfterConfigLoad(t *t assert.Equal(os.FileMode(0o600), tombstoneInfo.Mode().Perm()) } +func TestPersonProviderBoundCheckerUsesBoundSetup(t *testing.T) { + require := require.New(t) + cfg := config.NewDefaultConfig() + cfg.HomeDir = t.TempDir() + cfg.Data.DataDir = cfg.HomeDir + peopleConfig := personProviderTestConfig() + provider := peopleConfig.Providers["default"] + provider.Credential = peoplesweep.CredentialStored + provider.CredentialEnv = "" + peopleConfig.Providers["default"] = provider + cfg.People.Sweep = peopleConfig + ctx := withTestConfig(t, cfg) + + deps := defaultPersonProviderCommandDeps() + deps.setup.credentials = peoplesweep.NewFileCredentialStore(t.TempDir()) + deps = personProviderDepsForContext(ctx, deps) + checker, err := deps.newChecker(peopleConfig, testutil.NewSQLiteTestStore(t), deps.setup) + require.NoError(err) + require.NotNil(checker) +} + // TestPersonProviderAddCustomStdinKeepsSecretLocal catches an add operation // serializing a key into Cobra arguments, output, TOML, or config recovery // artifacts instead of publishing it only to the private credential store. @@ -1564,12 +1692,14 @@ func TestPersonProviderLifecycleJSONOutput(t *testing.T) { DaemonRestartRequired: true, }, used) + deps.isDaemonSubprocess = func() bool { return false } + deps.providerStoreOwnedByDaemon = func(context.Context) (bool, error) { return false, nil } removeRaw, err := executePersonProviderCommand(t, deps, "remove", "default", "--json") require.NoError(err) var removed personProviderRemoveOutput require.NoError(json.Unmarshal([]byte(removeRaw), &removed), removeRaw) assert.Equal(personProviderRemoveOutput{ - Name: "default", Removed: true, DaemonRestartRequired: true, + Name: "default", Removed: true, DaemonRestartRequired: false, }, removed) } @@ -1764,6 +1894,10 @@ func TestPersonProviderRemoveRevokesAndDeletesOnlyExactCredential(t *testing.T) require.NoError(err) _, err = st.EnsurePersonInferenceProfile(t.Context(), profile) require.NoError(err) + require.NoError(st.RecordPersonInferenceCheck(t.Context(), store.PersonInferenceCheck{ + ProfileFingerprint: profile.Fingerprint, CheckedAt: time.Now(), + DriverVersion: profile.DriverVersion, OutputMode: profile.OutputMode, ModelVersion: profile.Model, + })) _, _, err = st.GrantPersonInferenceConsent(t.Context(), profile.Fingerprint, "cli") require.NoError(err) deps := localPersonProviderDeps(loaded.People.Sweep, st, nil) @@ -1782,6 +1916,8 @@ func TestPersonProviderRemoveRevokesAndDeletesOnlyExactCredential(t *testing.T) } deps.setup.credentials = credentialStore + deps.isDaemonSubprocess = func() bool { return false } + deps.providerStoreOwnedByDaemon = func(context.Context) (bool, error) { return false, nil } output, err := executePersonProviderCommand(t, deps, "remove", "old") require.NoError(err) assert.Contains(output, "old") @@ -1796,6 +1932,9 @@ func TestPersonProviderRemoveRevokesAndDeletesOnlyExactCredential(t *testing.T) active, err := st.HasActivePersonInferenceConsent(t.Context(), profile.Fingerprint) require.NoError(err) assert.False(active) + checked, err := st.HasSuccessfulPersonInferenceCheck(t.Context(), profile.Fingerprint) + require.NoError(err) + assert.False(checked, "removal must discard the old credential's check") profiles, err := st.ListPersonInferenceProfiles(t.Context()) require.NoError(err) assert.Len(profiles, 1, "immutable audit profile must remain") @@ -1815,6 +1954,8 @@ func TestPersonProviderRemoveRevokesAndDeletesOnlyExactCredential(t *testing.T) activeDeps.restoreConfigFile = func(published, before config.ConfigFile) (config.ConfigFile, error) { return config.RestoreConfigFile(activePath, published, before) } + activeDeps.isDaemonSubprocess = func() bool { return false } + activeDeps.providerStoreOwnedByDaemon = func(context.Context) (bool, error) { return false, nil } _, err = executePersonProviderCommand(t, activeDeps, "remove", "default") require.ErrorContains(err, "active") } @@ -1876,6 +2017,8 @@ func TestPersonProviderRemoveUsesOneFreshConfigSnapshotForAllSideEffects(t *test } deps.setup.credentials = credentialStore + deps.isDaemonSubprocess = func() bool { return false } + deps.providerStoreOwnedByDaemon = func(context.Context) (bool, error) { return false, nil } _, err = executePersonProviderCommand(t, deps, "remove", "old") require.NoError(err) stillActive, err := st.HasActivePersonInferenceConsent(t.Context(), staleProfile.Fingerprint) @@ -1941,6 +2084,8 @@ func TestPersonProviderRemoveConfigConflictHasNoConsentOrCredentialSideEffects(t } deps.setup.credentials = credentialStore + deps.isDaemonSubprocess = func() bool { return false } + deps.providerStoreOwnedByDaemon = func(context.Context) (bool, error) { return false, nil } _, err = executePersonProviderCommand(t, deps, "remove", "old") require.ErrorIs(err, config.ErrConfigConflict) active, err := st.HasActivePersonInferenceConsent(t.Context(), profile.Fingerprint) diff --git a/cmd/msgvault/cmd/person_provider_test.go b/cmd/msgvault/cmd/person_provider_test.go index 6ecc00961..243bab20a 100644 --- a/cmd/msgvault/cmd/person_provider_test.go +++ b/cmd/msgvault/cmd/person_provider_test.go @@ -30,6 +30,7 @@ import ( "go.kenn.io/kit/daemon" "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/peoplesweep" + "go.kenn.io/msgvault/internal/personenrollment" "go.kenn.io/msgvault/internal/store" "go.kenn.io/msgvault/internal/testutil" "go.kenn.io/msgvault/internal/vector" @@ -144,7 +145,7 @@ func localPersonProviderDeps( openReadStore: func() (personProviderStore, func(), error) { return st, func() {}, nil }, - newChecker: func(peoplesweep.Config, personProviderStore) (personProviderChecker, error) { + newChecker: func(peoplesweep.Config, personProviderStore, personProviderSetupDeps) (personProviderChecker, error) { return checker, nil }, isDaemonSubprocess: func() bool { return true }, @@ -856,6 +857,10 @@ func TestPersonProviderRevokeIsIdempotent(t *testing.T) { st := testutil.NewSQLiteTestStore(t) _, err = st.EnsurePersonInferenceProfile(t.Context(), profile) require.NoError(err) + require.NoError(st.RecordPersonInferenceCheck(t.Context(), store.PersonInferenceCheck{ + ProfileFingerprint: profile.Fingerprint, CheckedAt: time.Now(), + DriverVersion: profile.DriverVersion, OutputMode: profile.OutputMode, ModelVersion: profile.Model, + })) _, _, err = st.GrantPersonInferenceConsent(t.Context(), profile.Fingerprint, "cli") require.NoError(err) deps := localPersonProviderDeps(config, st, nil) @@ -865,6 +870,7 @@ func TestPersonProviderRevokeIsIdempotent(t *testing.T) { var firstStatus personProviderStatusOutput require.NoError(json.Unmarshal([]byte(first), &firstStatus)) assert.False(firstStatus.Consent.Active) + assert.NotNil(firstStatus.Check, "ordinary consent revocation preserves the successful check") require.NotNil(firstStatus.Consent.LastRevoked) assert.Equal("cli", *firstStatus.Consent.LastRevoked.RevokedBy) @@ -966,7 +972,7 @@ func TestPersonProviderCheckAcceptsAProfileName(t *testing.T) { ModelVersion: "beta-model-v1", }} deps := localPersonProviderDeps(config, st, checker) - deps.newChecker = func(got peoplesweep.Config, _ personProviderStore) (personProviderChecker, error) { + deps.newChecker = func(got peoplesweep.Config, _ personProviderStore, _ personProviderSetupDeps) (personProviderChecker, error) { assert := newAssert(t) assert.Equal("beta", got.Provider.Name) return checker, nil @@ -1256,6 +1262,11 @@ func personProviderMutationNoticeFixture( return config.RestoreConfigFile(path, published, before) }, configHomeDir: func() string { return filepath.Dir(path) }, + removeWithDaemon: func(ctx context.Context, name, etag string) error { + record("daemon-remove") + _, err := personenrollment.NewService(path, st).RemoveProfile(ctx, etag, name, "", "cli", nil) + return err + }, proxy: func(*cobra.Command, []string, map[string]string) error { record("proxy") return nil @@ -1266,10 +1277,9 @@ func personProviderMutationNoticeFixture( // TestPersonProviderUseAndRemoveRecommendDaemonRestartWhenDaemonKeepsStartupConfig // pins the local half of the mutation boundary: a successful local config -// mutation must tell the operator that a running daemon keeps the people -// sweep configuration captured at startup, while the same mutation without -// a running daemon stays silent. The daemon subprocess case keeps the -// notice too: the parent daemon scheduled its sweep from startup config. +// mutation reports a pending restart only when a daemon keeps its startup +// config. Removal refuses subprocess execution because it needs the parent +// daemon's running policy, and uses the Settings operation from the frontend. func TestPersonProviderUseAndRemoveRecommendDaemonRestartWhenDaemonKeepsStartupConfig(t *testing.T) { scenarios := []struct { name string @@ -1295,6 +1305,12 @@ func TestPersonProviderUseAndRemoveRecommendDaemonRestartWhenDaemonKeepsStartupC deps, events := personProviderMutationNoticeFixture( t, scenario.isDaemonSubprocess, scenario.daemonOwnsStore) output, err := executePersonProviderCommand(t, deps, operation.verb, "beta") + if operation.verb == "remove" && scenario.isDaemonSubprocess { + require.ErrorContains(err, "cannot identify the running people provider policy") + assert.NotContains(*events, "store") + assert.NotContains(*events, "edit") + return + } require.NoError(err) assert.Contains(output, operation.success) if scenario.wantNotice { @@ -1304,10 +1320,8 @@ func TestPersonProviderUseAndRemoveRecommendDaemonRestartWhenDaemonKeepsStartupC assert.NotContains(output, "restart") } if operation.verb == "remove" { - if scenario.isDaemonSubprocess { - assert.Contains(*events, "store") - } else if scenario.daemonOwnsStore { - assert.Contains(*events, "proxy") + if scenario.daemonOwnsStore { + assert.Contains(*events, "daemon-remove") } else { assert.Contains(*events, "store") assert.NotContains(*events, "proxy") @@ -1321,13 +1335,13 @@ func TestPersonProviderUseAndRemoveRecommendDaemonRestartWhenDaemonKeepsStartupC // TestPersonProviderUseAndRemoveNoticeLiveIncompatibleDaemon pins the // restart guidance for a daemon that responds but fails the API // compatibility check. The compatibility-sensitive ownership signal finds -// no daemon, so the mutation stays local and nothing is proxied to the -// incompatible daemon, yet that live process still serves the people sweep -// config it captured at startup: use and remove must recommend a daemon -// restart instead of silently letting scheduled sweeps keep the stale -// selection. The daemon is faked with the same responding ping endpoint and -// runtime record pattern the restore-into-live-home guard tests use. +// no daemon, yet that live process still serves its startup policy. Use +// recommends a restart. Removal must refuse because it cannot revoke the +// running policy through the incompatible daemon. The test uses a responding +// ping endpoint and the runtime record pattern from the restore guards. func TestPersonProviderUseAndRemoveNoticeLiveIncompatibleDaemon(t *testing.T) { + cfg := testConfigValue() + newAssert := assert.New newRequire := require.New require := require.New(t) @@ -1366,6 +1380,8 @@ func TestPersonProviderUseAndRemoveNoticeLiveIncompatibleDaemon(t *testing.T) { savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = &config.Config{Data: config.DataConfig{DataDir: dataDir}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx defaults := defaultPersonProviderCommandDeps() for _, operation := range []struct { @@ -1385,7 +1401,14 @@ func TestPersonProviderUseAndRemoveNoticeLiveIncompatibleDaemon(t *testing.T) { return defaultOwnership(ctx) } deps.daemonAliveForRestartNotice = defaults.daemonAliveForRestartNotice - output, err := executePersonProviderCommand(t, deps, operation.verb, "beta") + output, err := executePersonProviderCommandContext(testCtx, t, deps, operation.verb, "beta") + if operation.verb == "remove" { + require.ErrorContains(err, "cannot identify the running people provider policy") + assert.NotContains(*events, "store") + assert.NotContains(*events, "edit") + assert.NotContains(*events, "daemon-remove") + return + } require.NoError(err) assert.Contains(output, operation.success) assert.Contains(output, "running daemon") @@ -1543,7 +1566,7 @@ func unreleasedCodexCommandDeps( ) personProviderCommandDeps { t.Helper() deps := localPersonProviderDeps(config, st, nil) - deps.newChecker = func(config peoplesweep.Config, st personProviderStore) (personProviderChecker, error) { + deps.newChecker = func(config peoplesweep.Config, st personProviderStore, _ personProviderSetupDeps) (personProviderChecker, error) { registry, err := peoplesweep.NewDriverRegistry(nil, starter, peoplesweep.NewReleasedCodexIsolationGate()) if err != nil { return nil, err @@ -1551,7 +1574,7 @@ func unreleasedCodexCommandDeps( return peoplesweep.NewRunner(config, st, registry, peoplesweep.NewCredentialResolver(nil, os.LookupEnv)) } - deps.newCodexClient = func(config peoplesweep.Config) (personProviderCodexClient, error) { + deps.newCodexClient = func(config peoplesweep.Config, _ personProviderSetupDeps) (personProviderCodexClient, error) { provider := configuredPersonProvider(config) registry, err := peoplesweep.NewDriverRegistry(nil, starter, peoplesweep.NewReleasedCodexIsolationGate()) if err != nil { @@ -1785,7 +1808,7 @@ func commandCodexScript( ) func(*bufio.Reader, io.Writer) error { t.Helper() return func(reader *bufio.Reader, writer io.Writer) error { - for id, want := range []string{"initialize", operation} { + for step, want := range []string{"initialize", "initialized", operation} { line, err := reader.ReadBytes('\n') if err != nil { return fmt.Errorf("read command Codex request: %w", err) @@ -1799,14 +1822,24 @@ func commandCodexScript( return err } *methods = append(*methods, request.Method) - if request.Method != want || request.ID != int64(id+1) { + wantID := int64(0) + switch step { + case 0: + wantID = 1 + case 2: + wantID = 2 + } + if request.Method != want || request.ID != wantID { return errors.New("unexpected Codex command transcript") } - if operation == "account/login/start" && id == 1 { + if step == 1 { + continue + } + if operation == "account/login/start" && step == 2 { assert.Equal(t, "chatgptDeviceCode", request.Params["type"]) } response := map[string]any{"id": request.ID, "result": map[string]any{}} - if id == 1 { + if step == 2 { response["result"] = result } encoded, err := json.Marshal(response) @@ -1816,7 +1849,7 @@ func commandCodexScript( if _, err := writer.Write(append(encoded, '\n')); err != nil { return err } - if operation == "account/login/start" && id == 1 { + if operation == "account/login/start" && step == 2 { completed, err := json.Marshal(map[string]any{ "method": "account/login/completed", "params": map[string]any{"success": true, "loginId": result["loginId"]}, @@ -1845,7 +1878,7 @@ func codexCommandDeps( opens.Add(1) return nil, func() {}, errors.New("archive store must not be opened") } - deps.newCodexClient = func(config peoplesweep.Config) (personProviderCodexClient, error) { + deps.newCodexClient = func(config peoplesweep.Config, _ personProviderSetupDeps) (personProviderCodexClient, error) { return peoplesweep.NewCodexAppServerDriver( configuredPersonProvider(config), starter, commandCodexGate{}, ) @@ -1873,7 +1906,7 @@ func TestPersonProviderLoginUsesDeviceCode(t *testing.T) { checks.Contains(output, "ABCD-1234") checks.Contains(output, "2026-08-23T12:30:00Z") checks.NotContains(output, "login-safe") - checks.Equal([]string{"initialize", "account/login/start"}, methods) + checks.Equal([]string{"initialize", "initialized", "account/login/start"}, methods) checks.Zero(opens.Load()) } @@ -1902,7 +1935,7 @@ func TestPersonProviderModelsListsSupportedEfforts(t *testing.T) { checks.Contains(output, "Test Model") checks.Contains(output, "medium") checks.Contains(output, "low, medium") - checks.Equal([]string{"initialize", "model/list"}, methods) + checks.Equal([]string{"initialize", "initialized", "model/list"}, methods) checks.Zero(opens.Load()) } diff --git a/cmd/msgvault/cmd/person_relationship_test.go b/cmd/msgvault/cmd/person_relationship_test.go index a003aff81..a0b48d8ce 100644 --- a/cmd/msgvault/cmd/person_relationship_test.go +++ b/cmd/msgvault/cmd/person_relationship_test.go @@ -53,7 +53,7 @@ func TestPersonRelationshipAddPostsTheDeclaredEdge(t *testing.T) { assert.NoError(err) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) @@ -65,6 +65,7 @@ func TestPersonRelationshipAddPostsTheDeclaredEdge(t *testing.T) { command := &cobra.Command{ Use: personRelationshipAddCmd.Use, Args: personRelationshipAddCmd.Args, RunE: personRelationshipAddCmd.RunE, } + command.SetContext(testCtx) command.SetOut(&output) command.SetArgs([]string{"3", "parent", "4"}) @@ -107,7 +108,7 @@ func TestPersonRelationshipListRendersBothDirections(t *testing.T) { assert.NoError(err) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) @@ -119,6 +120,7 @@ func TestPersonRelationshipListRendersBothDirections(t *testing.T) { command := &cobra.Command{ Use: personRelationshipListCmd.Use, Args: personRelationshipListCmd.Args, RunE: personRelationshipListCmd.RunE, } + command.SetContext(testCtx) command.SetOut(&output) command.SetArgs([]string{"3"}) @@ -164,7 +166,7 @@ func TestPersonRelationshipEndSendsIfMatchFromTheCurrentRevision(t *testing.T) { } })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) @@ -176,6 +178,7 @@ func TestPersonRelationshipEndSendsIfMatchFromTheCurrentRevision(t *testing.T) { command := &cobra.Command{ Use: personRelationshipEndCmd.Use, Args: personRelationshipEndCmd.Args, RunE: personRelationshipEndCmd.RunE, } + command.SetContext(testCtx) command.SetOut(&output) command.SetArgs([]string{"11", "2023-05"}) @@ -218,7 +221,7 @@ func TestRelationshipTypeUpdateUsesCurrentRevisionETag(t *testing.T) { } })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) @@ -235,6 +238,7 @@ func TestRelationshipTypeUpdateUsesCurrentRevisionETag(t *testing.T) { command := &cobra.Command{ Use: relationshipTypeUpdateCmd.Use, Args: relationshipTypeUpdateCmd.Args, RunE: relationshipTypeUpdateCmd.RunE, } + command.SetContext(testCtx) command.Flags().AddFlagSet(relationshipTypeUpdateCmd.Flags()) command.SetOut(&output) command.SetArgs([]string{"9", "--forward-label", "guide"}) @@ -254,13 +258,14 @@ func TestRelationshipTypeCreateRejectsAnEmptyReverseLabel(t *testing.T) { called = true })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) command := &cobra.Command{ Use: relationshipTypeCreateCmd.Use, Args: relationshipTypeCreateCmd.Args, RunE: relationshipTypeCreateCmd.RunE, } + command.SetContext(testCtx) command.SetOut(&bytes.Buffer{}) command.SetErr(&bytes.Buffer{}) command.SetArgs([]string{"mentor", "mentor", " "}) diff --git a/cmd/msgvault/cmd/person_scoring.go b/cmd/msgvault/cmd/person_scoring.go new file mode 100644 index 000000000..21f4b007d --- /dev/null +++ b/cmd/msgvault/cmd/person_scoring.go @@ -0,0 +1,186 @@ +package cmd + +import ( + "encoding/json/jsontext" + "encoding/json/v2" + "errors" + "fmt" + "strings" + + "github.com/spf13/cobra" + "go.kenn.io/msgvault/pkg/client/generated" +) + +func newPersonScoringCommand() *cobra.Command { + command := &cobra.Command{Use: "scoring", Short: "Inspect and run optional identity scoring"} + command.AddCommand(newPersonScoringStatusCommand(), + newPersonScoringConsentCommand("consent"), newPersonScoringConsentCommand("revoke"), + newPersonScoringRunCommand(), newPersonScoringHistoryCommand()) + return command +} + +func newPersonScoringStatusCommand() *cobra.Command { + var jsonOutput bool + command := &cobra.Command{Use: "status", Short: "Show scoring readiness and exact disclosure", Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + client, _, err := OpenHTTPStore(cmd.Context()) + if err != nil { + return err + } + defer func() { _ = client.Close() }() + status, err := client.GetIdentityScoringStatus(cmd.Context()) + if err != nil { + return err + } + if jsonOutput { + return writePersonScoringJSON(cmd, status) + } + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Enabled: %t\nReady: %t\nConsent active: %t\nCredential available: %t\n", + status.Enabled, status.Ready, status.ConsentActive, status.CredentialAvailable) + if status.DisclosureFingerprint != nil { + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Disclosure fingerprint: %s\n", *status.DisclosureFingerprint) + } + if status.Disclosure != nil { + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Provider: %s\nModel: %s\nPacket: %s\nRetention: %s\nPolicy: %s\n", + status.Disclosure.Endpoint, status.Disclosure.ModelID, status.Disclosure.PacketSchema, + status.Disclosure.RetentionDeclaration, status.Disclosure.PolicyVersion) + } + if status.Blocker != nil { + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Scoring blocker: %s\n", *status.Blocker) + } + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Data sent: %s\n", status.DataFields) + return nil + }} + command.Flags().BoolVar(&jsonOutput, "json", false, "Print complete status JSON") + return command +} + +func newPersonScoringConsentCommand(action string) *cobra.Command { + var jsonOutput bool + short := "Consent to the exact current disclosure" + if action == "revoke" { + short = "Revoke consent for the exact current disclosure" + } + command := &cobra.Command{Use: action + " ", + Short: short, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + fingerprint := strings.TrimSpace(args[0]) + if fingerprint == "" { + return usageErr(cmd, errors.New("disclosure fingerprint is required; run status first")) + } + client, _, err := OpenHTTPStore(cmd.Context()) + if err != nil { + return err + } + defer func() { _ = client.Close() }() + var result *generated.PersonMatchConsentDecisionResponse + if action == "consent" { + result, err = client.GrantIdentityScoringConsent(cmd.Context(), fingerprint) + } else { + result, err = client.RevokeIdentityScoringConsent(cmd.Context(), fingerprint) + } + if err != nil { + return err + } + if jsonOutput { + return writePersonScoringJSON(cmd, result) + } + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Consent active: %t\nChanged: %t\nDisclosure fingerprint: %s\n", + result.ConsentActive, result.Changed, result.DisclosureFingerprint) + return nil + }} + command.Flags().BoolVar(&jsonOutput, "json", false, "Print complete decision JSON") + return command +} + +func newPersonScoringRunCommand() *cobra.Command { + var jsonOutput bool + var limit int64 + command := &cobra.Command{Use: "run", Short: "Create review suggestions and journal one bounded scoring batch", Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + if limit < 0 { + return usageErr(cmd, errors.New("--limit must be positive")) + } + var limitValue *int64 + if limit > 0 { + limitValue = &limit + } + client, _, err := OpenHTTPStore(cmd.Context()) + if err != nil { + return err + } + defer func() { _ = client.Close() }() + response, err := client.ScoreIdentityMatches(cmd.Context(), limitValue) + if err != nil { + return err + } + if jsonOutput { + if err := writePersonScoringJSON(cmd, response); err != nil { + return err + } + } else { + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Processed: %d\n", response.Processed) + for _, row := range response.Results { + score := "n/a" + if row.Probability != nil { + score = fmt.Sprintf("%.3f", *row.Probability) + } + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Candidate %d: score=%s action=%s status=%s blockers=%s\n", + row.CandidateID, score, row.ProposedAction, row.Status, strings.Join(row.Blockers, ",")) + _, _ = fmt.Fprintf(cmd.OutOrStdout(), " Review token: %s\n", row.ReviewToken) + } + } + if response.ErrorData != nil { + return fmt.Errorf("%s: %s", response.ErrorData.Code, response.ErrorData.Message) + } + return nil + }} + command.Flags().Int64Var(&limit, "limit", 0, "Maximum candidates (defaults to configured batch size)") + command.Flags().BoolVar(&jsonOutput, "json", false, "Print complete scoring JSON") + return command +} + +func newPersonScoringHistoryCommand() *cobra.Command { + var candidateID, limit, beforeID int64 + var jsonOutput bool + command := &cobra.Command{Use: "history", Short: "List redacted scoring judgments", Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + if candidateID < 0 || beforeID < 0 || limit < 1 || limit > 100 { + return usageErr(cmd, errors.New("--candidate-id and --before-id must be nonnegative; --limit must be 1–100")) + } + client, _, err := OpenHTTPStore(cmd.Context()) + if err != nil { + return err + } + defer func() { _ = client.Close() }() + response, err := client.ListIdentityJudgments(cmd.Context(), candidateID, limit, beforeID) + if err != nil { + return err + } + if jsonOutput { + return writePersonScoringJSON(cmd, response) + } + for _, row := range response.Judgments { + score := "n/a" + if row.Probability != nil { + score = fmt.Sprintf("%.3f", *row.Probability) + } + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "%d candidate=%d score=%s outcome=%s status=%s\n", + row.ID, row.CandidateID, score, row.Outcome, row.Status) + } + if response.NextBeforeID != nil { + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Next before ID: %d\n", *response.NextBeforeID) + } + return nil + }} + command.Flags().Int64Var(&candidateID, "candidate-id", 0, "Filter by candidate ID (default all)") + command.Flags().Int64Var(&limit, "limit", 100, "Maximum judgments (1–100)") + command.Flags().Int64Var(&beforeID, "before-id", 0, "Return older judgments below this ID") + command.Flags().BoolVar(&jsonOutput, "json", false, "Print complete journal JSON") + return command +} + +func writePersonScoringJSON(cmd *cobra.Command, value any) error { + return json.MarshalEncode(jsontext.NewEncoder(cmd.OutOrStdout()), value, json.Deterministic(true)) +} diff --git a/cmd/msgvault/cmd/person_scoring_test.go b/cmd/msgvault/cmd/person_scoring_test.go new file mode 100644 index 000000000..3dfaa9588 --- /dev/null +++ b/cmd/msgvault/cmd/person_scoring_test.go @@ -0,0 +1,132 @@ +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/config" +) + +func runPersonScoringCLI(ctx context.Context, t *testing.T, args ...string) (string, error) { + t.Helper() + root := &cobra.Command{Use: "msgvault", SilenceErrors: true, SilenceUsage: true} + localFlag := rootCmd.PersistentFlags().Lookup("local") + savedChanged := localFlag.Changed + t.Cleanup(func() { localFlag.Changed = savedChanged }) + root.PersistentFlags().AddFlag(localFlag) + person := &cobra.Command{Use: "person"} + person.AddCommand(newPersonScoringCommand()) + root.AddCommand(person) + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(io.Discard) + root.SetArgs(append([]string{"person", "scoring"}, args...)) + err := root.ExecuteContext(ctx) + return output.String(), err +} + +func TestPersonScoringCLIUsesDaemonScoringRoutes(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + type request struct{ method, path, query, body string } + requests := make(chan request, 8) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + requests <- request{r.Method, r.URL.Path, r.URL.RawQuery, string(body)} + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/api/v1/identity/scoring/status": + _, _ = io.WriteString(w, `{"enabled":true,"model_id":"jev-1.13.0","minimum_probability":0.8,"disclosure":{"endpoint":"https://api.typesafe.ai/v1/systemone","model_id":"jev-1.13.0","packet_schema":"person-match-packet-v1","retention_declaration":"fixture policy","policy_version":"person-match-policy-v1"},"disclosure_fingerprint":"fixture-fingerprint","credential_available":true,"consent_active":true,"ready":true,"data_fields":"fixture data fields"}`) + case "/api/v1/identity/scoring/consent", "/api/v1/identity/scoring/revoke": + _, _ = io.WriteString(w, `{"disclosure_fingerprint":"fixture-fingerprint","consent_active":true,"changed":true}`) + case "/api/v1/identity/scoring/run": + _, _ = io.WriteString(w, `{"results":[{"candidate_id":17,"review_token":"token-17","model_id":"jev-1.13.0","packet_schema":"person-match-packet-v1","policy_version":"person-match-policy-v1","evidence_classes":["email"],"probability":0.81,"proposed_action":"needs_review","blockers":["independent_identity_evidence_required"],"status":"scored"}],"processed":1}`) + case "/api/v1/identity/scoring/history": + _, _ = io.WriteString(w, `{"judgments":[],"limit":2,"candidate_id":17}`) + default: + http.Error(w, "unexpected route", http.StatusNotFound) + } + })) + t.Cleanup(server.Close) + ctx := withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) + + out, err := runPersonScoringCLI(ctx, t, "status", "--json") + require.NoError(err) + assert.Contains(out, `"disclosure_fingerprint":"fixture-fingerprint"`) + assert.Equal("/api/v1/identity/scoring/status", (<-requests).path) + + _, err = runPersonScoringCLI(ctx, t, "consent", "fixture-fingerprint") + require.NoError(err) + decision := <-requests + assert.Equal("/api/v1/identity/scoring/consent", decision.path) + var body map[string]any + require.NoError(json.Unmarshal([]byte(decision.body), &body)) + assert.Equal("fixture-fingerprint", body["disclosure_fingerprint"]) + + out, err = runPersonScoringCLI(ctx, t, "run", "--limit", "1", "--json") + require.NoError(err) + assert.Contains(out, `"candidate_id":17`) + assert.Equal("/api/v1/identity/scoring/run", (<-requests).path) + + out, err = runPersonScoringCLI(ctx, t, "run") + require.NoError(err) + assert.Contains(out, "Candidate 17:") + assert.Equal("/api/v1/identity/scoring/run", (<-requests).path) + + out, err = runPersonScoringCLI(ctx, t, "history", "--candidate-id", "17", "--limit", "2", "--before-id", "99", "--json") + require.NoError(err) + assert.Contains(out, `"judgments":[]`) + historyRequest := <-requests + assert.Equal("/api/v1/identity/scoring/history", historyRequest.path) + assert.Contains(historyRequest.query, "before_id=99") + + _, err = runPersonScoringCLI(ctx, t, "revoke", "fixture-fingerprint") + require.NoError(err) + assert.Equal("/api/v1/identity/scoring/revoke", (<-requests).path) +} + +func TestPersonScoringConsentHelpDescriptions(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + consentHelp, err := runPersonScoringCLI(t.Context(), t, "consent", "--help") + require.NoError(err) + assert.Contains(consentHelp, "Consent to the exact current disclosure") + assert.NotContains(consentHelp, "Consent consent") + + revokeHelp, err := runPersonScoringCLI(t.Context(), t, "revoke", "--help") + require.NoError(err) + assert.Contains(revokeHelp, "Revoke consent for the exact current disclosure") +} + +func TestPersonScoringCLIRetainsPartialResultsOnBatchError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/api/v1/identity/scoring/run", r.URL.Path) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"results":[{"candidate_id":17,"review_token":"token-17","proposed_action":"needs_review","blockers":[],"status":"scored"}],"processed":1,"error":{"code":"consent_required","message":"Scoring stopped because consent was withdrawn"}}`) + })) + t.Cleanup(server.Close) + ctx := withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) + for _, tc := range []struct { + name string + args []string + result string + }{ + {"text", []string{"run"}, "Candidate 17:"}, + {"json", []string{"run", "--json"}, `"candidate_id":17`}, + } { + t.Run(tc.name, func(t *testing.T) { + out, err := runPersonScoringCLI(ctx, t, tc.args...) + assert.Contains(t, out, tc.result) + assert.ErrorContains(t, err, "consent_required") + }) + } +} diff --git a/cmd/msgvault/cmd/person_search_test.go b/cmd/msgvault/cmd/person_search_test.go index b7a33df92..9ccd339b1 100644 --- a/cmd/msgvault/cmd/person_search_test.go +++ b/cmd/msgvault/cmd/person_search_test.go @@ -53,15 +53,14 @@ func TestPersonSearchCommandUsesGeneratedRouteWithoutOpeningLocalStore(t *testin blockedDataDir := filepath.Join(t.TempDir(), "not-a-directory") requirements.NoError(os.WriteFile(blockedDataDir, []byte("blocks direct database access"), 0o600)) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ HomeDir: blockedDataDir, Data: config.DataConfig{DataDir: blockedDataDir}, Remote: config.RemoteConfig{ URL: server.URL, APIKey: "synthetic-daemon-key", AllowInsecure: true, }, }) - - command, stdout, stderr := newPersonSearchTestCommand(t) + command, stdout, stderr := newPersonSearchTestCommand(testCtx, t) command.SetArgs([]string{"--limit", "2", "synthetic", "systems", "architect"}) requirements.NoError(command.Execute()) @@ -81,11 +80,10 @@ func TestPersonSearchCommandJSONHasStableExplicitOrderedShape(t *testing.T) { writePersonSearchCommandResponse(t, w) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{ + testCtx := withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{ URL: server.URL, AllowInsecure: true, }}) - - command, stdout, stderr := newPersonSearchTestCommand(t) + command, stdout, stderr := newPersonSearchTestCommand(testCtx, t) command.SetArgs([]string{"--json", "synthetic", "architect"}) require.NoError(t, command.Execute()) @@ -104,11 +102,10 @@ func TestPersonSearchCommandJSONPreservesEmptyResultsArray(t *testing.T) { assert.NoError(t, json.NewEncoder(w).Encode(map[string]any{"results": []any{}})) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{ + testCtx := withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{ URL: server.URL, AllowInsecure: true, }}) - - command, stdout, stderr := newPersonSearchTestCommand(t) + command, stdout, stderr := newPersonSearchTestCommand(testCtx, t) command.SetArgs([]string{"--json", "nobody"}) require.NoError(t, command.Execute()) @@ -141,14 +138,13 @@ func TestPersonSearchCommandPreservesNilDisplayNameAcrossOutputModes(t *testing. }})) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{ + testCtx := withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{ URL: server.URL, AllowInsecure: true, }}) - t.Run("JSON null and order", func(t *testing.T) { checks := assert.New(t) requirements := require.New(t) - command, stdout, stderr := newPersonSearchTestCommand(t) + command, stdout, stderr := newPersonSearchTestCommand(testCtx, t) command.SetArgs([]string{"--json", "synthetic"}) requirements.NoError(command.Execute()) @@ -162,7 +158,7 @@ func TestPersonSearchCommandPreservesNilDisplayNameAcrossOutputModes(t *testing. t.Run("human dash and order", func(t *testing.T) { checks := assert.New(t) requirements := require.New(t) - command, stdout, stderr := newPersonSearchTestCommand(t) + command, stdout, stderr := newPersonSearchTestCommand(testCtx, t) command.SetArgs([]string{"synthetic"}) requirements.NoError(command.Execute()) @@ -184,10 +180,9 @@ func TestPersonSearchCommandRejectsInvalidInputBeforeDaemonRequest(t *testing.T) writePersonSearchCommandResponse(t, w) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{ + testCtx := withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{ URL: server.URL, AllowInsecure: true, }}) - tests := []struct { name string args []string @@ -199,7 +194,7 @@ func TestPersonSearchCommandRejectsInvalidInputBeforeDaemonRequest(t *testing.T) } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - command, _, _ := newPersonSearchTestCommand(t) + command, _, _ := newPersonSearchTestCommand(testCtx, t) command.SetArgs(test.args) err := command.Execute() require.ErrorContains(t, err, test.want) @@ -229,11 +224,11 @@ func TestPersonSearchCommandPropagatesDisabledAndStaleDaemonErrors(t *testing.T) })) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{ + testCtx := withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{ URL: server.URL, AllowInsecure: true, }}) - command, _, _ := newPersonSearchTestCommand(t) + command, _, _ := newPersonSearchTestCommand(testCtx, t) command.SetArgs([]string{"synthetic"}) err := command.Execute() requirements.Error(err) @@ -286,7 +281,8 @@ func TestDefaultVectorConfigBlocksCuratedPeopleButKeepsMessageEmbedding(t *testi configured.Vector.Embeddings.Model = "synthetic-default-policy-model" configured.Vector.Embeddings.Dimension = 4 configured.Vector.Embeddings.MaxRetries = 1 - withTestConfig(t, configured) + testCtx := withTestConfig(t, configured) + _ = testCtx requirements.NoError(configured.Save()) mainStore, err := store.Open(mainPath) @@ -307,27 +303,27 @@ func TestDefaultVectorConfigBlocksCuratedPeopleButKeepsMessageEmbedding(t *testi "email", "default-policy@example.test", "Observed Synthetic Name", ) requirements.NoError(err) - person, _, err := mainStore.CreatePersonFromParticipantContext(t.Context(), participantID) + person, _, err := mainStore.CreatePersonFromParticipantContext(testCtx, participantID) requirements.NoError(err) displayName := "Synthetic Curated Person" person, err = mainStore.UpdatePersonDisplayNameContext( - t.Context(), person.ID, person.Revision, &displayName, + testCtx, person.ID, person.Revision, &displayName, ) requirements.NoError(err) - features, err := setupVectorFeatures(t.Context(), mainStore, mainPath, false) + features, err := setupVectorFeatures(testCtx, mainStore, mainPath, false) requirements.NoError(err) requirements.NotNil(features) t.Cleanup(func() { _ = features.Close() }) generation, err := features.Backend.CreateGeneration( - t.Context(), features.Cfg.Embeddings.Model, + testCtx, features.Cfg.Embeddings.Model, features.Cfg.Embeddings.Dimension, features.Cfg.GenerationFingerprint(), ) requirements.NoError(err) - result, err := features.Runner.RunOnce(t.Context(), generation, testCLIEmbeddingPassScope()) + result, err := features.Runner.RunOnce(testCtx, generation, testCLIEmbeddingPassScope()) requirements.NoError(err) assertions.Equal(1, result.Succeeded, "message embedding must continue while people are disabled") - convergence, err := features.Convergence.CheckConvergence(t.Context(), generation) + convergence, err := features.Convergence.CheckConvergence(testCtx, generation) requirements.NoError(err) assertions.True(convergence.PersonCoverageComplete, "disabled person embeddings must not block a message generation") @@ -338,8 +334,8 @@ func TestDefaultVectorConfigBlocksCuratedPeopleButKeepsMessageEmbedding(t *testi assertions.Equal([]string{"Subject: Synthetic message subject\n\nSynthetic message body"}, inputsAfterWorker, "default vector config must not send curated person data") - requirements.NoError(features.Backend.ActivateGeneration(t.Context(), generation, false)) - _, err = features.PersonSearchEngine.Search(t.Context(), "synthetic person", 5) + requirements.NoError(features.Backend.ActivateGeneration(testCtx, generation, false)) + _, err = features.PersonSearchEngine.Search(testCtx, "synthetic person", 5) requirements.ErrorContains(err, "[vector.people] enabled = true") providerMu.Lock() assertions.Equal(inputsAfterWorker, providerInputs, @@ -350,10 +346,10 @@ func TestDefaultVectorConfigBlocksCuratedPeopleButKeepsMessageEmbedding(t *testi Enabled: true, RetentionPosture: "zero_data_retention", TrainingPosture: "no_training", } requirements.NoError(configured.Save()) - result, err = features.Runner.RunOnce(t.Context(), generation, testCLIEmbeddingPassScope()) + result, err = features.Runner.RunOnce(testCtx, generation, testCLIEmbeddingPassScope()) requirements.NoError(err) assertions.Zero(result.Succeeded, "unconsented people must be skipped without blocking messages") - _, err = features.PersonSearchEngine.Search(t.Context(), "synthetic person", 5) + _, err = features.PersonSearchEngine.Search(testCtx, "synthetic person", 5) requirements.ErrorIs(err, vector.ErrSemanticPersonEmbeddingConsentRequired) providerMu.Lock() assertions.Equal(inputsAfterWorker, providerInputs, @@ -362,16 +358,16 @@ func TestDefaultVectorConfigBlocksCuratedPeopleButKeepsMessageEmbedding(t *testi semanticProfile, err := configured.Vector.SemanticPersonEmbeddingProfile() requirements.NoError(err) - _, err = mainStore.EnsurePersonSemanticEmbeddingProfile(t.Context(), semanticProfile) + _, err = mainStore.EnsurePersonSemanticEmbeddingProfile(testCtx, semanticProfile) requirements.NoError(err) _, _, err = mainStore.GrantPersonSemanticEmbeddingConsent( - t.Context(), semanticProfile.Fingerprint, "test", + testCtx, semanticProfile.Fingerprint, "test", ) requirements.NoError(err) - result, err = features.Runner.RunOnce(t.Context(), generation, testCLIEmbeddingPassScope()) + result, err = features.Runner.RunOnce(testCtx, generation, testCLIEmbeddingPassScope()) requirements.NoError(err) assertions.Equal(1, result.Succeeded, "consented exact policy must run the person worker") - results, err := features.PersonSearchEngine.Search(t.Context(), "synthetic person", 5) + results, err := features.PersonSearchEngine.Search(testCtx, "synthetic person", 5) requirements.NoError(err) requirements.Len(results, 1) assertions.Equal(person.ID, results[0].Person.ID) @@ -380,17 +376,17 @@ func TestDefaultVectorConfigBlocksCuratedPeopleButKeepsMessageEmbedding(t *testi providerCallsAfterConsent := len(providerInputs) providerMu.Unlock() _, err = mainStore.RevokePersonSemanticEmbeddingConsent( - t.Context(), semanticProfile.Fingerprint, "test", + testCtx, semanticProfile.Fingerprint, "test", ) requirements.NoError(err) updatedName := "Synthetic Curated Person Updated" _, err = mainStore.UpdatePersonDisplayNameContext( - t.Context(), person.ID, person.Revision, &updatedName, + testCtx, person.ID, person.Revision, &updatedName, ) requirements.NoError(err) - _, err = features.Runner.RunOnce(t.Context(), generation, testCLIEmbeddingPassScope()) + _, err = features.Runner.RunOnce(testCtx, generation, testCLIEmbeddingPassScope()) requirements.NoError(err) - _, err = features.PersonSearchEngine.Search(t.Context(), "updated person", 5) + _, err = features.PersonSearchEngine.Search(testCtx, "updated person", 5) requirements.ErrorIs(err, vector.ErrSemanticPersonEmbeddingConsentRequired) providerMu.Lock() assertions.Len(providerInputs, providerCallsAfterConsent, @@ -402,6 +398,9 @@ func TestDefaultVectorConfigBlocksCuratedPeopleButKeepsMessageEmbedding(t *testi // catches deletion of a live config silently falling back to the authorized // startup snapshot. func TestCurrentSemanticPersonVectorConfigSourceFailsClosedAfterConfigRemoval(t *testing.T) { + cfg := testConfigValue() + _ = cfg + assertions := assert.New(t) requirements := require.New(t) configured := config.NewDefaultConfig() @@ -414,10 +413,11 @@ func TestCurrentSemanticPersonVectorConfigSourceFailsClosedAfterConfigRemoval(t configured.Vector.People = vector.PeopleConfig{ Enabled: true, RetentionPosture: "zero_data_retention", TrainingPosture: "no_training", } - withTestConfig(t, configured) + testCtx := withTestConfig(t, configured) + _ = testCtx requirements.NoError(configured.Save()) - source := currentSemanticPersonVectorConfigSource() + source := currentSemanticPersonVectorConfigSource(invocationFromContext(testCtx)) current, err := source() requirements.NoError(err) assertions.True(current.People.Enabled) @@ -428,7 +428,7 @@ func TestCurrentSemanticPersonVectorConfigSourceFailsClosedAfterConfigRemoval(t assertions.False(current.People.Enabled, "a missing live config must not reuse the authorized startup policy") - cfg = nil + invocationFromContext(testCtx).cfg = nil _, err = source() requirements.ErrorContains(err, "configuration is unavailable", "an absent runtime config must fail closed instead of using the startup policy") @@ -505,7 +505,7 @@ func TestCompletedBuildActivatesWithoutPersonRequestsAfterLivePolicyDrift(t *tes configured.Vector.People = vector.PeopleConfig{ Enabled: true, RetentionPosture: "zero_data_retention", TrainingPosture: "no_training", } - withTestConfig(t, configured) + testCtx := withTestConfig(t, configured) require.NoError(configured.Save()) mainStore, err := store.Open(mainPath) @@ -532,7 +532,7 @@ func TestCompletedBuildActivatesWithoutPersonRequestsAfterLivePolicyDrift(t *tes ) require.NoError(err) - features, err := setupVectorFeatures(t.Context(), mainStore, mainPath, false) + features, err := setupVectorFeatures(testCtx, mainStore, mainPath, false) require.NoError(err) t.Cleanup(func() { _ = features.Close() }) generation, err := features.Backend.CreateGeneration( @@ -569,6 +569,8 @@ func TestCompletedBuildActivatesWithoutPersonRequestsAfterLivePolicyDrift(t *tes // profile document through the provider and SQLite person index to the // authenticated generated client used by the CLI. func TestPersonSearchProductionCompositionDoesNotPublishReadyWithoutThePersonEngine(t *testing.T) { + useLocal := false + requirements := require.New(t) assertions := assert.New(t) @@ -615,7 +617,7 @@ func TestPersonSearchProductionCompositionDoesNotPublishReadyWithoutThePersonEng configured.Vector.People = vector.PeopleConfig{ Enabled: true, RetentionPosture: "zero_data_retention", TrainingPosture: "no_training", } - withTestConfig(t, configured) + testCtx := withTestConfig(t, configured) requirements.NoError(configured.Save()) savedUseLocal := useLocal useLocal = false @@ -627,41 +629,41 @@ func TestPersonSearchProductionCompositionDoesNotPublishReadyWithoutThePersonEng requirements.NoError(mainStore.InitSchema()) semanticProfile, err := configured.Vector.SemanticPersonEmbeddingProfile() requirements.NoError(err) - _, err = mainStore.EnsurePersonSemanticEmbeddingProfile(t.Context(), semanticProfile) + _, err = mainStore.EnsurePersonSemanticEmbeddingProfile(testCtx, semanticProfile) requirements.NoError(err) _, _, err = mainStore.GrantPersonSemanticEmbeddingConsent( - t.Context(), semanticProfile.Fingerprint, "test", + testCtx, semanticProfile.Fingerprint, "test", ) requirements.NoError(err) participantID, err := mainStore.EnsureParticipantByIdentifier( "email", "synthetic-architect@example.test", "Observed Synthetic Name", ) requirements.NoError(err) - person, created, err := mainStore.CreatePersonFromParticipantContext(t.Context(), participantID) + person, created, err := mainStore.CreatePersonFromParticipantContext(testCtx, participantID) requirements.NoError(err) requirements.True(created) displayName := "Synthetic Architect" person, err = mainStore.UpdatePersonDisplayNameContext( - t.Context(), person.ID, person.Revision, &displayName, + testCtx, person.ID, person.Revision, &displayName, ) requirements.NoError(err) - document, err := mainStore.LoadPersonSemanticDocumentContext(t.Context(), person.ID) + document, err := mainStore.LoadPersonSemanticDocumentContext(testCtx, person.ID) requirements.NoError(err) requirements.Contains(document.Text, displayName) requirements.NotContains(document.Text, "synthetic-architect@example.test") - features, err := setupVectorFeatures(t.Context(), mainStore, mainPath, false) + features, err := setupVectorFeatures(testCtx, mainStore, mainPath, false) requirements.NoError(err) requirements.NotNil(features) t.Cleanup(func() { _ = features.Close() }) requirements.NotNil(features.PersonSearchEngine, "runtime composition must build the concrete person engine") generation, err := features.Backend.CreateGeneration( - t.Context(), features.Cfg.Embeddings.Model, + testCtx, features.Cfg.Embeddings.Model, features.Cfg.Embeddings.Dimension, features.Cfg.GenerationFingerprint(), ) requirements.NoError(err) - requirements.NoError(features.Backend.ActivateGeneration(t.Context(), generation, false), + requirements.NoError(features.Backend.ActivateGeneration(testCtx, generation, false), "a pre-feature active generation can exist without person coverage") apiServer := api.NewServerWithOptions(api.ServerOptions{ @@ -675,7 +677,7 @@ func TestPersonSearchProductionCompositionDoesNotPublishReadyWithoutThePersonEng t.Cleanup(httpServer.Close) unauthorizedRequest, err := http.NewRequestWithContext( - context.Background(), http.MethodPost, httpServer.URL+"/api/v1/people/search", + testCtx, http.MethodPost, httpServer.URL+"/api/v1/people/search", strings.NewReader(`{"query":"architect"}`), ) requirements.NoError(err) @@ -687,7 +689,7 @@ func TestPersonSearchProductionCompositionDoesNotPublishReadyWithoutThePersonEng "semantic person search route remains protected") unindexedRequest, err := http.NewRequestWithContext( - context.Background(), http.MethodPost, httpServer.URL+"/api/v1/people/search", + testCtx, http.MethodPost, httpServer.URL+"/api/v1/people/search", strings.NewReader(`{"query":"architect"}`), ) requirements.NoError(err) @@ -707,14 +709,14 @@ func TestPersonSearchProductionCompositionDoesNotPublishReadyWithoutThePersonEng "an unindexed upgraded person corpus must not incur a query provider call") providerMu.Unlock() - result, err := features.Runner.RunOnce(t.Context(), generation, testCLIEmbeddingPassScope()) + result, err := features.Runner.RunOnce(testCtx, generation, testCLIEmbeddingPassScope()) requirements.NoError(err) assertions.Equal(1, result.Succeeded, "one curated person document embedded") configured.Remote = config.RemoteConfig{ URL: httpServer.URL, APIKey: configured.Server.APIKey, AllowInsecure: true, } - command, stdout, stderr := newPersonSearchTestCommand(t) + command, stdout, stderr := newPersonSearchTestCommand(testCtx, t) command.SetArgs([]string{"--json", "architect"}) requirements.NoError(command.Execute()) assertions.JSONEq(`{"results":[{"id":`+ @@ -728,7 +730,7 @@ func TestPersonSearchProductionCompositionDoesNotPublishReadyWithoutThePersonEng "canonical document and free-text query use the same configured provider") } -func newPersonSearchTestCommand(t *testing.T) (*cobra.Command, *bytes.Buffer, *bytes.Buffer) { +func newPersonSearchTestCommand(ctx context.Context, t *testing.T) (*cobra.Command, *bytes.Buffer, *bytes.Buffer) { t.Helper() savedLimit, savedJSON := personSearchLimit, personSearchJSON personSearchLimit, personSearchJSON = defaultPersonSearchLimit, false @@ -743,7 +745,7 @@ func newPersonSearchTestCommand(t *testing.T) (*cobra.Command, *bytes.Buffer, *b stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{} command.SetOut(stdout) command.SetErr(stderr) - command.SetContext(context.Background()) + command.SetContext(ctx) return command, stdout, stderr } diff --git a/cmd/msgvault/cmd/person_sweep.go b/cmd/msgvault/cmd/person_sweep.go index 5fb550751..f1194f960 100644 --- a/cmd/msgvault/cmd/person_sweep.go +++ b/cmd/msgvault/cmd/person_sweep.go @@ -12,6 +12,7 @@ import ( "time" "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/peoplesweep" "go.kenn.io/msgvault/internal/personfacts" "go.kenn.io/msgvault/internal/store" @@ -32,6 +33,7 @@ type personSweepCommandStore interface { } type personSweepCommandDeps struct { + bind func(context.Context) personSweepCommandDeps config func() peoplesweep.Config openStore func() (personSweepCommandStore, func(), error) newRunner func(peoplesweep.Config, personSweepCommandStore) (personSweepRunner, error) @@ -101,18 +103,45 @@ type personSweepHistoryOutput struct { func defaultPersonSweepCommandDeps() personSweepCommandDeps { return personSweepCommandDeps{ - config: func() peoplesweep.Config { return cfg.People.Sweep }, + bind: func(ctx context.Context) personSweepCommandDeps { + deps := defaultPersonSweepCommandDeps() + state := invocationFromContext(ctx) + var currentCfg *config.Config + if state != nil && state.cfg != nil { + currentCfg = state.cfg + } + deps.config = func() peoplesweep.Config { + if currentCfg == nil { + return peoplesweep.Config{} + } + return currentCfg.People.Sweep + } + deps.newRunner = func(sweepConfig peoplesweep.Config, commandStore personSweepCommandStore) (personSweepRunner, error) { + if currentCfg == nil { + return nil, errors.New("configuration is unavailable") + } + st, ok := commandStore.(*store.Store) + if !ok { + return nil, errors.New("people sweep production store is unavailable") + } + productionConfig := *currentCfg + productionConfig.People.Sweep = sweepConfig + return newProductionPersonSweepWorker(&productionConfig, st) + } + deps.openStore = func() (personSweepCommandStore, func(), error) { + return openWritableStoreAndInitForInvocation(state) + } + return deps + }, + config: func() peoplesweep.Config { return peoplesweep.Config{} }, openStore: func() (personSweepCommandStore, func(), error) { - return openWritableStoreAndInit() + return nil, nil, errors.New("configuration is unavailable") }, newRunner: func(config peoplesweep.Config, commandStore personSweepCommandStore) (personSweepRunner, error) { - st, ok := commandStore.(*store.Store) - if !ok { + if _, ok := commandStore.(*store.Store); !ok { return nil, errors.New("people sweep production store is unavailable") } - productionConfig := *cfg - productionConfig.People.Sweep = config - return newProductionPersonSweepWorker(&productionConfig, st) + return nil, errors.New("configuration is unavailable") }, isDaemonSubprocess: isDaemonCLISubprocess, lookupEnv: os.LookupEnv, @@ -139,6 +168,9 @@ func newPersonSweepRunCommand(deps personSweepCommandDeps) *cobra.Command { command := &cobra.Command{ Use: "run", Short: "Run bounded person maintenance", Args: cobra.NoArgs, RunE: func(command *cobra.Command, args []string) error { + if invocationFromContext(command.Context()) != nil && deps.bind != nil { + deps = deps.bind(command.Context()) + } config := deps.config() personSet := command.Flags().Changed("person") if personSet && personID <= 0 { @@ -200,6 +232,9 @@ func newPersonSweepStatusCommand(deps personSweepCommandDeps) *cobra.Command { command := &cobra.Command{ Use: statusValue, Short: "Show redacted person maintenance state", Args: cobra.NoArgs, RunE: func(command *cobra.Command, args []string) error { + if invocationFromContext(command.Context()) != nil && deps.bind != nil { + deps = deps.bind(command.Context()) + } if !deps.isDaemonSubprocess() { return deps.proxy(command, args, nil) } @@ -246,6 +281,9 @@ func newPersonSweepHistoryCommand(deps personSweepCommandDeps) *cobra.Command { command := &cobra.Command{ Use: "history", Short: "Show redacted person maintenance history", Args: cobra.NoArgs, RunE: func(command *cobra.Command, args []string) error { + if invocationFromContext(command.Context()) != nil && deps.bind != nil { + deps = deps.bind(command.Context()) + } if !deps.isDaemonSubprocess() { return deps.proxy(command, args, nil) } diff --git a/cmd/msgvault/cmd/person_test.go b/cmd/msgvault/cmd/person_test.go index df4aa2628..4e28815da 100644 --- a/cmd/msgvault/cmd/person_test.go +++ b/cmd/msgvault/cmd/person_test.go @@ -2,12 +2,15 @@ package cmd import ( "bytes" + "context" "encoding/json" "errors" "fmt" + "log/slog" "net/http" "net/http/httptest" "os" + "strconv" "sync/atomic" "testing" @@ -15,7 +18,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.kenn.io/kit/daemon" + "go.kenn.io/msgvault/internal/api" "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/testutil" "go.kenn.io/msgvault/pkg/client/generated" ) @@ -66,7 +71,7 @@ func TestPersonPromoteAcceptsCreatedResponse(t *testing.T) { assert.NoError(err) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) @@ -80,6 +85,7 @@ func TestPersonPromoteAcceptsCreatedResponse(t *testing.T) { Args: personPromoteCmd.Args, RunE: personPromoteCmd.RunE, } + command.SetContext(testCtx) command.SetOut(&output) command.SetArgs([]string{"42"}) @@ -122,7 +128,7 @@ func TestPersonSetDisplayNameClearSendsNull(t *testing.T) { } })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) @@ -138,6 +144,7 @@ func TestPersonSetDisplayNameClearSendsNull(t *testing.T) { Args: personSetDisplayNameCmd.Args, RunE: personSetDisplayNameCmd.RunE, } + command.SetContext(testCtx) command.Flags().BoolVar(&personClearDisplayName, "clear", false, "") command.SetOut(&output) command.SetArgs([]string{"7", "--clear"}) @@ -174,7 +181,7 @@ func TestPersonDeleteSendsIfMatchFromLatestRead(t *testing.T) { } })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) @@ -184,6 +191,7 @@ func TestPersonDeleteSendsIfMatchFromLatestRead(t *testing.T) { Args: personDeleteCmd.Args, RunE: personDeleteCmd.RunE, } + command.SetContext(testCtx) command.SetOut(&output) command.SetArgs([]string{"7"}) @@ -192,12 +200,13 @@ func TestPersonDeleteSendsIfMatchFromLatestRead(t *testing.T) { assert.Contains(output.String(), "Deleted person 7") } -func executePersonMergeCLI(t *testing.T, command *cobra.Command, args ...string) string { +func executePersonMergeCLI(ctx context.Context, t *testing.T, command *cobra.Command, args ...string) string { t.Helper() var output bytes.Buffer command.SetOut(&output) command.SetErr(&output) command.SetArgs(args) + command.SetContext(ctx) require.NoError(t, command.Execute(), output.String()) return output.String() } @@ -298,52 +307,51 @@ func TestPersonMergeCommandsUseConfiguredRemote(t *testing.T) { } })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) - - mergeJSONOutput := executePersonMergeCLI(t, newPersonMergeCommand(), "7", "9", + mergeJSONOutput := executePersonMergeCLI(testCtx, t, newPersonMergeCommand(), "7", "9", "--survivor-revision", "3", "--absorbed-revision", "2", "--idempotency-key", "remote-merge", "--json") assertions.Contains(mergeJSONOutput, `"review_candidates":[]`) - mergeOutput := executePersonMergeCLI(t, newPersonMergeCommand(), "7", "9", + mergeOutput := executePersonMergeCLI(testCtx, t, newPersonMergeCommand(), "7", "9", "--survivor-revision", "3", "--absorbed-revision", "2", "--idempotency-key", "remote-merge") assertions.Contains(mergeOutput, "Merge: 12") assertions.Contains(mergeOutput, "Absorbed UID: absorbed-uid") assertions.Contains(mergeOutput, "Identity revision: 42") assertions.Contains(mergeOutput, "Cache state: ready") - splitJSONOutput := executePersonMergeCLI(t, newPersonSplitCommand(), "7", "--merge-id", "12", + splitJSONOutput := executePersonMergeCLI(testCtx, t, newPersonSplitCommand(), "7", "--merge-id", "12", "--participant", "90", "--revision", "4", "--idempotency-key", "remote-split", "--json") assertions.Contains(splitJSONOutput, `"ambiguous_rows":[]`) - splitOutput := executePersonMergeCLI(t, newPersonSplitCommand(), "7", "--merge-id", "12", + splitOutput := executePersonMergeCLI(testCtx, t, newPersonSplitCommand(), "7", "--merge-id", "12", "--participant", "90", "--revision", "4", "--idempotency-key", "remote-split") assertions.Contains(splitOutput, "Split: 13") assertions.Contains(splitOutput, "Exact reversal: true") assertions.Contains(splitOutput, "Identity revision: 43") assertions.Contains(splitOutput, "Cache state: stale") - rootSplitOutput := executePersonMergeCLI(t, newPersonSplitCommand(), "7", "--merge-id", "12", + rootSplitOutput := executePersonMergeCLI(testCtx, t, newPersonSplitCommand(), "7", "--merge-id", "12", "--revision", "4", "--idempotency-key", "remote-root-split", "--json") assertions.Contains(rootSplitOutput, `"exact_reversal":true`) assertions.Equal([][]int64{{90}, {90}, nil}, splitParticipants) - executePersonMergeCLI(t, newPersonMergeHistoryCommand(), "7", "--json") - assertions.Contains(executePersonMergeCLI(t, newPersonMergeHistoryCommand(), "7"), "MERGE") - detailJSONOutput := executePersonMergeCLI(t, newPersonMergeShowCommand(), "12", "--json") + executePersonMergeCLI(testCtx, t, newPersonMergeHistoryCommand(), "7", "--json") + assertions.Contains(executePersonMergeCLI(testCtx, t, newPersonMergeHistoryCommand(), "7"), "MERGE") + detailJSONOutput := executePersonMergeCLI(testCtx, t, newPersonMergeShowCommand(), "12", "--json") for _, field := range []string{"participants", "rows", "splits", "review_candidates"} { assertions.Contains(detailJSONOutput, `"`+field+`":[]`) } - assertions.Contains(executePersonMergeCLI(t, newPersonMergeShowCommand(), "12"), "Merge: 12") - snapshot := executePersonMergeCLI(t, newPersonMergeShowCommand(), "12", "--snapshot", "--json") + assertions.Contains(executePersonMergeCLI(testCtx, t, newPersonMergeShowCommand(), "12"), "Merge: 12") + snapshot := executePersonMergeCLI(testCtx, t, newPersonMergeShowCommand(), "12", "--snapshot", "--json") assertions.JSONEq(`{"persons":[{"id":7}],"rows":{"person_names":[1]}}`, string(extractPersonMergeSnapshot(t, snapshot))) assertions.Contains( - executePersonMergeCLI(t, newPersonMergeShowCommand(), "12", "--snapshot"), + executePersonMergeCLI(testCtx, t, newPersonMergeShowCommand(), "12", "--snapshot"), `Snapshot: {"persons":[{"id":7}],"rows":{"person_names":[1]}}`) - executePersonMergeCLI(t, newPersonMergeCandidateCommand(), "21", + executePersonMergeCLI(testCtx, t, newPersonMergeCandidateCommand(), "21", "--person-id", "7", "--revision", "4", "--decision", "rejected", "--json") - candidateOutput := executePersonMergeCLI(t, newPersonMergeCandidateCommand(), "21", + candidateOutput := executePersonMergeCLI(testCtx, t, newPersonMergeCandidateCommand(), "21", "--person-id", "7", "--revision", "4", "--decision", "rejected") assertions.Contains(candidateOutput, "State: rejected") assertions.Contains(candidateOutput, `Person ETag: "person-7-r5"`) @@ -392,7 +400,7 @@ func TestPersonMergeCommandUsesExistingLocalDaemon(t *testing.T) { server := httptest.NewServer(mux) t.Cleanup(server.Close) dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) runtime := daemonRuntimeForHTTPServer(t, server, daemonAPIKeyFingerprint("")) _, err := daemonRuntimeStore(dataDir).Write(runtime.Record) require.NoError(t, err) @@ -402,14 +410,14 @@ func TestPersonMergeCommandUsesExistingLocalDaemon(t *testing.T) { return nil, errors.New("unreachable") }) - output := executePersonMergeCLI(t, newPersonMergeHistoryCommand(), "7", "--json") + output := executePersonMergeCLI(testCtx, t, newPersonMergeHistoryCommand(), "7", "--json") assert.JSONEq(t, `[]`, output) assert.Equal(t, 1, requests) } func TestPersonMergeCLIValidationHappensBeforeOpeningStore(t *testing.T) { dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) tests := []struct { name string command *cobra.Command @@ -432,6 +440,7 @@ func TestPersonMergeCLIValidationHappensBeforeOpeningStore(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { test.command.SetArgs(test.args) + test.command.SetContext(testCtx) err := test.command.Execute() require.ErrorContains(t, err, test.want) }) @@ -440,3 +449,36 @@ func TestPersonMergeCLIValidationHappensBeforeOpeningStore(t *testing.T) { require.NoError(t, err) assert.Empty(t, entries, "invalid commands must not initialize the archive") } + +func TestPersonPromoteSeedsNameThroughDaemonAdapter(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + st := testutil.NewTestStore(t) + id, err := st.EnsureParticipant("promotion@example.com", "Alex Example", "example.com") + require.NoError(err) + srv := api.NewServer(&config.Config{}, &storeAPIAdapter{store: st}, nil, slog.New(slog.DiscardHandler)) + server := httptest.NewServer(srv.Router()) + t.Cleanup(server.Close) + testCtx := withStoreResolverConfig(t, &config.Config{Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}}) + savedJSON := personJSON + personJSON = false + t.Cleanup(func() { personJSON = savedJSON }) + var output bytes.Buffer + promote := &cobra.Command{Use: personPromoteCmd.Use, Args: personPromoteCmd.Args, RunE: personPromoteCmd.RunE} + promote.SetOut(&output) + promote.SetArgs([]string{strconv.FormatInt(id, 10)}) + require.NoError(promote.ExecuteContext(testCtx)) + assert.Contains(output.String(), "Display name: Alex Example") + person, err := st.PersonForParticipants([]int64{id}) + require.NoError(err) + require.NotNil(person) + directory, err := runPersonDirectoryCommand(testCtx, t) + require.NoError(err) + assert.Contains(directory, "Alex Example") + output.Reset() + get := &cobra.Command{Use: personGetCmd.Use, Args: personGetCmd.Args, RunE: personGetCmd.RunE} + get.SetOut(&output) + get.SetArgs([]string{strconv.FormatInt(person.ID, 10)}) + require.NoError(get.ExecuteContext(testCtx)) + assert.Contains(output.String(), "Display name: Alex Example") +} diff --git a/cmd/msgvault/cmd/person_tracking_test.go b/cmd/msgvault/cmd/person_tracking_test.go index 036ca6940..ccb8ffa39 100644 --- a/cmd/msgvault/cmd/person_tracking_test.go +++ b/cmd/msgvault/cmd/person_tracking_test.go @@ -2,6 +2,7 @@ package cmd import ( "bytes" + "context" "encoding/json" "net/http" "net/http/httptest" @@ -15,7 +16,7 @@ import ( ) func runPersonTrackingCommand( - t *testing.T, template *cobra.Command, jsonOutput bool, args ...string, + ctx context.Context, t *testing.T, template *cobra.Command, jsonOutput bool, args ...string, ) (string, error) { t.Helper() savedJSON := personJSON @@ -26,6 +27,7 @@ func runPersonTrackingCommand( command.SetOut(&output) command.SetErr(&output) command.SetArgs(args) + command.SetContext(ctx) err := command.Execute() return output.String(), err } @@ -53,15 +55,14 @@ func TestPersonTrackAndUntrackReplaceState(t *testing.T) { _, _ = w.Write([]byte(`{"person_id":7,"tracked":false,"tracked_at":null}`)) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) - - output, err := runPersonTrackingCommand(t, personTrackCmd, false, "7") + output, err := runPersonTrackingCommand(testCtx, t, personTrackCmd, false, "7") require.NoError(err) assert.Equal("Person 7: tracked\n", output) - output, err = runPersonTrackingCommand(t, personUntrackCmd, false, "7") + output, err = runPersonTrackingCommand(testCtx, t, personUntrackCmd, false, "7") require.NoError(err) assert.Equal("Person 7: untracked\n", output) assert.Equal(int32(2), requests.Load()) @@ -73,11 +74,10 @@ func TestPersonUntrackJSONIncludesNullTrackedAt(t *testing.T) { _, _ = w.Write([]byte(`{"person_id":7,"tracked":false,"tracked_at":null}`)) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) - - output, err := runPersonTrackingCommand(t, personUntrackCmd, true, "7") + output, err := runPersonTrackingCommand(testCtx, t, personUntrackCmd, true, "7") require.NoError(t, err) assert.JSONEq(t, `{"person_id":7,"tracked":false,"tracked_at":null}`, output) } @@ -88,11 +88,10 @@ func TestPersonTrackRejectsInvalidIDBeforeNetwork(t *testing.T) { requests.Add(1) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) - - _, err := runPersonTrackingCommand(t, personTrackCmd, false, "0") + _, err := runPersonTrackingCommand(testCtx, t, personTrackCmd, false, "0") require.Error(t, err) require.ErrorContains(t, err, "positive integer") assert.Zero(t, requests.Load()) @@ -105,11 +104,10 @@ func TestPersonTrackReturnsStructuredAPIError(t *testing.T) { _, _ = w.Write([]byte(`{"error":"person_profile_not_found","message":"Person profile not found"}`)) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) - - _, err := runPersonTrackingCommand(t, personTrackCmd, false, "7") + _, err := runPersonTrackingCommand(testCtx, t, personTrackCmd, false, "7") require.Error(t, err) assert.ErrorContains(t, err, "Person profile not found") } diff --git a/cmd/msgvault/cmd/person_vector_policy_source.go b/cmd/msgvault/cmd/person_vector_policy_source.go index fb377fb54..a889194b4 100644 --- a/cmd/msgvault/cmd/person_vector_policy_source.go +++ b/cmd/msgvault/cmd/person_vector_policy_source.go @@ -8,11 +8,12 @@ import ( "go.kenn.io/msgvault/internal/vector" ) -func currentSemanticPersonVectorConfigSource() vector.SemanticPersonEmbeddingConfigSource { +func currentSemanticPersonVectorConfigSource(state *invocation) vector.SemanticPersonEmbeddingConfigSource { return func() (vector.Config, error) { - if cfg == nil { + if state == nil || state.cfg == nil { return vector.Config{}, errors.New("semantic person embedding runtime configuration is unavailable") } + cfg := state.cfg configPath := strings.TrimSpace(cfg.ConfigFilePath()) if configPath == "" { return vector.Config{}, errors.New("semantic person embedding runtime configuration path is unavailable") diff --git a/cmd/msgvault/cmd/plaud.go b/cmd/msgvault/cmd/plaud.go new file mode 100644 index 000000000..7f7ca8875 --- /dev/null +++ b/cmd/msgvault/cmd/plaud.go @@ -0,0 +1,482 @@ +package cmd + +import ( + "context" + "encoding/json/v2" + "errors" + "fmt" + "io" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/modelcontextprotocol/go-sdk/auth" + "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/plaud" + "go.kenn.io/msgvault/internal/store" +) + +const plaudConfigHint = `Add to your config.toml: + + [[plaud]] + identifier = "work" + account_email = "you@example.com" + enabled = true + # schedule = "30 */6 * * *" + +Then run 'msgvault add-plaud work' on the daemon host to authorize via browser` + +const plaudCheckOwnerOnlyFlag = "check-owner-only" + +func resolvePlaudSource(args []string, cfg *config.Config) (*config.PlaudSource, error) { + if cfg == nil { + return nil, errors.New("configuration is unavailable") + } + if len(cfg.Plaud) == 0 { + return nil, errors.New("no [[plaud]] sources configured\n\n" + plaudConfigHint) + } + if len(args) > 0 { + src := cfg.GetPlaudSource(args[0]) + if src == nil { + return nil, fmt.Errorf("no [[plaud]] entry with identifier %q", args[0]) + } + return src, nil + } + if len(cfg.Plaud) > 1 { + return nil, errors.New("multiple [[plaud]] sources configured; pass an identifier") + } + src := cfg.Plaud[0] + return &src, nil +} + +func plaudManager(src *config.PlaudSource, state *invocation) *plaud.Manager { + return plaud.NewManager(src.Endpoint, state.cfg.TokensDir(), state.logger) +} + +func newAddPlaudCmd() *cobra.Command { + cmd := newAddPlaudLocalCmd() + cmd.RunE = func(cmd *cobra.Command, args []string) error { + if !isDaemonCLISubprocess() { + checkOnly, err := cmd.Flags().GetBool(plaudCheckOwnerOnlyFlag) + if err != nil { + return fmt.Errorf("read Plaud owner-check flag: %w", err) + } + if !checkOnly { + if err := preflightAddPlaudAuthorize(cmd, args); err != nil { + return err + } + } + return runDaemonCLICommandHTTPFromCobra(cmd, args) + } + return runAddPlaudLocal(cmd, args) + } + return cmd +} + +func newAddPlaudLocalCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "add-plaud [identifier]", + Short: "Authorize and register a Plaud cloud account", + Long: "Authorize a configured Plaud account using browser OAuth on the daemon host. The live account email must match account_email before registration.", + Args: cobra.MaximumNArgs(1), + RunE: runAddPlaudLocal, + } + registerOAuthPreflightedFlag(cmd) + cmd.Flags().Bool(plaudCheckOwnerOnlyFlag, false, "Internal: Check the archive owner before browser authorization") + if err := cmd.Flags().MarkHidden(plaudCheckOwnerOnlyFlag); err != nil { + panic(err) + } + return cmd +} + +func preflightAddPlaudAuthorize(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + if IsRemoteMode(state) { + return errors.New("add-plaud cannot run through a configured remote: the localhost OAuth callback runs on the daemon host; run msgvault add-plaud there, or SSH with localhost:8091 forwarded and run msgvault --local add-plaud on the daemon host") + } + src, err := resolvePlaudSource(args, state.cfg) + if err != nil { + return err + } + email, err := src.EffectiveAccountEmail() + if err != nil { + return err + } + runArgs, err := daemonCLIArgsFromCobra(cmd, args) + if err != nil { + return err + } + // The daemon owns the archive. Check its binding before replacing tokens. + runArgs = append(runArgs, "--"+plaudCheckOwnerOnlyFlag) + if err := runDaemonCLICommandHTTPWithEnv(cmd, runArgs, nil, false, false); err != nil { + return err + } + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Authorizing %s with Plaud...\n", src.Identifier) + if _, err = authorizePlaudAccount(cmd.Context(), plaudManager(src, state), src.Identifier, email); err != nil { + return fmt.Errorf("authorize Plaud: %w", err) + } + if err := cmd.Flags().Set(oauthPreflightedFlag, "true"); err != nil { + return fmt.Errorf("record Plaud authorization preflight: %w", err) + } + return nil +} + +func runAddPlaudLocal(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + src, err := resolvePlaudSource(args, state.cfg) + if err != nil { + return err + } + email, err := src.EffectiveAccountEmail() + if err != nil { + return err + } + if err := validatePlaudOwnerBeforeAuthorization(state, src.Identifier, email); err != nil { + return err + } + checkOnly, err := cmd.Flags().GetBool(plaudCheckOwnerOnlyFlag) + if err != nil { + return fmt.Errorf("read Plaud owner-check flag: %w", err) + } + if checkOnly { + return nil + } + done, err := oauthPreflighted(cmd) + if err != nil { + return err + } + mgr := plaudManager(src, state) + var live string + if !done { + live, err = authorizePlaudAccount(cmd.Context(), mgr, src.Identifier, email) + if err != nil { + return fmt.Errorf("authorize Plaud: %w", err) + } + } else { + session, err := plaud.Connect(cmd.Context(), mgr.Endpoint(), mgr.Handler(src.Identifier)) + if err != nil { + return err + } + defer func() { _ = session.Close() }() + live, err = session.CurrentUser(cmd.Context()) + if err != nil { + return fmt.Errorf("confirm Plaud account: %w", err) + } + // Validate live identity before creating a source. + if err = validatePlaudLiveEmail(email, live); err != nil { + return err + } + } + st, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) + if err != nil { + return err + } + defer cleanup() + if _, err = registerPlaudAccount(st, src.Identifier, email, live); err != nil { + return err + } + if err = runPostSourceCreateMigrationsForInvocation(st, state); err != nil { + return fmt.Errorf("post-source-create migrations: %w", err) + } + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Plaud account %s authorized. Confirmed account email: %s. Run msgvault sync-plaud %s\n", src.Identifier, email, src.Identifier) + return nil +} + +func authorizePlaudAccount(ctx context.Context, mgr *plaud.Manager, identifier, email string) (string, error) { + var live string + err := mgr.AuthorizeWithValidation(ctx, identifier, func(ctx context.Context, handler auth.OAuthHandler) error { + session, err := plaud.Connect(ctx, mgr.Endpoint(), handler) + if err != nil { + return fmt.Errorf("connect to Plaud: %w", err) + } + defer func() { _ = session.Close() }() + live, err = session.CurrentUser(ctx) + if err != nil { + return fmt.Errorf("confirm Plaud account: %w", err) + } + return validatePlaudLiveEmail(email, live) + }) + if err != nil { + return "", err + } + return live, nil +} + +func validatePlaudLiveEmail(configured, live string) error { + expected, err := (config.PlaudSource{AccountEmail: configured}).EffectiveAccountEmail() + if err != nil { + return err + } + actual, err := (config.PlaudSource{AccountEmail: live}).EffectiveAccountEmail() + if err != nil { + return errors.New("plaud returned an invalid account email") + } + if expected != actual { + return errors.New("plaud live account does not match configured account_email; authorize the matching account") + } + return nil +} + +func registerPlaudAccount(st *store.Store, identifier, email, live string) (*store.Source, error) { + if err := validatePlaudLiveEmail(email, live); err != nil { + return nil, err + } + return plaud.RegisterSource(st, identifier, email) +} + +func validatePlaudOwnerBeforeAuthorization(state *invocation, identifier, email string) error { + st, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) + if err != nil { + return fmt.Errorf("open archive before Plaud authorization: %w", err) + } + defer cleanup() + + registered, err := st.GetSourceByTypeAndIdentifier(sourceTypePlaud, identifier) + if errors.Is(err, store.ErrSourceNotFound) { + return nil + } + if err != nil { + return fmt.Errorf("find existing Plaud source before authorization: %w", err) + } + if registered.SyncConfig.Valid { + if err := plaud.ValidateOwner(registered, email); err != nil { + return fmt.Errorf("validate existing Plaud source owner: %w", err) + } + } + return nil +} + +func newSyncPlaudCmd() *cobra.Command { + cmd := &cobra.Command{Use: "sync-plaud [identifier]", Short: "Sync Plaud recordings, transcripts, and notes", Long: `Archive Plaud cloud recordings as searchable meetings. Every run checks full +transcripts, speaker labels and all note tabs for edits. With no identifier, +all configured accounts are synced. --limit starts with newest recordings and +rotates through least recently checked records. --after applies locally and implies +--full. --probe prints tool schemas and first-page counts without personal content. +Cloud Sync and Plaud transcription must already be enabled; audio is not downloaded.`, Args: cobra.MaximumNArgs(1), RunE: runSyncPlaud} + cmd.Flags().Int("limit", 0, "max records hydrated per run; newest recordings first, then rotate through least recently checked (0 = unlimited)") + cmd.Flags().String("after", "", "recordings after YYYY-MM-DD (implies --full)") + cmd.Flags().Bool("full", false, "force archive repair while retaining stable recording IDs") + cmd.Flags().Bool("probe", false, "print tool schemas and counts without meeting content") + return addManualSyncCacheFlags(cmd) +} +func plaudImportOptions(cmd *cobra.Command) (plaud.ImportOptions, error) { + limit, err := cmd.Flags().GetInt("limit") + if err != nil { + return plaud.ImportOptions{}, fmt.Errorf("read Plaud limit flag: %w", err) + } + if limit < 0 { + return plaud.ImportOptions{}, usageErr(cmd, errors.New("--limit must be nonnegative")) + } + full, err := cmd.Flags().GetBool("full") + if err != nil { + return plaud.ImportOptions{}, fmt.Errorf("read Plaud full flag: %w", err) + } + after, err := cmd.Flags().GetString("after") + if err != nil { + return plaud.ImportOptions{}, fmt.Errorf("read Plaud after flag: %w", err) + } + opts := plaud.ImportOptions{Limit: limit, Full: full} + if after != "" { + date, err := time.Parse("2006-01-02", after) + if err != nil { + return opts, usageErr(cmd, fmt.Errorf("invalid --after %q (expected YYYY-MM-DD): %w", after, err)) + } + opts.CreatedAfter = &date + opts.Full = true + } + return opts, nil +} +func runSyncPlaud(cmd *cobra.Command, args []string) error { + opts, err := plaudImportOptions(cmd) + if err != nil { + return err + } + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + if !isDaemonCLISubprocess() { + return runDaemonCLICommandHTTPFromCobra(cmd, args) + } + var sources []config.PlaudSource + if len(args) > 0 || len(state.cfg.Plaud) == 1 { + src, err := resolvePlaudSource(args, state.cfg) + if err != nil { + return err + } + sources = []config.PlaudSource{*src} + } else { + sources = state.cfg.Plaud + } + if len(sources) == 0 { + return errors.New("no [[plaud]] sources configured\n\n" + plaudConfigHint) + } + probe, err := cmd.Flags().GetBool("probe") + if err != nil { + return fmt.Errorf("read Plaud probe flag: %w", err) + } + if probe { + src, err := resolvePlaudSource(args, state.cfg) + if err != nil { + return err + } + return probePlaud(cmd, src) + } + st, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) + if err != nil { + return err + } + defer cleanup() + ctx, stop := signal.NotifyContext(cmd.Context(), os.Interrupt, syscall.SIGTERM) + defer stop() + total := &plaud.ImportSummary{} + refresh := func() error { return rebuildCacheAfterManualSync(state.cfg.DatabaseDSN(), state) } + for _, src := range sources { + if err = registeredPlaudSource(st, src); err != nil { + return finishPlaudImport(ctx, src.Identifier, total, err, refresh) + } + if err = ctx.Err(); err != nil { + return finishPlaudImport(ctx, src.Identifier, total, err, refresh) + } + email, emailErr := src.EffectiveAccountEmail() + if emailErr != nil { + return finishPlaudImport(ctx, src.Identifier, total, emailErr, refresh) + } + mgr := plaudManager(&src, state) + session, connectErr := plaud.Connect(ctx, mgr.Endpoint(), mgr.Handler(src.Identifier)) + if connectErr != nil { + return finishPlaudImport(ctx, src.Identifier, total, connectErr, refresh) + } + runOpts := opts + runOpts.Identifier = src.Identifier + runOpts.AccountEmail = email + runOpts.Progress = func(current, count int, title string) { + _, _ = fmt.Fprintf(cmd.OutOrStdout(), " %d/%d %s\n", current, count, title) + } + sum, importErr := plaud.NewImporter(st, session).Import(ctx, runOpts) + _ = session.Close() + if sum != nil { + total.MeetingsAdded += sum.MeetingsAdded + total.MeetingsUpdated += sum.MeetingsUpdated + } + if err = finishPlaudImport(ctx, src.Identifier, total, importErr, refresh); err != nil { + return err + } + if sum != nil { + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Plaud %s: %d processed, %d added, %d updated (%s)\n", src.Identifier, sum.MeetingsProcessed, sum.MeetingsAdded, sum.MeetingsUpdated, sum.Duration.Round(time.Second)) + } + } + if err = ctx.Err(); err != nil { + return finishPlaudImport(ctx, sources[len(sources)-1].Identifier, total, err, refresh) + } + return refresh() +} +func registeredPlaudSource(st *store.Store, src config.PlaudSource) error { + registered, err := st.GetSourceByTypeAndIdentifier(sourceTypePlaud, src.Identifier) + if errors.Is(err, store.ErrSourceNotFound) { + return fmt.Errorf("plaud source %q is not registered; run msgvault add-plaud %s first", src.Identifier, src.Identifier) + } + if err != nil { + return fmt.Errorf("find registered Plaud source: %w", err) + } + email, err := src.EffectiveAccountEmail() + if err != nil { + return err + } + return plaud.ValidateOwner(registered, email) +} +func finishPlaudImport(ctx context.Context, identifier string, sum *plaud.ImportSummary, importErr error, refresh func() error) error { + operationErr := importErr + if ctx.Err() != nil { + operationErr = errors.Join(operationErr, ctx.Err()) + } + if operationErr == nil { + return nil + } + operationErr = fmt.Errorf("plaud sync %s failed: %w", identifier, operationErr) + if sum != nil && sum.MeetingsAdded+sum.MeetingsUpdated > 0 && refresh != nil { + return errors.Join(operationErr, refresh()) + } + return operationErr +} +func runConfiguredPlaudSync(ctx context.Context, st *store.Store, src config.PlaudSource) error { + if err := registeredPlaudSource(st, src); err != nil { + return err + } + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + email, err := src.EffectiveAccountEmail() + if err != nil { + return err + } + mgr := plaudManager(&src, state) + session, err := plaud.Connect(ctx, mgr.Endpoint(), mgr.Handler(src.Identifier)) + if err != nil { + return err + } + defer func() { _ = session.Close() }() + sum, err := plaud.NewImporter(st, session).Import(ctx, plaud.ImportOptions{Identifier: src.Identifier, AccountEmail: email}) + return finishScheduledPlaudImport(ctx, src.Identifier, sum, err, rebuildCacheAfterScheduledSync) +} +func finishScheduledPlaudImport(ctx context.Context, identifier string, sum *plaud.ImportSummary, importErr error, refresh func(context.Context, string) error) error { + detached := context.WithoutCancel(ctx) + finish := func() error { + if refresh == nil { + return nil + } + return refresh(detached, "plaud:"+identifier) + } + if err := finishPlaudImport(ctx, identifier, sum, importErr, finish); err != nil { + return err + } + return finish() +} + +type plaudProbeSession interface { + ToolInventory(ctx context.Context) ([]plaud.ToolInfo, error) + ListFiles(ctx context.Context, page, pageSize int) (plaud.FilePage, error) +} + +func probePlaud(cmd *cobra.Command, src *config.PlaudSource) error { + mgr := plaudManager(src, invocationFromCommand(cmd)) + session, err := plaud.Connect(cmd.Context(), mgr.Endpoint(), mgr.Handler(src.Identifier)) + if err != nil { + return err + } + defer func() { _ = session.Close() }() + return runPlaudProbe(cmd.Context(), cmd.OutOrStdout(), session) +} + +func runPlaudProbe(ctx context.Context, out io.Writer, session plaudProbeSession) error { + tools, err := session.ToolInventory(ctx) + if err != nil { + return fmt.Errorf("plaud tool inventory failed: %w", err) + } + _, _ = fmt.Fprintln(out, "Tools:") + for _, tool := range tools { + schema, err := json.Marshal(tool.InputSchema, json.Deterministic(true)) + if err != nil { + return fmt.Errorf("format Plaud tool schema failed: %w", err) + } + _, _ = fmt.Fprintf(out, " %s\n Input schema: %s\n", tool.Name, strings.TrimSpace(string(schema))) + } + page, err := session.ListFiles(ctx, 1, 100) + if err != nil { + return fmt.Errorf("plaud first-page count failed; verify provider contract: %w", err) + } + _, _ = fmt.Fprintf(out, "First-page recordings: %d\n", len(page.Files)) + return nil +} + +func init() { rootCmd.AddCommand(newAddPlaudCmd(), newSyncPlaudCmd()) } diff --git a/cmd/msgvault/cmd/plaud_test.go b/cmd/msgvault/cmd/plaud_test.go new file mode 100644 index 000000000..86ffb97a3 --- /dev/null +++ b/cmd/msgvault/cmd/plaud_test.go @@ -0,0 +1,394 @@ +package cmd + +import ( + "bytes" + "context" + "encoding/json/v2" + "errors" + "flag" + "fmt" + "net" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strconv" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/kit/daemon" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/circleback" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/plaud" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil" +) + +func TestPlaudCommandDiscovery(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + for _, name := range []string{"add-plaud", "sync-plaud"} { + cmd, _, err := rootCmd.Find([]string{name}) + require.NoError(err) + assert.Equal(name, cmd.Name()) + } + cmd, _, err := rootCmd.Find([]string{"sync-plaud"}) + require.NoError(err) + for _, name := range []string{"limit", "full", "after", "probe", "build-cache", "no-build-cache"} { + require.NotNil(cmd.Flags().Lookup(name)) + } +} +func TestPlaudRemoteRefusesOAuthBeforeProxy(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + server, requests := newDaemonCLIRunnerTestServer(t, nil, `{"type":"complete"}`) + ctx := configureRemoteDaemonForTest(t, server.URL) + invocationFromContext(ctx).cfg.Plaud = []config.PlaudSource{{Identifier: "work", AccountEmail: "owner@example.com"}} + cmd := newAddPlaudCmd() + cmd.SetContext(ctx) + cmd.SetArgs([]string{"work"}) + err := cmd.Execute() + require.Error(err) + assert.Contains(err.Error(), "daemon host") + assert.Contains(err.Error(), "SSH") + assert.Zero(requests.Load()) +} +func TestPlaudInvalidSyncFlagsBeforeProxy(t *testing.T) { + for _, args := range [][]string{{"--limit=-1"}, {"--after=not-a-date"}} { + t.Run(args[0], func(t *testing.T) { + server, requests := newDaemonCLIRunnerTestServer(t, nil, `{"type":"complete"}`) + cmd := newSyncPlaudCmd() + cmd.SetContext(configureRemoteDaemonForTest(t, server.URL)) + cmd.SetArgs(args) + require.Error(t, cmd.Execute()) + assert.Zero(t, requests.Load()) + }) + } +} +func TestPlaudProbeRequiresIdentifierForMultipleAccounts(t *testing.T) { + t.Setenv(daemonCLISubprocessEnv, strconv.Itoa(os.Getppid())) + cfg := config.NewDefaultConfig() + cfg.HomeDir = t.TempDir() + cfg.Plaud = []config.PlaudSource{ + {Identifier: "work", AccountEmail: "owner@example.com"}, + {Identifier: "personal", AccountEmail: "personal@example.com"}, + } + ctx, cancel := context.WithCancel(testInvocationContext(context.Background(), cfg, invocationOptions{})) + cancel() + cmd := newSyncPlaudCmd() + cmd.SetContext(ctx) + cmd.SetArgs([]string{"--probe"}) + err := cmd.Execute() + require.Error(t, err) + assert.Contains(t, err.Error(), "multiple [[plaud]] sources configured; pass an identifier") +} +func TestPlaudRegistrationRejectsLiveMismatchBeforeCreatingSource(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + st := testutil.NewTestStore(t) + for _, email := range []string{"other@example.com", "Display "} { + _, err := registerPlaudAccount(st, "work", "owner@example.com", email) + require.Error(err) + sources, err := st.ListSources("plaud") + require.NoError(err) + assert.Empty(sources) + } + src, err := registerPlaudAccount(st, "work", " Owner@Example.COM ", "owner@example.com") + require.NoError(err) + _, err = registerPlaudAccount(st, "work", "other@example.com", "other@example.com") + require.Error(err) + got, err := st.GetSourceByTypeAndIdentifier("plaud", "work") + require.NoError(err) + assert.Equal(src.ID, got.ID) +} +func TestPlaudOwnerMismatchStopsBeforeAuthorization(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + http.Error(w, "unauthorized", http.StatusUnauthorized) + })) + defer server.Close() + + cfg := config.NewDefaultConfig() + cfg.HomeDir = t.TempDir() + cfg.Data.DataDir = cfg.HomeDir + state := testInvocationWithConfig(cfg) + func() { + st, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) + require.NoError(err) + defer cleanup() + _, err = registerPlaudAccount(st, "work", "owner@example.com", "owner@example.com") + require.NoError(err) + }() + + cfg.Plaud = []config.PlaudSource{{ + Identifier: "work", + AccountEmail: "replacement@example.com", + Endpoint: server.URL, + }} + rawSource, err := json.Marshal(cfg.Plaud[0]) + require.NoError(err) + child := exec.CommandContext(t.Context(), os.Args[0], "-test.run=^TestPlaudOwnerCheckProcess$", "--", "add-plaud", "work") //nolint:gosec // Re-executes this test binary with fixed arguments. + child.Env = append(daemonCLIChildEnv(os.Environ(), os.Getpid(), nil), + "MSGVAULT_TEST_PLAUD_SOURCE="+string(rawSource), "MSGVAULT_TEST_PLAUD_HOME="+cfg.HomeDir) + output, err := child.CombinedOutput() + require.Error(err) + assert.Contains(string(output), "plaud source owner differs or is unconfirmed") + assert.Zero(requests.Load()) +} +func TestPlaudOwnerPreflightAllowsUnboundSource(t *testing.T) { + require := require.New(t) + cfg := config.NewDefaultConfig() + cfg.HomeDir = t.TempDir() + cfg.Data.DataDir = cfg.HomeDir + state := testInvocationWithConfig(cfg) + func() { + st, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) + require.NoError(err) + defer cleanup() + _, err = st.GetOrCreateSource("plaud", "work") + require.NoError(err) + }() + + require.NoError(validatePlaudOwnerBeforeAuthorization(state, "work", "owner@example.com")) +} + +func TestPlaudAuthorizationPreflightUsesOwningDaemon(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + t.Setenv(daemonCLISubprocessEnv, "") + cfg := lifecycleTestConfig(t.TempDir()) + var oauthRequests atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + oauthRequests.Add(1) + http.Error(w, "unexpected authorization", http.StatusUnauthorized) + })) + t.Cleanup(upstream.Close) + cfg.Plaud = []config.PlaudSource{{Identifier: "work", AccountEmail: "replacement@example.com", Endpoint: upstream.URL}} + st, err := store.Open(cfg.DatabaseDSN()) + require.NoError(err) + require.NoError(st.InitSchema()) + _, err = plaud.RegisterSource(st, "work", "owner@example.com") + require.NoError(err) + require.NoError(st.Close()) + owner, err := tryAcquireWriteOwnerLock(cfg.Data.DataDir) + require.NoError(err) + t.Cleanup(func() { _ = owner.Close() }) + require.NoError(os.MkdirAll(cfg.TokensDir(), 0700)) + tokenPath := plaud.NewManager("", cfg.TokensDir(), nil).TokenPath("work") + require.NoError(os.WriteFile(tokenPath, []byte("existing token"), 0600)) + + var requests atomic.Int32 + mux := http.NewServeMux() + mux.Handle("/api/ping", daemon.NewPingHandler(daemon.PingHandlerOptions{Service: daemonService, Version: Version})) + mux.HandleFunc("/api/v1/cli/run", func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + var req daemonCLIRunTestRequest + if !assert.NoError(json.UnmarshalRead(r.Body, &req)) { + return + } + assert.Contains([][]string{ + {"add-plaud", "work", "--check-owner-only"}, + {"add-plaud", "--check-owner-only", "work"}, + }, req.Args) + rawSource, err := json.Marshal(cfg.Plaud[0]) + if !assert.NoError(err) { + return + } + child := exec.CommandContext(r.Context(), os.Args[0], "-test.run=^TestPlaudOwnerCheckProcess$", "--") //nolint:gosec // Re-executes this test binary; request arguments come from the fixture CLI. + child.Args = append(child.Args, req.Args...) + child.Env = append(daemonCLIChildEnv(os.Environ(), os.Getpid(), nil), + "MSGVAULT_TEST_PLAUD_SOURCE="+string(rawSource), "MSGVAULT_TEST_PLAUD_HOME="+cfg.HomeDir) + output, runErr := child.CombinedOutput() + w.Header().Set("Content-Type", "application/x-ndjson") + assert.NoError(json.MarshalWrite(w, api.CLIRunEvent{Type: "stderr", Data: string(output)})) + _, _ = fmt.Fprintln(w) + event := api.CLIRunEvent{Type: "complete"} + if runErr != nil { + event = api.CLIRunEvent{Type: "error", Error: classifyDaemonCLIWaitErr(runErr, req.Args).Error()} + } + assert.NoError(json.MarshalWrite(w, event)) + _, _ = fmt.Fprintln(w) + }) + server := httptest.NewServer(mux) + t.Cleanup(server.Close) + host, port, err := net.SplitHostPort(server.Listener.Addr().String()) + require.NoError(err) + _, err = daemonRuntimeStore(cfg.Data.DataDir).Write(daemon.RuntimeRecord{ + PID: os.Getpid(), Network: daemon.NetworkTCP, Address: server.Listener.Addr().String(), + Service: daemonService, Version: Version, + Metadata: map[string]string{ + runtimeHost: host, runtimePort: port, + runtimeAPIVersion: strconv.Itoa(daemonAPIVersion), runtimeAPISchemaVersion: api.APISchemaVersion, + runtimeAuthFingerprint: daemonAPIKeyFingerprint(""), runtimeCreateTime: matchingProcessCreateTime(t), + }, + }) + require.NoError(err) + cmd := newAddPlaudCmd() + var stderr bytes.Buffer + cmd.SetErr(&stderr) + cmd.SetContext(testInvocationContext(t.Context(), cfg, invocationOptions{})) + cmd.SetArgs([]string{"work"}) + err = cmd.Execute() + require.ErrorIs(err, errCLISubprocessProxied) + assert.Contains(stderr.String(), "plaud source owner differs or is unconfirmed") + + // An explicit check-only frontend invocation must also stop before OAuth. + cfg.Plaud[0].AccountEmail = "owner@example.com" + cmd = newAddPlaudCmd() + cmd.SetContext(testInvocationContext(t.Context(), cfg, invocationOptions{})) + cmd.SetArgs([]string{"work", "--check-owner-only"}) + require.NoError(cmd.Execute()) + assert.Equal(int32(2), requests.Load()) + assert.Zero(oauthRequests.Load()) + token, err := os.ReadFile(tokenPath) + require.NoError(err) + assert.Equal("existing token", string(token)) +} + +// TestPlaudOwnerCheckProcess runs the production command in its own process so +// the daemon-child marker does not bypass the frontend's archive-owner check. +func TestPlaudOwnerCheckProcess(t *testing.T) { + rawSource := os.Getenv("MSGVAULT_TEST_PLAUD_SOURCE") + if rawSource == "" { + return + } + require := require.New(t) + dataDir := os.Getenv("MSGVAULT_TEST_PLAUD_HOME") + require.NotEmpty(dataDir) + cfg := lifecycleTestConfig(dataDir) + cfg.Plaud = []config.PlaudSource{{}} + require.NoError(json.Unmarshal([]byte(rawSource), &cfg.Plaud[0])) + args := flag.Args() + require.NotEmpty(args) + cmd := newAddPlaudCmd() + cmd.SetContext(testInvocationContext(t.Context(), cfg, invocationOptions{})) + cmd.SetArgs(args[1:]) + if err := cmd.Execute(); err != nil { + os.Exit(1) + } + os.Exit(0) +} +func TestPlaudScheduledMissingSourceStopsBeforeAuth(t *testing.T) { + st := testutil.NewTestStore(t) + ctx, cancel := context.WithCancel(testInvocationContext(context.Background(), config.NewDefaultConfig(), invocationOptions{})) + cancel() + err := runConfiguredPlaudSync(ctx, st, config.PlaudSource{Identifier: "work", AccountEmail: "owner@example.com"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "add-plaud work") +} +func TestPlaudPartialCanceledImportRefreshesDetachedContext(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + calls := 0 + refreshErr := errors.New("refresh failed") + err := finishScheduledPlaudImport(ctx, "work", &plaud.ImportSummary{MeetingsAdded: 1}, context.Canceled, func(refreshCtx context.Context, name string) error { + calls++ + require.NoError(refreshCtx.Err()) + assert.Equal("plaud:work", name) + return refreshErr + }) + assert.Equal(1, calls) + require.ErrorIs(err, context.Canceled) + require.ErrorIs(err, refreshErr) + calls = 0 + err = finishPlaudImport(context.Background(), "work", &plaud.ImportSummary{}, errors.New("failed"), func() error { calls++; return nil }) + require.Error(err) + assert.Zero(calls) +} + +type plaudProbeFixture struct { + inventoryErr error + listErr error +} + +func (f plaudProbeFixture) ToolInventory(context.Context) ([]plaud.ToolInfo, error) { + return []plaud.ToolInfo{{Name: "list_files", Description: "Find recordings", InputSchema: []byte(`{"type":"object"}`)}}, f.inventoryErr +} +func (f plaudProbeFixture) ListFiles(context.Context, int, int) (plaud.FilePage, error) { + return plaud.FilePage{Files: []plaud.File{{ID: "private-id", Name: "Private meeting title"}}}, f.listErr +} + +func TestPlaudProbePreservesFailureCause(t *testing.T) { + for _, tc := range []struct { + name string + session plaudProbeFixture + cause error + }{ + {"inventory", plaudProbeFixture{inventoryErr: context.Canceled}, context.Canceled}, + {"list", plaudProbeFixture{listErr: plaud.ErrContract}, plaud.ErrContract}, + } { + t.Run(tc.name, func(t *testing.T) { + var out bytes.Buffer + err := runPlaudProbe(t.Context(), &out, tc.session) + require.ErrorIs(t, err, tc.cause) + assert.Contains(t, err.Error(), tc.cause.Error()) + }) + } +} +func TestPlaudProbePrintsToolsAndCountsOnly(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + var out bytes.Buffer + require.NoError(runPlaudProbe(t.Context(), &out, plaudProbeFixture{})) + assert.Contains(out.String(), "list_files") + assert.Contains(out.String(), "1") + assert.NotContains(out.String(), "private-id") + assert.NotContains(out.String(), "Private meeting title") +} +func TestRemovePlaudAccountDeletesOnlyItsToken(t *testing.T) { + require := require.New(t) + tmp := t.TempDir() + cfg := &config.Config{HomeDir: tmp, Data: config.DataConfig{DataDir: tmp}} + st, err := store.Open(filepath.Join(tmp, "msgvault.db")) + require.NoError(err) + require.NoError(st.InitSchema()) + _, err = st.GetOrCreateSource("plaud", "work") + require.NoError(err) + require.NoError(st.Close()) + mgr := plaud.NewManager("", cfg.TokensDir(), nil) + require.NoError(os.MkdirAll(cfg.TokensDir(), 0700)) + token := mgr.TokenPath("work") + other := mgr.TokenPath("other") + cb := circleback.NewManager("", cfg.TokensDir(), nil).TokenPath("work") + for _, p := range []string{token, other, cb} { + require.NoError(os.WriteFile(p, []byte(`{}`), 0600)) + } + root := newTestRootCmd() + root.SetContext(testInvocationContext(t.Context(), cfg, invocationOptions{})) + root.AddCommand(newRemoveAccountLocalTestCmd()) + root.SetArgs([]string{"remove-account", "work", "--yes", "--type", "plaud"}) + require.NoError(root.Execute()) + _, err = os.Stat(token) + require.ErrorIs(err, os.ErrNotExist) + for _, p := range []string{other, cb} { + _, err = os.Stat(p) + require.NoError(err) + } +} + +func TestPlaudManualMissingSourceStopsBeforeAuth(t *testing.T) { + t.Setenv(daemonCLISubprocessEnv, strconv.Itoa(os.Getppid())) + tmp := t.TempDir() + cfg := &config.Config{HomeDir: tmp, Data: config.DataConfig{DataDir: tmp}, Plaud: []config.PlaudSource{{Identifier: "work", AccountEmail: "owner@example.com", Endpoint: "http://127.0.0.1:1/mcp"}}} + cmd := newSyncPlaudCmd() + cmd.SetContext(testInvocationContext(t.Context(), cfg, invocationOptions{})) + cmd.SetArgs([]string{"work"}) + err := cmd.Execute() + require.Error(t, err) + assert.Contains(t, err.Error(), "add-plaud work") +} + +func TestPlaudManualCacheClassification(t *testing.T) { + assert.True(t, manualSyncCLICommand([]string{"sync-plaud", "work"})) + assert.False(t, manualSyncCLICommand([]string{"sync-plaud", "work", "--probe"})) + assert.False(t, manualSyncCLICommand([]string{"sync-plaud", "work", "--probe=true"})) +} diff --git a/cmd/msgvault/cmd/provider_credentials.go b/cmd/msgvault/cmd/provider_credentials.go index 8884d4ae9..a17049f30 100644 --- a/cmd/msgvault/cmd/provider_credentials.go +++ b/cmd/msgvault/cmd/provider_credentials.go @@ -61,13 +61,6 @@ func resolvePersonEnrichmentSuppression(cfg *config.Config) (string, error) { if cfg == nil { return "", errors.New("person enrichment suppression config is unavailable") } - if cfg.People.Enrichment.SuppressionKeyEnv != providercredentials.StoredSuppressionEnvironment { - value, ok := os.LookupEnv(cfg.People.Enrichment.SuppressionKeyEnv) - if !ok || value == "" { - return "", errors.New("person enrichment suppression key is unavailable") - } - return value, nil - } snapshot, err := providercredentials.Read(cfg.TokensDir()) if err != nil { return "", err @@ -76,15 +69,21 @@ func resolvePersonEnrichmentSuppression(cfg *config.Config) (string, error) { if err != nil { return "", err } - if !configured || value == "" { - return "", errors.New("stored person enrichment suppression key is unavailable") + if configured { + return value, nil + } + if cfg.People.Enrichment.SuppressionKeyEnv != providercredentials.StoredSuppressionEnvironment { + value, ok := os.LookupEnv(cfg.People.Enrichment.SuppressionKeyEnv) + if ok && value != "" { + return value, nil + } } - return value, nil + return "", errors.New("person enrichment suppression key is unavailable") } func personEnrichmentEnvironmentLookup(cfg *config.Config) personenrichment.CredentialLookup { return func(name string) (string, bool) { - if name != providercredentials.StoredSuppressionEnvironment { + if name != providercredentials.StoredSuppressionEnvironment && (cfg == nil || name == "" || name != cfg.People.Enrichment.SuppressionKeyEnv) { return os.LookupEnv(name) } value, err := resolvePersonEnrichmentSuppression(cfg) diff --git a/cmd/msgvault/cmd/provider_credentials_test.go b/cmd/msgvault/cmd/provider_credentials_test.go index d1a7e8616..c7150af86 100644 --- a/cmd/msgvault/cmd/provider_credentials_test.go +++ b/cmd/msgvault/cmd/provider_credentials_test.go @@ -49,6 +49,8 @@ func TestVectorProviderCredentialResolutionUsesOneStartupSnapshot(t *testing.T) requirements := require.New(t) t.Setenv("TEXT_EMBEDDING_KEY", "environment-secret") cfg := config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Data.DataDir = t.TempDir() cfg.Vector.Embeddings.Endpoint = "https://embeddings.example.test/v1" cfg.Vector.Embeddings.APIKeyEnv = "TEXT_EMBEDDING_KEY" @@ -90,6 +92,8 @@ func TestPersonEnrichmentProviderCredentialLookupUsesStableNameAndReloadsStore(t requirements := require.New(t) t.Setenv("SHARED_EXA_KEY", "environment-secret") cfg := config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Data.DataDir = t.TempDir() profile := personenrichment.ProviderProfile{ Name: "exa-primary", Kind: personenrichment.ProviderExa, @@ -120,6 +124,8 @@ func TestStoredSuppressionLookupFeedsReservedRuntimeEnvironmentAndFailsClosed(t requirements := require.New(t) assertions := assert.New(t) cfg := config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Data.DataDir = t.TempDir() cfg.People.Enrichment.SuppressionKeyEnv = providercredentials.StoredSuppressionEnvironment empty, err := providercredentials.Read(cfg.TokensDir()) @@ -152,9 +158,9 @@ func TestDefaultPersonEnrichmentLookupReadsRuntimeConfig(t *testing.T) { ) requirements.NoError(err) - withTestConfig(t, nil) - deps := defaultPersonEnrichmentCommandDeps() - cfg = configured + testCtx := withTestConfig(t, configured) + _ = testCtx + deps := defaultPersonEnrichmentCommandDeps(testCtx) value, ok := deps.lookupEnv(providercredentials.StoredSuppressionEnvironment) assertions.True(ok) @@ -165,6 +171,8 @@ func TestPersonEnrichmentGateLoadsStableStoredCredentialOnlyAfterSuppression(t * requirements := require.New(t) t.Setenv("SCHEDULE_PROVIDER_KEY", "") cfg := config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Data.DataDir = t.TempDir() profile := scheduleTestEnrichmentProfile(t) empty, err := providercredentials.Read(cfg.TokensDir()) @@ -189,7 +197,7 @@ func TestPersonEnrichmentGateLoadsStableStoredCredentialOnlyAfterSuppression(t * ) requirements.NoError(err) - authorization, err := gate.Authorize(t.Context(), personenrichment.EgressInput{ + authorization, err := gate.Authorize(testCtx, personenrichment.EgressInput{ Request: personenrichment.Request{Identity: personenrichment.Identity{Email: "person@example.com"}}, Profile: profile, }) @@ -204,6 +212,8 @@ func TestDefaultCLIProxyLookupsNeverForwardStoredCredentials(t *testing.T) { requirements := require.New(t) t.Setenv("TEST_PROVIDER_KEY", "") cfg := config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Data.DataDir = t.TempDir() cfg.People.Sweep = personProviderTestConfig() cfg.People.Enrichment = personEnrichmentCLIConfig(providercredentials.StoredSuppressionEnvironment) @@ -222,11 +232,12 @@ func TestDefaultCLIProxyLookupsNeverForwardStoredCredentials(t *testing.T) { providercredentials.PersonEnrichmentID(cfg.People.Enrichment.Providers[0].Name), cfg.People.Enrichment.Providers[0].Endpoint, "stored-enrichment-secret") requirements.NoError(err) - withTestConfig(t, cfg) + testCtx = withTestConfig(t, cfg) + _ = testCtx sweepDeps := defaultPersonSweepCommandDeps() assertions.Empty(personSweepForwardEnv(cfg.People.Sweep, sweepDeps.lookupEnv)) - enrichmentDeps := defaultPersonEnrichmentCommandDeps() + enrichmentDeps := defaultPersonEnrichmentCommandDeps(testCtx) value, ok := enrichmentDeps.proxyLookupEnv(providercredentials.StoredSuppressionEnvironment) assertions.False(ok) assertions.Empty(value) diff --git a/cmd/msgvault/cmd/provider_identity_refresh.go b/cmd/msgvault/cmd/provider_identity_refresh.go index aeaf407f7..73f43596c 100644 --- a/cmd/msgvault/cmd/provider_identity_refresh.go +++ b/cmd/msgvault/cmd/provider_identity_refresh.go @@ -2,6 +2,7 @@ package cmd import ( "context" + "errors" "go.kenn.io/msgvault/internal/gmail" "go.kenn.io/msgvault/internal/provideridentity" @@ -11,16 +12,16 @@ import ( var fastmailIdentityInventoryFactory provideridentity.Factory = provideridentity.NewFastmailInventory -func newMessageSyncer(client gmail.API, st *store.Store, opts *msgsync.Options) *msgsync.Syncer { +func newMessageSyncer(client gmail.API, st *store.Store, opts *msgsync.Options, state *invocation) *msgsync.Syncer { if opts == nil { opts = msgsync.DefaultOptions() } configured := *opts - configured.RemoteImages = configuredRemoteImageFetcher() - return withAutomaticProviderIdentityRefresh(msgsync.New(client, st, &configured), st) + configured.RemoteImages = configuredRemoteImageFetcher(state.cfg) + return withAutomaticProviderIdentityRefresh(msgsync.New(client, st, &configured), st, state) } -func withAutomaticProviderIdentityRefresh(syncer *msgsync.Syncer, st *store.Store) *msgsync.Syncer { +func withAutomaticProviderIdentityRefresh(syncer *msgsync.Syncer, st *store.Store, state *invocation) *msgsync.Syncer { return syncer.WithSuccessfulSyncHook( "provider identity refresh", func(ctx context.Context, source *store.Source, mailboxChanged bool) error { @@ -31,9 +32,12 @@ func withAutomaticProviderIdentityRefresh(syncer *msgsync.Syncer, st *store.Stor if !mailboxChanged { refresh = provideridentity.AutoRefreshIfDue } + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } _, _, err := refresh( ctx, - cfg, + state.cfg, st, source.ID, fastmailIdentityInventoryFactory, diff --git a/cmd/msgvault/cmd/provider_identity_refresh_test.go b/cmd/msgvault/cmd/provider_identity_refresh_test.go index 5706789d1..f2945882e 100644 --- a/cmd/msgvault/cmd/provider_identity_refresh_test.go +++ b/cmd/msgvault/cmd/provider_identity_refresh_test.go @@ -29,6 +29,8 @@ func (i *scheduledProviderInventory) ListIdentityRecords(context.Context) ([]fas } func TestAutomaticProviderIdentityRefreshIsOptInAndRunsAfterIMAPCompletion(t *testing.T) { + cfg := testConfigValue() + st := testutil.NewTestStore(t) const sourceIdentifier = "imaps://user@example.test@imap.example.test:993" source, err := st.GetOrCreateSource(sourceTypeIMAP, sourceIdentifier) @@ -47,13 +49,14 @@ func TestAutomaticProviderIdentityRefreshIsOptInAndRunsAfterIMAPCompletion(t *te cfg = &config.Config{Fastmail: []config.FastmailSource{{ SourceID: source.ID, APIToken: "not-called-token", }}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) calls := 0 fastmailIdentityInventoryFactory = func(string) provideridentity.Inventory { calls++ return &scheduledProviderInventory{} } - summary := runAutomaticProviderSync(t, st, sourceIdentifier) + summary := runAutomaticProviderSync(testCtx, t, st, sourceIdentifier) requirements.NotNil(summary) assertions.Zero(calls) @@ -65,6 +68,7 @@ func TestAutomaticProviderIdentityRefreshIsOptInAndRunsAfterIMAPCompletion(t *te cfg = &config.Config{Fastmail: []config.FastmailSource{{ SourceID: source.ID, APIToken: "provider-token", AutoConfirmIdentities: true, }}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) fastmailIdentityInventoryFactory = func(string) provideridentity.Inventory { return &scheduledProviderInventory{records: []fastmail.Record{ {Identifier: "old@example.test", State: "disabled", Kind: "masked-email"}, @@ -72,7 +76,7 @@ func TestAutomaticProviderIdentityRefreshIsOptInAndRunsAfterIMAPCompletion(t *te }} } - summary := runAutomaticProviderSync(t, st, sourceIdentifier) + summary := runAutomaticProviderSync(testCtx, t, st, sourceIdentifier) requirements.NotNil(summary) identities, err := st.ListAccountIdentities(source.ID) @@ -89,8 +93,9 @@ func TestAutomaticProviderIdentityRefreshIsOptInAndRunsAfterIMAPCompletion(t *te func setUpScheduledProviderIdentityRefresh( t *testing.T, st *store.Store, -) (*store.Source, *scheduledProviderInventory) { +) (*store.Source, *scheduledProviderInventory, context.Context) { t.Helper() + cfg := testConfigValue() const sourceIdentifier = "gmail-user@example.test" source, err := st.GetOrCreateSource(sourceTypeGmail, sourceIdentifier) require.NoError(t, err) @@ -106,18 +111,20 @@ func setUpScheduledProviderIdentityRefresh( cfg = &config.Config{Fastmail: []config.FastmailSource{{ SourceID: source.ID, APIToken: "provider-token", AutoConfirmIdentities: true, }}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) inventory := &scheduledProviderInventory{records: []fastmail.Record{{ Identifier: "historical@example.test", State: "deleted", Kind: "masked-email", }}} fastmailIdentityInventoryFactory = func(string) provideridentity.Inventory { return inventory } - return source, inventory + return source, inventory, testCtx } // runNoOpIncrementalProviderSync drives one incremental sync whose cursor // already equals the mailbox's current history, so the run is a no-op. func runNoOpIncrementalProviderSync( + ctx context.Context, t *testing.T, st *store.Store, source *store.Source, @@ -126,8 +133,8 @@ func runNoOpIncrementalProviderSync( client := gmail.NewMockAPI() client.Profile = &gmail.Profile{EmailAddress: source.Identifier, HistoryID: 100} options := msgsync.DefaultOptions() - syncer := newMessageSyncer(client, st, options).WithLogger(slog.New(slog.DiscardHandler)) - summary, err := syncer.Incremental(t.Context(), source) + syncer := newMessageSyncer(client, st, options, invocationFromContext(ctx)).WithLogger(slog.New(slog.DiscardHandler)) + summary, err := syncer.Incremental(ctx, source) require.NoError(t, err) require.NotNil(t, summary) return summary @@ -141,11 +148,11 @@ func TestAutomaticProviderIdentityRefreshSkipsProviderInventoryOnNoOpIncremental requirements := require.New(t) assertions := assert.New(t) st := testutil.NewTestStore(t) - source, inventory := setUpScheduledProviderIdentityRefresh(t, st) - requirements.NoError(st.RecordProviderIdentityRefreshOutcomeContext(t.Context(), source.ID, nil), + source, inventory, testCtx := setUpScheduledProviderIdentityRefresh(t, st) + requirements.NoError(st.RecordProviderIdentityRefreshOutcomeContext(testCtx, source.ID, nil), "the previous refresh succeeded recently") - runNoOpIncrementalProviderSync(t, st, source) + runNoOpIncrementalProviderSync(testCtx, t, st, source) assertions.Zero(inventory.calls, "an unchanged mailbox with a fresh inventory must not cost a provider round trip") @@ -161,9 +168,9 @@ func TestAutomaticProviderIdentityRefreshRunsOnNoOpIncrementalSyncWhenNeverRefre requirements := require.New(t) assertions := assert.New(t) st := testutil.NewTestStore(t) - source, inventory := setUpScheduledProviderIdentityRefresh(t, st) + source, inventory, testCtx := setUpScheduledProviderIdentityRefresh(t, st) - runNoOpIncrementalProviderSync(t, st, source) + runNoOpIncrementalProviderSync(testCtx, t, st, source) assertions.Equal(1, inventory.calls, "a source that has never refreshed owes an inventory read even without mailbox history") @@ -172,7 +179,7 @@ func TestAutomaticProviderIdentityRefreshRunsOnNoOpIncrementalSyncWhenNeverRefre requirements.Len(identities, 1) assertions.Equal("historical@example.test", identities[0].Address) - runNoOpIncrementalProviderSync(t, st, source) + runNoOpIncrementalProviderSync(testCtx, t, st, source) assertions.Equal(1, inventory.calls, "the successful refresh is recorded, so the next no-op sync skips the provider") @@ -185,12 +192,12 @@ func TestAutomaticProviderIdentityRefreshRetriesOnNoOpIncrementalSyncAfterFailur requirements := require.New(t) assertions := assert.New(t) st := testutil.NewTestStore(t) - source, inventory := setUpScheduledProviderIdentityRefresh(t, st) + source, inventory, testCtx := setUpScheduledProviderIdentityRefresh(t, st) requirements.NoError(st.RecordProviderIdentityRefreshOutcomeContext( - t.Context(), source.ID, errors.New("provider unavailable"), + testCtx, source.ID, errors.New("provider unavailable"), ), "the previous refresh failed") - runNoOpIncrementalProviderSync(t, st, source) + runNoOpIncrementalProviderSync(testCtx, t, st, source) assertions.Equal(1, inventory.calls, "a failed refresh owes a retry on the next sync, no-op or not") @@ -204,15 +211,15 @@ func TestAutomaticProviderIdentityRefreshRunsAfterIncrementalGmailCompletion(t * requirements := require.New(t) assertions := assert.New(t) st := testutil.NewTestStore(t) - source, inventory := setUpScheduledProviderIdentityRefresh(t, st) + source, inventory, testCtx := setUpScheduledProviderIdentityRefresh(t, st) client := gmail.NewMockAPI() client.Profile = &gmail.Profile{EmailAddress: source.Identifier, HistoryID: 200} client.HistoryID = 200 options := msgsync.DefaultOptions() - syncer := newMessageSyncer(client, st, options).WithLogger(slog.New(slog.DiscardHandler)) + syncer := newMessageSyncer(client, st, options, invocationFromContext(testCtx)).WithLogger(slog.New(slog.DiscardHandler)) - summary, err := syncer.Incremental(t.Context(), source) + summary, err := syncer.Incremental(testCtx, source) requirements.NoError(err) requirements.NotNil(summary) @@ -225,6 +232,7 @@ func TestAutomaticProviderIdentityRefreshRunsAfterIncrementalGmailCompletion(t * } func runAutomaticProviderSync( + ctx context.Context, t *testing.T, st *store.Store, sourceIdentifier string, @@ -234,8 +242,8 @@ func runAutomaticProviderSync( client.Profile = &gmail.Profile{EmailAddress: "user@example.test", HistoryID: 100} options := msgsync.DefaultOptions() options.SourceType = sourceTypeIMAP - syncer := newMessageSyncer(client, st, options).WithLogger(slog.New(slog.DiscardHandler)) - summary, err := syncer.Full(t.Context(), sourceIdentifier) + syncer := newMessageSyncer(client, st, options, invocationFromContext(ctx)).WithLogger(slog.New(slog.DiscardHandler)) + summary, err := syncer.Full(ctx, sourceIdentifier) require.NoError(t, err) return summary } diff --git a/cmd/msgvault/cmd/purge_excluded_media.go b/cmd/msgvault/cmd/purge_excluded_media.go index 32251b98f..6592ce601 100644 --- a/cmd/msgvault/cmd/purge_excluded_media.go +++ b/cmd/msgvault/cmd/purge_excluded_media.go @@ -25,15 +25,34 @@ const ( ) type purgeExcludedMediaDeps struct { + bind func(context.Context) purgeExcludedMediaDeps openStore func() (*store.Store, func(), error) config func() *config.Config removeFile func(string) error } -func defaultPurgeExcludedMediaDeps() purgeExcludedMediaDeps { +func defaultPurgeExcludedMediaDeps(contexts ...context.Context) purgeExcludedMediaDeps { + if len(contexts) > 0 { + deps := defaultPurgeExcludedMediaDeps() + if deps.bind != nil { + return deps.bind(contexts[0]) + } + } return purgeExcludedMediaDeps{ - openStore: openWritableStoreAndInitForIngest, - config: func() *config.Config { return cfg }, + bind: func(ctx context.Context) purgeExcludedMediaDeps { + deps := defaultPurgeExcludedMediaDeps() + if state := invocationFromContext(ctx); state != nil && state.cfg != nil { + deps.config = func() *config.Config { return state.cfg } + deps.openStore = func() (*store.Store, func(), error) { + return openWritableStoreAndInitForIngestInvocation(state) + } + } + return deps + }, + openStore: func() (*store.Store, func(), error) { + return nil, nil, errors.New("configuration is unavailable") + }, + config: func() *config.Config { return nil }, removeFile: os.Remove, } } @@ -84,6 +103,9 @@ Applying the purge requires an interactive confirmation or --yes. This command never deletes media from a provider.`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { + if invocationFromContext(cmd.Context()) != nil && deps.bind != nil { + deps = deps.bind(cmd.Context()) + } dryRun, err := cmd.Flags().GetBool("dry-run") if err != nil { return fmt.Errorf("read --dry-run flag: %w", err) diff --git a/cmd/msgvault/cmd/purge_excluded_media_test.go b/cmd/msgvault/cmd/purge_excluded_media_test.go index 2d3fa5bbf..3905ad6a8 100644 --- a/cmd/msgvault/cmd/purge_excluded_media_test.go +++ b/cmd/msgvault/cmd/purge_excluded_media_test.go @@ -35,6 +35,8 @@ func newPurgeMediaFixture(t *testing.T) purgeMediaFixture { Data: config.DataConfig{DataDir: dataDir}, Beeper: config.BeeperConfig{MediaScope: string(attachmentpolicy.ScopeDirect)}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx source, err := st.GetOrCreateSource(sourceTypeBeeper, "signal") require.NoError(t, err) newMessage := func(sourceMessageID, conversationType string, participants int) int64 { @@ -237,6 +239,8 @@ func TestPurgeExcludedMediaRetainsUnresolvedRostersUnderParticipantLimit(t *test Data: config.DataConfig{DataDir: t.TempDir()}, Teams: config.TeamsConfig{MediaMaxParticipants: 4, MaxMediaMB: 1}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx source, err := st.GetOrCreateSource(sourceTypeTeams, "me@example.com") require.NoError(err) conversationID, err := st.EnsureConversationWithType(source.ID, "team/channel", "channel", "Releases") diff --git a/cmd/msgvault/cmd/query.go b/cmd/msgvault/cmd/query.go index 93cf755fc..38a6bcb49 100644 --- a/cmd/msgvault/cmd/query.go +++ b/cmd/msgvault/cmd/query.go @@ -9,12 +9,14 @@ import ( "io" "strconv" "strings" + "time" "github.com/spf13/cobra" "go.kenn.io/msgvault/internal/query" ) var queryFormat string +var queryFresh bool var queryCmd = &cobra.Command{ Use: "query [sql]", @@ -54,10 +56,32 @@ func runHTTPQuery(cmd *cobra.Command, sqlStr string) error { } defer func() { _ = st.Close() }() - result, err := st.RunSQLQuery(cmd.Context(), sqlStr) + result, accepted, err := st.RunSQLQueryWithFresh(cmd.Context(), sqlStr, queryFresh) if err != nil { return fmt.Errorf("query: %w", err) } + for accepted != nil { + _, err := fmt.Fprintf(cmd.ErrOrStderr(), "Analytics cache build %s: %s; waiting for completion\n", + accepted.Status, accepted.JobID) + if err != nil { + return fmt.Errorf("write cache build status: %w", err) + } + if err := st.WaitForCacheBuild(cmd.Context(), accepted.JobID); err != nil { + return fmt.Errorf("query: %w", err) + } + result, accepted, err = st.RunSQLQueryWithFresh(cmd.Context(), sqlStr, false) + if err != nil { + return fmt.Errorf("query: %w", err) + } + } + if strings.ToLower(strings.TrimSpace(queryFormat)) != outputFormatJSON && result.Cache != nil { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Analytics cache published %s; generation %s", + result.Cache.PublishedAt.Format(time.RFC3339), result.Cache.Generation) + if result.Cache.StaleReason != "" { + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "; stale: %s", result.Cache.StaleReason) + } + _, _ = fmt.Fprintln(cmd.ErrOrStderr()) + } return writeQueryResult(cmd.OutOrStdout(), result, queryFormat) } @@ -68,7 +92,7 @@ func writeQueryResult(w io.Writer, result *query.QueryResult, format string) err format = strings.ToLower(strings.TrimSpace(format)) switch format { case outputFormatJSON: - return writeJSON(w, result.Columns, result.Rows) + return writeJSON(w, result) case "csv": return writeCSV(w, result.Columns, result.Rows) case "table": @@ -78,14 +102,7 @@ func writeQueryResult(w io.Writer, result *query.QueryResult, format string) err } } -func writeJSON( - w io.Writer, cols []string, rows [][]any, -) error { - result := query.QueryResult{ - Columns: cols, - Rows: rows, - RowCount: len(rows), - } +func writeJSON(w io.Writer, result *query.QueryResult) error { enc := jsontext.NewEncoder(w, jsontext.WithIndentPrefix(""), jsontext.WithIndent(" ")) return json.MarshalEncode(enc, result, json.Deterministic(true)) @@ -193,6 +210,7 @@ func writeTable( func init() { rootCmd.AddCommand(queryCmd) + queryCmd.Flags().BoolVar(&queryFresh, "fresh", false, "Wait for analytics to include writes committed before this request, then return rows") queryCmd.Flags().StringVar( &queryFormat, "format", outputFormatJSON, "Output format: json, csv, or table", diff --git a/cmd/msgvault/cmd/query_test.go b/cmd/msgvault/cmd/query_test.go index 6ec7e2815..81eb89251 100644 --- a/cmd/msgvault/cmd/query_test.go +++ b/cmd/msgvault/cmd/query_test.go @@ -2,6 +2,7 @@ package cmd import ( "bytes" + "context" "encoding/json" "encoding/json/jsontext" "io" @@ -10,6 +11,7 @@ import ( "net/http/httptest" "sync/atomic" "testing" + "time" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" @@ -20,6 +22,10 @@ import ( ) func TestQueryCommand_UsesLocalDaemonHTTPAndPreservesJSONOutput(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + useLocal := false + require := require.New(t) assert := assert.New(t) dataDir := t.TempDir() @@ -30,20 +36,25 @@ func TestQueryCommand_UsesLocalDaemonHTTPAndPreservesJSONOutput(t *testing.T) { savedLogger := logger savedUseLocal := useLocal savedQueryFormat := queryFormat + savedQueryFresh := queryFresh t.Cleanup(func() { cfg = savedCfg logger = savedLogger useLocal = savedUseLocal queryFormat = savedQueryFormat + queryFresh = savedQueryFresh }) cfg = &config.Config{ HomeDir: dataDir, Data: config.DataConfig{DataDir: dataDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx logger = slog.New(slog.DiscardHandler) useLocal = true queryFormat = outputFormatJSON + queryFresh = false var stdout bytes.Buffer var stderr bytes.Buffer @@ -52,6 +63,7 @@ func TestQueryCommand_UsesLocalDaemonHTTPAndPreservesJSONOutput(t *testing.T) { Args: queryCmd.Args, RunE: queryCmd.RunE, } + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetErr(&stderr) cmd.SetArgs([]string{"SELECT subject FROM messages"}) @@ -68,6 +80,114 @@ func TestQueryCommand_UsesLocalDaemonHTTPAndPreservesJSONOutput(t *testing.T) { }`, stdout.String(), "stdout JSON") } +func TestQueryCommandWaitsForAcceptedBuild(t *testing.T) { + for _, test := range []struct { + name string + fresh bool + outcome string + wantErr string + }{ + {name: "plain query", outcome: "published"}, + {name: "fresh query", fresh: true, outcome: "published"}, + {name: "failed build", fresh: true, outcome: "failed", wantErr: "synthetic build failure"}, + {name: "missing job", outcome: "missing", wantErr: "Analytics cache build not found"}, + {name: "canceled request", fresh: true, outcome: "canceled"}, + } { + t.Run(test.name, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + dataDir := t.TempDir() + var queries, polls atomic.Int32 + mux := http.NewServeMux() + mux.Handle("/api/ping", daemon.NewPingHandler(daemon.PingHandlerOptions{ + Service: daemonService, Version: Version, + })) + mux.HandleFunc("POST /api/v1/query", func(w http.ResponseWriter, r *http.Request) { + var req struct { + SQL string `json:"sql"` + Fresh bool `json:"fresh"` + } + if !assert.NoError(json.NewDecoder(r.Body).Decode(&req)) { + w.WriteHeader(http.StatusBadRequest) + return + } + assert.Equal("SELECT id FROM messages", req.SQL) + w.Header().Set("Content-Type", "application/json") + if queries.Add(1) == 1 { + assert.Equal(test.fresh, req.Fresh) + w.WriteHeader(http.StatusAccepted) + _, _ = w.Write([]byte(`{"status":"queued","job_id":"synthetic-job"}`)) + return + } + assert.False(req.Fresh, "retry must read the completed publication") + assert.GreaterOrEqual(polls.Load(), int32(3), "query must wait for publication") + _, _ = w.Write([]byte(`{"columns":["id"],"rows":[[9007199254740993]],"row_count":1}`)) + }) + mux.HandleFunc("GET /api/v1/cache-builds/synthetic-job", func(w http.ResponseWriter, r *http.Request) { + poll := polls.Add(1) + w.Header().Set("Content-Type", "application/json") + switch test.outcome { + case "failed": + _, _ = w.Write([]byte(`{"job_id":"synthetic-job","status":"failed","error":"synthetic build failure"}`)) + case "missing": + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"error":"cache_build_not_found","message":"Analytics cache build not found"}`)) + case "canceled": + cancel() + <-r.Context().Done() + default: + status := "published" + switch poll { + case 1: + status = "queued" + case 2: + status = "running" + } + assert.NoError(json.NewEncoder(w).Encode(map[string]string{ + "job_id": "synthetic-job", "status": status, + })) + } + }) + server := httptest.NewServer(mux) + t.Cleanup(server.Close) + writeStatsHTTPDaemonRuntime(t, dataDir, server) + savedFormat, savedFresh := queryFormat, queryFresh + t.Cleanup(func() { + queryFormat, queryFresh = savedFormat, savedFresh + }) + cfg := &config.Config{HomeDir: dataDir, Data: config.DataConfig{DataDir: dataDir}} + testCtx := testInvocationContext(ctx, cfg, invocationOptions{useLocal: true}) + queryFormat, queryFresh = outputFormatJSON, test.fresh + var stdout, stderr bytes.Buffer + cmd := &cobra.Command{ + Use: "query", Args: queryCmd.Args, RunE: queryCmd.RunE, + SilenceErrors: true, SilenceUsage: true, + } + cmd.SetOut(&stdout) + cmd.SetErr(&stderr) + cmd.SetContext(testCtx) + cmd.SetArgs([]string{"SELECT id FROM messages"}) + err := cmd.Execute() + if test.outcome == "published" { + require.NoError(err) + assert.JSONEq(`{"columns":["id"],"rows":[[9007199254740993]],"row_count":1}`, stdout.String()) + assert.Equal(int32(2), queries.Load()) + } else { + if test.outcome == "canceled" { + require.ErrorIs(err, context.Canceled) + } else { + require.ErrorContains(err, test.wantErr) + } + assert.Empty(stdout.String(), "failed queries must not emit a result") + assert.Equal(int32(1), queries.Load()) + } + assert.Contains(stderr.String(), "synthetic-job") + }) + } +} + func TestWriteQueryResult_PlainDecimalNumbers(t *testing.T) { result := &query.QueryResult{ Columns: []string{"name", "message_count", "id", "ratio"}, @@ -114,6 +234,23 @@ func TestWriteQueryResult_PlainDecimalNumbers(t *testing.T) { } } +func TestWriteQueryResultIncludesCacheFreshness(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + result := &query.QueryResult{ + Columns: []string{"count"}, Rows: [][]any{{int64(1)}}, RowCount: 1, + Cache: &query.CacheFreshness{ + Generation: "synthetic-generation", PublishedAt: time.Date(2026, 9, 23, 0, 0, 0, 0, time.UTC), + StaleReason: "1 new message", PendingAdditions: 1, + }, + } + var out bytes.Buffer + require.NoError(writeQueryResult(&out, result, "json")) + assert.Contains(out.String(), `"generation": "synthetic-generation"`) + assert.Contains(out.String(), `"stale_reason": "1 new message"`) + assert.Contains(out.String(), `"pending_additions": 1`) +} + func TestWriteQueryResult_FormatCaseInsensitive(t *testing.T) { result := &query.QueryResult{ Columns: []string{"n"}, diff --git a/cmd/msgvault/cmd/rebuild_fts_test.go b/cmd/msgvault/cmd/rebuild_fts_test.go index fab3da984..32ed69ec3 100644 --- a/cmd/msgvault/cmd/rebuild_fts_test.go +++ b/cmd/msgvault/cmd/rebuild_fts_test.go @@ -14,6 +14,9 @@ import ( ) func TestRebuildFTSUsesLocalDaemonHTTPAndPreservesStderr(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) dataDir := t.TempDir() @@ -45,10 +48,14 @@ func TestRebuildFTSUsesLocalDaemonHTTPAndPreservesStderr(t *testing.T) { HomeDir: dataDir, Data: config.DataConfig{DataDir: dataDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true doneErr := captureStderr(t) root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(&cobra.Command{ Use: rebuildFTSCmd.Use, Short: rebuildFTSCmd.Short, diff --git a/cmd/msgvault/cmd/relationships_index_http_e2e_test.go b/cmd/msgvault/cmd/relationships_index_http_e2e_test.go index e56befbb9..7515797b0 100644 --- a/cmd/msgvault/cmd/relationships_index_http_e2e_test.go +++ b/cmd/msgvault/cmd/relationships_index_http_e2e_test.go @@ -136,6 +136,8 @@ func (f relationshipIndexHTTPFixture) newServer(t *testing.T, disableLegacyViews Data: config.DataConfig{DataDir: filepath.Dir(f.dbPath)}, Server: config.ServerConfig{APIPort: 8080}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx server := api.NewServerWithOptions(api.ServerOptions{ Config: cfg, Store: &storeAPIAdapter{ diff --git a/cmd/msgvault/cmd/remove_account.go b/cmd/msgvault/cmd/remove_account.go index 5b8048dca..092946ef4 100644 --- a/cmd/msgvault/cmd/remove_account.go +++ b/cmd/msgvault/cmd/remove_account.go @@ -17,6 +17,7 @@ import ( imaplib "go.kenn.io/msgvault/internal/imap" "go.kenn.io/msgvault/internal/microsoft" "go.kenn.io/msgvault/internal/oauth" + "go.kenn.io/msgvault/internal/plaud" "go.kenn.io/msgvault/internal/slack" "go.kenn.io/msgvault/internal/sourceops" "go.kenn.io/msgvault/internal/store" @@ -122,6 +123,12 @@ func confirmRemoveAccount(r io.Reader, w io.Writer) (bool, error) { } func runRemoveAccountLocal(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger yes, err := cmd.Flags().GetBool("yes") if err != nil { return fmt.Errorf("read --yes flag: %w", err) @@ -135,7 +142,7 @@ func runRemoveAccountLocal(cmd *cobra.Command, args []string) error { return usageErr(cmd, err) } - s, cleanup, err := openWritableStoreAndInit() + s, cleanup, err := openWritableStoreAndInitForInvocation(state) if err != nil { return err } @@ -344,6 +351,12 @@ func runRemoveAccountLocal(cmd *cobra.Command, args []string) error { "Warning: could not remove Microsoft Graph token: %v\n", err, ) } + case sourceTypeMSMail: + if err := newGraphMailManager(state).DeleteToken(source.Identifier); err != nil { + fmt.Fprintf(os.Stderr, + "Warning: could not remove Microsoft Graph mail token: %v\n", err, + ) + } case sourceTypeDiscord: // Discord credential cleanup is part of the lifecycle-locked cascade // above so a concurrent guild registration cannot lose its bot token. @@ -371,6 +384,11 @@ func runRemoveAccountLocal(cmd *cobra.Command, args []string) error { ) } } + case sourceTypePlaud: + mgr := plaud.NewManager("", cfg.TokensDir(), logger) + if err := mgr.DeleteToken(source.Identifier); err != nil { + fmt.Fprintf(os.Stderr, "Warning: could not remove Plaud token: %v\n", err) + } case sourceTypeCircleback: circlebackMgr := circleback.NewManager("", cfg.TokensDir(), logger) if err := circlebackMgr.DeleteToken(source.Identifier); err != nil { diff --git a/cmd/msgvault/cmd/remove_account_test.go b/cmd/msgvault/cmd/remove_account_test.go index 00e1cf83c..81ceb263c 100644 --- a/cmd/msgvault/cmd/remove_account_test.go +++ b/cmd/msgvault/cmd/remove_account_test.go @@ -97,9 +97,12 @@ func seedQueryableMessageWithAttachment(t *testing.T, s *store.Store) { require.NoError(s.UpsertAttachment(msgID, "a.pdf", "application/pdf", storagePath, "hash-a", 10)) } -func executeRemoveAccount(t *testing.T) error { +func executeRemoveAccount(t *testing.T, contexts ...context.Context) error { t.Helper() root := newTestRootCmd() + if len(contexts) > 0 { + root.SetContext(contexts[0]) + } root.AddCommand(newRemoveAccountLocalTestCmd()) root.SetArgs([]string{"remove-account", "test@example.com", "--yes"}) if err := root.Execute(); err != nil { @@ -109,6 +112,9 @@ func executeRemoveAccount(t *testing.T) error { } func TestRemoveAccountUsesDaemonCLIRunnerAndPreservesStreams(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) requests := &atomic.Int32{} @@ -152,11 +158,14 @@ func TestRemoveAccountUsesDaemonCLIRunnerAndPreservesStreams(t *testing.T) { AllowInsecure: true, }, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = false var stdout bytes.Buffer var stderr bytes.Buffer cmd := newRemoveAccountCmd() + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetErr(&stderr) cmd.SetArgs([]string{"alice@example.com", "--yes", "--type", "gmail"}) @@ -169,6 +178,9 @@ func TestRemoveAccountUsesDaemonCLIRunnerAndPreservesStreams(t *testing.T) { } func TestRemoveAccountSourceIDUsesDaemonCLIRunner(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) @@ -207,9 +219,12 @@ func TestRemoveAccountSourceIDUsesDaemonCLIRunner(t *testing.T) { AllowInsecure: true, }, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = false cmd := newRemoveAccountCmd() + cmd.SetContext(testCtx) cmd.SetArgs([]string{"--source-id", "42", "--yes"}) require.NoError(cmd.Execute()) assert.Equal(int32(1), requests.Load()) @@ -239,6 +254,8 @@ func TestRemoveAccountSelectorValidation(t *testing.T) { } func TestRemoveAccountTypeRejectsSameTypeDisplayCollision(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) @@ -257,8 +274,11 @@ func TestRemoveAccountTypeRejectsSameTypeDisplayCollision(t *testing.T) { savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = &config.Config{HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(newRemoveAccountLocalTestCmd()) root.SetArgs([]string{"remove-account", "Work", "--yes", "--type", "gmail"}) err = root.Execute() @@ -273,6 +293,8 @@ func TestRemoveAccountTypeRejectsSameTypeDisplayCollision(t *testing.T) { } func TestRemoveAccountSourceIDDeletesOnlyExactSource(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) @@ -290,8 +312,11 @@ func TestRemoveAccountSourceIDDeletesOnlyExactSource(t *testing.T) { savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = &config.Config{HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(newRemoveAccountLocalTestCmd()) root.SetArgs([]string{ "remove-account", "--source-id", strconv.FormatInt(imap.ID, 10), "--yes", @@ -309,6 +334,9 @@ func TestRemoveAccountSourceIDDeletesOnlyExactSource(t *testing.T) { } func TestRemoveAccountPromptsBeforeDaemonCLIRunner(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) requests := &atomic.Int32{} @@ -351,11 +379,14 @@ func TestRemoveAccountPromptsBeforeDaemonCLIRunner(t *testing.T) { AllowInsecure: true, }, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = false var stdout bytes.Buffer var stderr bytes.Buffer cmd := newRemoveAccountCmd() + cmd.SetContext(testCtx) cmd.SetIn(bytes.NewBufferString("y\n")) cmd.SetOut(&stdout) cmd.SetErr(&stderr) @@ -370,6 +401,8 @@ func TestRemoveAccountPromptsBeforeDaemonCLIRunner(t *testing.T) { } func TestRemoveAccountCmd_DeletesUniqueAttachmentFiles(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) tmpDir := t.TempDir() attachmentsDir := filepath.Join(tmpDir, "attachments") @@ -390,8 +423,11 @@ func TestRemoveAccountCmd_DeletesUniqueAttachmentFiles(t *testing.T) { HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(newRemoveAccountLocalTestCmd()) root.SetArgs([]string{"remove-account", "alice@example.com", "--yes"}) require.NoError(root.Execute(), "remove-account") @@ -401,6 +437,8 @@ func TestRemoveAccountCmd_DeletesUniqueAttachmentFiles(t *testing.T) { } func TestRemoveAccountCmd_PreservesSharedAttachments(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) tmpDir := t.TempDir() attachmentsDir := filepath.Join(tmpDir, "attachments") @@ -425,8 +463,11 @@ func TestRemoveAccountCmd_PreservesSharedAttachments(t *testing.T) { HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(newRemoveAccountLocalTestCmd()) root.SetArgs([]string{"remove-account", "alice@example.com", "--yes"}) require.NoError(root.Execute(), "remove-account") @@ -436,6 +477,8 @@ func TestRemoveAccountCmd_PreservesSharedAttachments(t *testing.T) { } func TestRemoveAccountCmd_DeletesUniquePackedMappings(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) tmpDir := t.TempDir() attachmentsDir := filepath.Join(tmpDir, "attachments") @@ -473,8 +516,11 @@ func TestRemoveAccountCmd_DeletesUniquePackedMappings(t *testing.T) { HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(newRemoveAccountLocalTestCmd()) root.SetArgs([]string{"remove-account", "alice@example.com", "--yes"}) getOutput := captureStdout(t) @@ -495,12 +541,14 @@ func TestRemoveAccountCmd_DeletesUniquePackedMappings(t *testing.T) { require.ErrorIs(err, fs.ErrNotExist, "removed blob is no longer addressable by hash") _, _, err = bs.Open(thumbnailHash) require.ErrorIs(err, fs.ErrNotExist, "removed thumbnail is no longer addressable by hash") - recs, err := removed.ListPackRecords() + recs, err := removed.ListPackRecordsContext(t.Context()) require.NoError(err) assert.Len(t, recs, 1, "logical deletion leaves immutable pack reclamation to repack") } func TestRemoveAccountCmd_SkipsDeletionDuringActiveSync(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) tmpDir := t.TempDir() @@ -527,8 +575,11 @@ func TestRemoveAccountCmd_SkipsDeletionDuringActiveSync(t *testing.T) { HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(newRemoveAccountLocalTestCmd()) root.SetArgs([]string{"remove-account", "alice@example.com", "--yes"}) require.NoError(root.Execute(), "remove-account") @@ -553,6 +604,8 @@ func TestRemoveAccountCmd_SkipsDeletionDuringActiveSync(t *testing.T) { // though the sync worker may still be writing attachment files. The // pre-RemoveSource check must catch this and skip file deletion. func TestRemoveAccountCmd_SkipsDeletionWhenRemovedAccountHasActiveSync(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) tmpDir := t.TempDir() attachmentsDir := filepath.Join(tmpDir, "attachments") @@ -580,8 +633,11 @@ func TestRemoveAccountCmd_SkipsDeletionWhenRemovedAccountHasActiveSync(t *testin HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(newRemoveAccountLocalTestCmd()) // --yes bypasses the initial GetActiveSync guard so we exercise the // later file-deletion path. @@ -593,6 +649,8 @@ func TestRemoveAccountCmd_SkipsDeletionWhenRemovedAccountHasActiveSync(t *testin } func TestRemoveAccountConfirmedDoesNotBypassActiveSyncGuard(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) tmpDir := t.TempDir() @@ -614,8 +672,11 @@ func TestRemoveAccountConfirmedDoesNotBypassActiveSyncGuard(t *testing.T) { HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(newRemoveAccountLocalTestCmd()) root.SetArgs([]string{"remove-account", "alice@example.com", "--confirmed"}) @@ -625,6 +686,8 @@ func TestRemoveAccountConfirmedDoesNotBypassActiveSyncGuard(t *testing.T) { } func TestRemoveAccountCmd_RejectsPathTraversal(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) tmpDir := t.TempDir() attachmentsDir := filepath.Join(tmpDir, "attachments") @@ -649,8 +712,11 @@ func TestRemoveAccountCmd_RejectsPathTraversal(t *testing.T) { HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(newRemoveAccountLocalTestCmd()) root.SetArgs([]string{"remove-account", "alice@example.com", "--yes"}) require.NoError(root.Execute(), "remove-account") @@ -668,6 +734,8 @@ func TestRemoveAccountCmd_RequiresEmail(t *testing.T) { } func TestRemoveAccountCmd_NotFound(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) tmpDir := t.TempDir() dbPath := tmpDir + "/msgvault.db" @@ -684,8 +752,11 @@ func TestRemoveAccountCmd_NotFound(t *testing.T) { HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(newRemoveAccountLocalTestCmd()) root.SetArgs([]string{ "remove-account", "nobody@example.com", "--yes", @@ -697,6 +768,8 @@ func TestRemoveAccountCmd_NotFound(t *testing.T) { } func TestRemoveAccountCmd_WithYesFlag(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) tmpDir := t.TempDir() dbPath := tmpDir + "/msgvault.db" @@ -716,8 +789,11 @@ func TestRemoveAccountCmd_WithYesFlag(t *testing.T) { HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(newRemoveAccountLocalTestCmd()) root.SetArgs([]string{ "remove-account", "test@example.com", "--yes", @@ -737,6 +813,8 @@ func TestRemoveAccountCmd_WithYesFlag(t *testing.T) { } func TestRemoveAccountCmd_HoldsCacheLockThroughRebuild(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) tmpDir := t.TempDir() @@ -751,6 +829,8 @@ func TestRemoveAccountCmd_HoldsCacheLockThroughRebuild(t *testing.T) { savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = &config.Config{HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx _, err = buildCache(dbPath, cfg.AnalyticsDir(), true) require.NoError(err, "initial cache build") @@ -767,7 +847,7 @@ func TestRemoveAccountCmd_HoldsCacheLockThroughRebuild(t *testing.T) { t.Cleanup(func() { removeAccountAfterCascadeHook = nil }) removeDone := make(chan error, 1) - go func() { removeDone <- executeRemoveAccount(t) }() + go func() { removeDone <- executeRemoveAccount(t, testCtx) }() select { case <-cascadePaused: @@ -805,6 +885,8 @@ func TestRemoveAccountCmd_HoldsCacheLockThroughRebuild(t *testing.T) { // staleness probes compare against, so the sync state must be gone and the // next probe must demand a full rebuild. func TestRemoveAccountCmd_FailedCacheRebuildInvalidatesSyncState(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) tmpDir := t.TempDir() @@ -823,6 +905,8 @@ func TestRemoveAccountCmd_FailedCacheRebuildInvalidatesSyncState(t *testing.T) { HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx _, err = buildCache(dbPath, cfg.AnalyticsDir(), true) require.NoError(err, "initial cache build") @@ -833,9 +917,9 @@ func TestRemoveAccountCmd_FailedCacheRebuildInvalidatesSyncState(t *testing.T) { buildCacheWriteStateFile = func(string, []byte, os.FileMode) error { return errors.New("simulated rebuild failure") } - defer func() { buildCacheWriteStateFile = os.WriteFile }() + defer func() { buildCacheWriteStateFile = writeCacheStateFile }() - err = executeRemoveAccount(t) + err = executeRemoveAccount(t, testCtx) require.Error(err, "mandatory cache rebuild failure must reach the command") require.ErrorContains(err, "account was removed") require.ErrorContains(err, "analytics cache refresh failed") @@ -857,11 +941,13 @@ func TestRemoveAccountCmd_FailedCacheRebuildInvalidatesSyncState(t *testing.T) { _, sourceErr := s.GetSourceByIdentifier("test@example.com") require.ErrorIs(sourceErr, store.ErrSourceNotFound, "source deletion committed") - _, lockErr := query.AcquireReadyCacheReadLock(context.Background(), cfg.AnalyticsDir()) + _, lockErr := query.AcquireReadyCacheReadLock(testCtx, cfg.AnalyticsDir()) assert.ErrorIs(lockErr, query.ErrCacheUnavailable) } func TestRemoveAccountCmd_CascadeFailureRestoresCache(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) tmpDir := t.TempDir() @@ -884,13 +970,15 @@ func TestRemoveAccountCmd_CascadeFailureRestoresCache(t *testing.T) { savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = &config.Config{HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx tokenPath := oauth.TokenFilePath(cfg.TokensDir(), "test@example.com") require.NoError(os.MkdirAll(filepath.Dir(tokenPath), 0o755)) require.NoError(os.WriteFile(tokenPath, []byte("token"), 0o600)) _, err = buildCache(dbPath, cfg.AnalyticsDir(), true) require.NoError(err, "initial cache build") - err = executeRemoveAccount(t) + err = executeRemoveAccount(t, testCtx) require.Error(err, "cascade failure must reach the command") require.ErrorContains(err, "simulated cascade failure") @@ -911,6 +999,8 @@ func TestRemoveAccountCmd_CascadeFailureRestoresCache(t *testing.T) { } func TestRemoveAccountCmd_CascadeFailureJoinsRecoveryFailure(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) tmpDir := t.TempDir() @@ -932,15 +1022,17 @@ func TestRemoveAccountCmd_CascadeFailureJoinsRecoveryFailure(t *testing.T) { savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = &config.Config{HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx _, err = buildCache(dbPath, cfg.AnalyticsDir(), true) require.NoError(err, "initial cache build") buildCacheWriteStateFile = func(string, []byte, os.FileMode) error { return errors.New("simulated recovery failure") } - t.Cleanup(func() { buildCacheWriteStateFile = os.WriteFile }) + t.Cleanup(func() { buildCacheWriteStateFile = writeCacheStateFile }) - err = executeRemoveAccount(t) + err = executeRemoveAccount(t, testCtx) require.Error(err) require.ErrorContains(err, "simulated cascade failure") require.ErrorContains(err, "simulated recovery failure") @@ -950,6 +1042,8 @@ func TestRemoveAccountCmd_CascadeFailureJoinsRecoveryFailure(t *testing.T) { } func TestRemoveAccountCmd_LockFailureLeavesSourceUntouched(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) tmpDir := t.TempDir() dbPath := filepath.Join(tmpDir, "msgvault.db") @@ -963,6 +1057,8 @@ func TestRemoveAccountCmd_LockFailureLeavesSourceUntouched(t *testing.T) { savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = &config.Config{HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx _, err = buildCache(dbPath, cfg.AnalyticsDir(), true) require.NoError(err, "initial cache build") @@ -971,7 +1067,7 @@ func TestRemoveAccountCmd_LockFailureLeavesSourceUntouched(t *testing.T) { require.NoError(os.Mkdir(statePath, 0o755)) require.NoError(os.WriteFile(filepath.Join(statePath, "keep"), []byte("x"), 0o600)) - err = executeRemoveAccount(t) + err = executeRemoveAccount(t, testCtx) require.Error(err, "invalidation failure must abort removal") require.ErrorContains(err, "invalidate") @@ -987,6 +1083,8 @@ func TestRemoveAccountCmd_LockFailureLeavesSourceUntouched(t *testing.T) { // keeps its DuckDB engine, so read_parquet over the messages glob must // return zero rows instead of failing on an empty directory. func TestRemoveAccountCmd_LastAccountLeavesReadableEmptyCache(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) tmpDir := t.TempDir() dbPath := filepath.Join(tmpDir, "msgvault.db") @@ -1004,11 +1102,14 @@ func TestRemoveAccountCmd_LastAccountLeavesReadableEmptyCache(t *testing.T) { HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx _, err = buildCache(dbPath, cfg.AnalyticsDir(), true) require.NoError(err, "initial cache build") root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(newRemoveAccountLocalTestCmd()) root.SetArgs([]string{"remove-account", "test@example.com", "--yes"}) require.NoError(root.Execute(), "remove last account") @@ -1020,6 +1121,8 @@ func TestRemoveAccountCmd_LastAccountLeavesReadableEmptyCache(t *testing.T) { func TestRemoveAccountCmd_DuplicateIdentifierRequiresType( t *testing.T, ) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) tmpDir := t.TempDir() @@ -1042,9 +1145,12 @@ func TestRemoveAccountCmd_DuplicateIdentifierRequiresType( HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx // Without --type should fail root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(newRemoveAccountLocalTestCmd()) root.SetArgs([]string{ "remove-account", "dup@example.com", "--yes", @@ -1056,6 +1162,7 @@ func TestRemoveAccountCmd_DuplicateIdentifierRequiresType( // With --type should succeed root2 := newTestRootCmd() + root2.SetContext(testCtx) root2.AddCommand(newRemoveAccountLocalTestCmd()) root2.SetArgs([]string{ "remove-account", "dup@example.com", @@ -1077,6 +1184,8 @@ func TestRemoveAccountCmd_DuplicateIdentifierRequiresType( } func TestRemoveAccountCmd_GmailRemovesToken(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) tmpDir := t.TempDir() dbPath := tmpDir + "/msgvault.db" @@ -1101,8 +1210,11 @@ func TestRemoveAccountCmd_GmailRemovesToken(t *testing.T) { HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(newRemoveAccountLocalTestCmd()) root.SetArgs([]string{ "remove-account", "tok@example.com", "--yes", @@ -1115,6 +1227,8 @@ func TestRemoveAccountCmd_GmailRemovesToken(t *testing.T) { } func TestRemoveAccountCmd_TeamsRemovesGraphToken(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) tmpDir := t.TempDir() dbPath := tmpDir + "/msgvault.db" @@ -1142,8 +1256,11 @@ func TestRemoveAccountCmd_TeamsRemovesGraphToken(t *testing.T) { ClientID: "client-id", }, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(newRemoveAccountLocalTestCmd()) root.SetArgs([]string{ "remove-account", "tok@example.com", "--yes", "--type", "teams", @@ -1156,6 +1273,8 @@ func TestRemoveAccountCmd_TeamsRemovesGraphToken(t *testing.T) { } func TestRemoveAccountCmd_DiscordDeletesTokenOnlyAfterFinalBotReference(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) tmpDir := t.TempDir() @@ -1179,9 +1298,12 @@ func TestRemoveAccountCmd_DiscordDeletesTokenOnlyAfterFinalBotReference(t *testi savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = &config.Config{HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx remove := func(guildID string) { root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(newRemoveAccountLocalTestCmd()) root.SetArgs([]string{"remove-account", guildID, "--yes", "--type", sourceTypeDiscord}) require.NoError(root.Execute()) @@ -1196,6 +1318,8 @@ func TestRemoveAccountCmd_DiscordDeletesTokenOnlyAfterFinalBotReference(t *testi } func TestRemoveAccountCmd_DiscordPreservesTokenDuringActiveSync(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) tmpDir := t.TempDir() st, err := store.Open(filepath.Join(tmpDir, "msgvault.db")) @@ -1215,8 +1339,11 @@ func TestRemoveAccountCmd_DiscordPreservesTokenDuringActiveSync(t *testing.T) { savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = &config.Config{HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(newRemoveAccountLocalTestCmd()) root.SetArgs([]string{"remove-account", source.Identifier, "--yes", "--type", sourceTypeDiscord}) require.NoError(root.Execute()) @@ -1225,6 +1352,8 @@ func TestRemoveAccountCmd_DiscordPreservesTokenDuringActiveSync(t *testing.T) { } func TestRemoveAccountCmd_DiscordPreservesTokenWhenRemainingBindingCannotResolve(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) tmpDir := t.TempDir() @@ -1247,9 +1376,12 @@ func TestRemoveAccountCmd_DiscordPreservesTokenWhenRemainingBindingCannotResolve savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = &config.Config{HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx readStderr := captureStderr(t) root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(newRemoveAccountLocalTestCmd()) root.SetArgs([]string{"remove-account", removed.Identifier, "--yes", "--type", sourceTypeDiscord}) require.NoError(root.Execute()) @@ -1261,6 +1393,8 @@ func TestRemoveAccountCmd_DiscordPreservesTokenWhenRemainingBindingCannotResolve } func TestDiscordAddLifecycleBlocksFinalCredentialRemovalUntilGuildRegistration(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) tmpDir := t.TempDir() @@ -1282,6 +1416,8 @@ func TestDiscordAddLifecycleBlocksFinalCredentialRemovalUntilGuildRegistration(t savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = &config.Config{HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx credentialSaved := make(chan struct{}) resumeAdd := make(chan struct{}) @@ -1319,6 +1455,7 @@ func TestDiscordAddLifecycleBlocksFinalCredentialRemovalUntilGuildRegistration(t removeDone := make(chan error, 1) go func() { root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(newRemoveAccountLocalTestCmd()) root.SetArgs([]string{"remove-account", first.Identifier, "--yes", "--type", sourceTypeDiscord}) removeDone <- root.Execute() @@ -1363,6 +1500,8 @@ func TestDiscordAddLifecycleBlocksFinalCredentialRemovalUntilGuildRegistration(t } func TestRemoveAccountCmd_CirclebackRemovesToken(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) tmpDir := t.TempDir() dbPath := tmpDir + "/msgvault.db" @@ -1386,8 +1525,11 @@ func TestRemoveAccountCmd_CirclebackRemovesToken(t *testing.T) { HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(newRemoveAccountLocalTestCmd()) root.SetArgs([]string{ "remove-account", "tok@example.com", "--yes", "--type", sourceTypeCircleback, @@ -1400,6 +1542,8 @@ func TestRemoveAccountCmd_CirclebackRemovesToken(t *testing.T) { } func TestRemoveAccountCmd_RemovesNotionMeetingSource(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) tmpDir := t.TempDir() dbPath := filepath.Join(tmpDir, "msgvault.db") @@ -1413,7 +1557,10 @@ func TestRemoveAccountCmd_RemovesNotionMeetingSource(t *testing.T) { savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = &config.Config{HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(newRemoveAccountLocalTestCmd()) root.SetArgs([]string{ "remove-account", "notion-personal", "--yes", "--type", sourceTypeNotionMeetings, @@ -1428,6 +1575,8 @@ func TestRemoveAccountCmd_RemovesNotionMeetingSource(t *testing.T) { } func TestRemoveAccountCmd_SlackRemovesToken(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) tmpDir := t.TempDir() dbPath := tmpDir + "/msgvault.db" @@ -1452,8 +1601,11 @@ func TestRemoveAccountCmd_SlackRemovesToken(t *testing.T) { HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(newRemoveAccountLocalTestCmd()) root.SetArgs([]string{ "remove-account", "T01:UME", "--yes", "--type", sourceTypeSlack, @@ -1466,6 +1618,8 @@ func TestRemoveAccountCmd_SlackRemovesToken(t *testing.T) { } func TestRemoveAccountCmd_NonGmailSkipsToken(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) tmpDir := t.TempDir() dbPath := tmpDir + "/msgvault.db" @@ -1490,8 +1644,11 @@ func TestRemoveAccountCmd_NonGmailSkipsToken(t *testing.T) { HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(newRemoveAccountLocalTestCmd()) root.SetArgs([]string{ "remove-account", "imp@example.com", "--yes", @@ -1529,6 +1686,8 @@ func TestResolveSource_IMAPDisplayName(t *testing.T) { } func TestRemoveAccountCmd_ClosedStdinReturnsError(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) tmpDir := t.TempDir() dbPath := tmpDir + "/msgvault.db" @@ -1547,6 +1706,8 @@ func TestRemoveAccountCmd_ClosedStdinReturnsError(t *testing.T) { HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx // Replace stdin with a closed pipe to simulate EOF r, w, err := os.Pipe() @@ -1562,6 +1723,7 @@ func TestRemoveAccountCmd_ClosedStdinReturnsError(t *testing.T) { // Run WITHOUT --yes so it tries to read confirmation root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(newRemoveAccountLocalTestCmd()) root.SetArgs([]string{"remove-account", "eof@example.com"}) diff --git a/cmd/msgvault/cmd/repack_attachments_test.go b/cmd/msgvault/cmd/repack_attachments_test.go index 9fe7565e9..b3ea9fb71 100644 --- a/cmd/msgvault/cmd/repack_attachments_test.go +++ b/cmd/msgvault/cmd/repack_attachments_test.go @@ -21,12 +21,14 @@ func TestRepackAttachmentsAlwaysProxiesThroughDaemonCLIRunner(t *testing.T) { `{"type":"stdout","data":"Repacked 0 blob(s) (0B) from 0 pack(s) into 0 pack(s); removed 0 old pack(s).\n"}`, `{"type":"complete"}`, ) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx var stdout bytes.Buffer cmd := &cobra.Command{ Use: repackAttachmentsCmd.Use, Args: repackAttachmentsCmd.Args, RunE: repackAttachmentsCmd.RunE, } + cmd.SetContext(testCtx) cmd.SetOut(&stdout) require.NoError(cmd.Execute()) assert.Equal(1, int(requests.Load())) diff --git a/cmd/msgvault/cmd/repair_dates.go b/cmd/msgvault/cmd/repair_dates.go index f5c548908..a0e0e5cea 100644 --- a/cmd/msgvault/cmd/repair_dates.go +++ b/cmd/msgvault/cmd/repair_dates.go @@ -13,6 +13,7 @@ import ( "time" "github.com/spf13/cobra" + "go.kenn.io/kit/atomicfile" "go.kenn.io/msgvault/internal/mime" "go.kenn.io/msgvault/internal/store" ) @@ -77,7 +78,12 @@ func runRepairDatesLocal( now time.Time, ) (runErr error) { ctx := cmd.Context() - st, cleanup, err := openWritableStoreAndInit() + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + st, cleanup, err := openWritableStoreAndInitForInvocation(state) if err != nil { return err } @@ -500,31 +506,17 @@ func writeDateRepairLedger(path string, ledger *dateRepairLedger) error { if err := os.MkdirAll(dir, 0o700); err != nil { return fmt.Errorf("create repairs directory: %w", err) } - file, err := os.CreateTemp(dir, ".dates-*.tmp") + file, err := atomicfile.Create(path) if err != nil { return fmt.Errorf("create temporary ledger: %w", err) } - tempPath := file.Name() - defer func() { _ = os.Remove(tempPath) }() - - if err := file.Chmod(0o600); err != nil { - _ = file.Close() - return fmt.Errorf("set ledger permissions: %w", err) - } + defer func() { _ = file.Abort() }() encoder := jsontext.NewEncoder(file, jsontext.WithIndentPrefix(""), jsontext.WithIndent(" ")) if err := json.MarshalEncode(encoder, ledger, json.Deterministic(true)); err != nil { - _ = file.Close() return fmt.Errorf("encode ledger: %w", err) } - if err := file.Sync(); err != nil { - _ = file.Close() - return fmt.Errorf("sync ledger: %w", err) - } - if err := file.Close(); err != nil { - return fmt.Errorf("close ledger: %w", err) - } - if err := replaceOutputFile(tempPath, path); err != nil { + if err := file.Commit(); err != nil { return fmt.Errorf("publish ledger: %w", err) } return nil diff --git a/cmd/msgvault/cmd/repair_dates_test.go b/cmd/msgvault/cmd/repair_dates_test.go index c6f089add..9309d4ddc 100644 --- a/cmd/msgvault/cmd/repair_dates_test.go +++ b/cmd/msgvault/cmd/repair_dates_test.go @@ -32,7 +32,8 @@ func TestRepairDatesAlwaysProxiesThroughDaemonCLIRunner(t *testing.T) { `{"type":"stdout","data":"Repaired 1 message(s).\n"}`, `{"type":"complete"}`, ) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx t.Setenv(daemonCLISubprocessEnv, "") var apply bool @@ -40,6 +41,7 @@ func TestRepairDatesAlwaysProxiesThroughDaemonCLIRunner(t *testing.T) { cmd := &cobra.Command{ Use: repairDatesCmd.Use, Args: repairDatesCmd.Args, RunE: repairDatesCmd.RunE, } + cmd.SetContext(testCtx) cmd.Flags().BoolVar(&apply, "apply", false, "write repaired dates") cmd.SetArgs([]string{"--apply"}) cmd.SetOut(&stdout) @@ -50,6 +52,8 @@ func TestRepairDatesAlwaysProxiesThroughDaemonCLIRunner(t *testing.T) { } func TestRunRepairDatesLocalDryRunApplyAndIdempotency(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) dataDir := t.TempDir() @@ -58,6 +62,8 @@ func TestRunRepairDatesLocalDryRunApplyAndIdempotency(t *testing.T) { HomeDir: dataDir, Data: config.DataConfig{DataDir: dataDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx t.Cleanup(func() { cfg = savedCfg }) st, err := store.OpenForTest(cfg.DatabaseDSN()) @@ -119,7 +125,7 @@ func TestRunRepairDatesLocalDryRunApplyAndIdempotency(t *testing.T) { now := time.Date(2026, 7, 23, 12, 0, 0, 123, time.UTC) var dryRunOut bytes.Buffer dryRunCmd := &cobra.Command{} - dryRunCmd.SetContext(context.Background()) + dryRunCmd.SetContext(testCtx) dryRunCmd.SetOut(&dryRunOut) require.NoError(runRepairDatesLocal(dryRunCmd, false, now)) assert.Contains(dryRunOut.String(), "Repairable: 2") @@ -145,7 +151,7 @@ func TestRunRepairDatesLocalDryRunApplyAndIdempotency(t *testing.T) { var applyOut bytes.Buffer applyCmd := &cobra.Command{} - applyCmd.SetContext(context.Background()) + applyCmd.SetContext(testCtx) applyCmd.SetOut(&applyOut) require.NoError(runRepairDatesLocal(applyCmd, true, now)) assert.Contains(applyOut.String(), "Repaired 2 message(s)") @@ -179,12 +185,12 @@ func TestRunRepairDatesLocalDryRunApplyAndIdempotency(t *testing.T) { st, err = store.OpenForTest(cfg.DatabaseDSN()) require.NoError(err) - pending, err := st.ListActivityProjectionQueueContext(t.Context(), 10) + pending, err := st.ListActivityProjectionQueueContext(testCtx, 10) require.NoError(err) assert.Len(pending, 2, "date repair must leave both changed messages queued for real projection") projectDateRepairActivity(t, st) - pending, err = st.ListActivityProjectionQueueContext(t.Context(), 10) + pending, err = st.ListActivityProjectionQueueContext(testCtx, 10) require.NoError(err) assert.Empty(pending) require.NoError(st.Close()) @@ -192,7 +198,7 @@ func TestRunRepairDatesLocalDryRunApplyAndIdempotency(t *testing.T) { var secondApplyOut bytes.Buffer secondApplyCmd := &cobra.Command{} - secondApplyCmd.SetContext(context.Background()) + secondApplyCmd.SetContext(testCtx) secondApplyCmd.SetOut(&secondApplyOut) require.NoError(runRepairDatesLocal(secondApplyCmd, true, now)) assert.Contains(secondApplyOut.String(), "Nothing to repair") @@ -204,6 +210,8 @@ func TestRunRepairDatesLocalDryRunApplyAndIdempotency(t *testing.T) { } func TestRunRepairDatesLocalReportsUnresolvedReasons(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) dataDir := t.TempDir() @@ -212,6 +220,8 @@ func TestRunRepairDatesLocalReportsUnresolvedReasons(t *testing.T) { HomeDir: dataDir, Data: config.DataConfig{DataDir: dataDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx t.Cleanup(func() { cfg = savedCfg }) st, err := store.OpenForTest(cfg.DatabaseDSN()) @@ -267,7 +277,7 @@ func TestRunRepairDatesLocalReportsUnresolvedReasons(t *testing.T) { now := time.Date(2026, 7, 23, 12, 0, 0, 0, time.UTC) var dryRunOut bytes.Buffer dryRunCmd := &cobra.Command{} - dryRunCmd.SetContext(context.Background()) + dryRunCmd.SetContext(testCtx) dryRunCmd.SetOut(&dryRunOut) require.NoError(runRepairDatesLocal(dryRunCmd, false, now)) @@ -397,6 +407,8 @@ func TestDateRepairUsesAnalyticsCacheOnlyForSQLite(t *testing.T) { } func TestRunRepairDatesLocalInvalidatesAndUnlocksCacheWhenApplyFails(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) dataDir := t.TempDir() savedCfg := cfg @@ -404,6 +416,8 @@ func TestRunRepairDatesLocalInvalidatesAndUnlocksCacheWhenApplyFails(t *testing. HomeDir: dataDir, Data: config.DataConfig{DataDir: dataDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx t.Cleanup(func() { cfg = savedCfg }) st, err := store.OpenForTest(cfg.DatabaseDSN()) @@ -447,7 +461,8 @@ func TestRunRepairDatesLocalInvalidatesAndUnlocksCacheWhenApplyFails(t *testing. require.NoError(os.WriteFile(statePath, []byte("{}\n"), 0o600)) cmd := &cobra.Command{} - cmd.SetContext(context.Background()) + cmd.SetContext(testCtx) + cmd.SetContext(testCtx) cmd.SetOut(&bytes.Buffer{}) err = runRepairDatesLocal( cmd, @@ -467,6 +482,8 @@ func TestRunRepairDatesLocalInvalidatesAndUnlocksCacheWhenApplyFails(t *testing. } func TestRunRepairDatesLocalReportsCommittedRepairWhenContactInvalidationFails(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) dataDir := t.TempDir() @@ -475,6 +492,8 @@ func TestRunRepairDatesLocalReportsCommittedRepairWhenContactInvalidationFails(t HomeDir: dataDir, Data: config.DataConfig{DataDir: dataDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx t.Cleanup(func() { cfg = savedCfg }) st, err := store.OpenForTest(cfg.DatabaseDSN()) @@ -516,7 +535,8 @@ func TestRunRepairDatesLocalReportsCommittedRepairWhenContactInvalidationFails(t require.NoError(st.Close()) cmd := &cobra.Command{} - cmd.SetContext(context.Background()) + cmd.SetContext(testCtx) + cmd.SetContext(testCtx) cmd.SetOut(&bytes.Buffer{}) err = runRepairDatesLocal( cmd, @@ -530,7 +550,7 @@ func TestRunRepairDatesLocalReportsCommittedRepairWhenContactInvalidationFails(t "the repair committed before invalidation failed") st, err = store.OpenForTest(cfg.DatabaseDSN()) require.NoError(err) - pending, err := st.ListActivityProjectionQueueContext(t.Context(), 10) + pending, err := st.ListActivityProjectionQueueContext(testCtx, 10) require.NoError(err) require.Len(pending, 1) assert.Equal(messageID, pending[0].MessageID, diff --git a/cmd/msgvault/cmd/repair_derived.go b/cmd/msgvault/cmd/repair_derived.go index 915b833f3..d4215f471 100644 --- a/cmd/msgvault/cmd/repair_derived.go +++ b/cmd/msgvault/cmd/repair_derived.go @@ -40,11 +40,16 @@ Examples: msgvault repair-derived --source-type beeper --identifier instagramgo`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg if !isDaemonCLISubprocess() { return runDaemonCLICommandHTTPFromCobra(cmd, args) } - s, cleanup, err := openWritableStoreAndInitForIngest() + s, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) if err != nil { return err } @@ -69,12 +74,12 @@ Examples: sum, rerr := rederive.Run(ctx, s, src.SourceType, src.Identifier, src.ID, progress) if ctx.Err() != nil { _, _ = fmt.Fprintln(cmd.OutOrStdout(), "\nInterrupted — re-run repair-derived to finish (idempotent).") - return rebuildCacheAfterWrite(cfg.DatabaseDSN()) + return rebuildCacheAfterWrite(cfg.DatabaseDSN(), state) } if rerr != nil { return errors.Join( fmt.Errorf("repair failed for %s: %w", label, rerr), - rebuildCacheAfterWrite(cfg.DatabaseDSN()), + rebuildCacheAfterWrite(cfg.DatabaseDSN(), state), ) } _, _ = fmt.Fprint(cmd.OutOrStdout(), formatRepairDerivedSummary(label, sum)) @@ -86,7 +91,7 @@ Examples: } } - return rebuildCacheAfterWrite(cfg.DatabaseDSN()) + return rebuildCacheAfterWrite(cfg.DatabaseDSN(), state) }, } cmd.Flags().StringArrayVar(&repairDerivedSourceTypes, "source-type", nil, diff --git a/cmd/msgvault/cmd/repair_encoding.go b/cmd/msgvault/cmd/repair_encoding.go index 0b1c772b0..2ff48efc0 100644 --- a/cmd/msgvault/cmd/repair_encoding.go +++ b/cmd/msgvault/cmd/repair_encoding.go @@ -7,12 +7,14 @@ import ( "errors" "fmt" "io" + "log/slog" "os" "strings" "unicode/utf8" "github.com/spf13/cobra" "go.kenn.io/msgvault/internal/calsync" + "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/gcal" "go.kenn.io/msgvault/internal/mime" "go.kenn.io/msgvault/internal/store" @@ -38,6 +40,11 @@ For each invalid field, it: 2. If re-parsing fails, attempts charset detection (Windows-1252, Latin-1, etc.) 3. As a last resort, replaces invalid bytes with the replacement character +Invalid RFC 822 Message-ID values are reported and left unchanged because +replacing bytes could make distinct identifiers collide. Recover their +original values separately from a verified source. The analytics cache +exports invalid Message-IDs as NULL. + This is useful after a sync that may have produced invalid UTF-8 due to charset detection issues in the MIME parser.`, RunE: func(cmd *cobra.Command, args []string) error { @@ -50,8 +57,14 @@ charset detection issues in the MIME parser.`, func runRepairEncodingLocal(cmd *cobra.Command) (runErr error) { ctx := cmd.Context() + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger - s, cleanup, err := openWritableStoreAndInit() + s, cleanup, err := openWritableStoreAndInitForInvocation(state) if err != nil { return err } @@ -79,7 +92,7 @@ func runRepairEncodingLocal(cmd *cobra.Command) (runErr error) { }() } - reembedNeededIDs, err := repairEncoding(s) + reembedNeededIDs, err := repairEncoding(s, logger) if err != nil { return err } @@ -91,7 +104,7 @@ func runRepairEncodingLocal(cmd *cobra.Command) (runErr error) { // embed_gen makes the message read as "needs embedding" again. // No-op when vector search is disabled — the column is harmless. if len(reembedNeededIDs) > 0 { - if err := repairResetEmbeddings(ctx, s, reembedNeededIDs); err != nil { + if err := repairResetEmbeddings(ctx, s, reembedNeededIDs, cfg); err != nil { fmt.Fprintf(os.Stderr, "Warning: %v\n", err) } } @@ -136,11 +149,11 @@ func runRepairEncodingLocal(cmd *cobra.Command) (runErr error) { // When vector search is disabled, openVectorBackendForRepair returns a nil // backend and this still resets embed_gen (a harmless no-op on the main DB // column) while the watermark step short-circuits. -func repairResetEmbeddings(ctx context.Context, s *store.Store, reembedNeededIDs []int64) error { +func repairResetEmbeddings(ctx context.Context, s *store.Store, reembedNeededIDs []int64, cfg *config.Config) error { // 1. Open the vector backend up front. This triggers (and marks the // ledger for) the one-time upgrade backfill BEFORE we clear embed_gen. // nil backend + nil closeFn when vector search is disabled. - backend, closeFn, err := openVectorBackendForRepair(ctx, s) + backend, closeFn, err := openVectorBackendForRepair(ctx, s, cfg) if err != nil { return fmt.Errorf("failed to open vector backend for re-embedding: %w", err) } @@ -177,6 +190,7 @@ func repairResetEmbeddings(ctx context.Context, s *store.Store, reembedNeededIDs // repairStats tracks repair statistics. type repairStats struct { + unrepairedIDs int subjects int bodyTexts int bodyHTMLs int @@ -192,17 +206,25 @@ type repairStats struct { skippedRows int } +func repairLogger(logger *slog.Logger) *slog.Logger { + if logger != nil { + return logger + } + return slog.New(slog.DiscardHandler) +} + // repairEncoding runs all repair passes over s and returns the IDs of // messages whose embedding inputs (subject, body_text, or body_html) // were modified. Callers reset embed_gen to NULL (via s.ResetEmbedGen) on // those ids so the scan-and-fill worker re-embeds them and semantic search // results don't stay stale against the repaired text. Snippet-only repairs // are NOT included because the embedder doesn't read snippet. -func repairEncoding(s *store.Store) (reembedNeededIDs []int64, err error) { +func repairEncoding(s *store.Store, logger *slog.Logger) (reembedNeededIDs []int64, err error) { + logger = repairLogger(logger) stats := &repairStats{} // Repair message text fields - reembedNeededIDs, err = repairMessageFields(s, stats) + reembedNeededIDs, err = repairMessageFields(s, stats, logger) if err != nil { return nil, err } @@ -210,26 +232,29 @@ func repairEncoding(s *store.Store) (reembedNeededIDs []int64, err error) { // Repair denormalized conversation previews after all message snippets so // each preview is derived once from the final message state. This also // catches previews stranded by a repair run from an older version. - if err := repairConversationPreviews(s, stats); err != nil { + if err := repairConversationPreviews(s, stats, logger); err != nil { return nil, err } // Repair display names in participants and message_recipients - if err := repairDisplayNames(s, stats); err != nil { + if err := repairDisplayNames(s, stats, logger); err != nil { return nil, err } // Repair other string fields that could have encoding issues - if err := repairOtherStrings(s, stats); err != nil { + if err := repairOtherStrings(s, stats, logger); err != nil { return nil, err } // Summary + if stats.unrepairedIDs > 0 { + fmt.Fprintf(os.Stderr, "Warning: %d RFC 822 Message-ID value(s) contain invalid UTF-8 and were left unchanged to avoid identifier collisions. Recover their original values separately from a verified source; the analytics cache exports them as NULL.\n", stats.unrepairedIDs) + } total := stats.subjects + stats.bodyTexts + stats.bodyHTMLs + stats.snippets + stats.displayNames + stats.labels + stats.filenames + stats.convTitles + stats.convSourceIDs + stats.convPreviews + stats.emailAddrs + stats.domains if total == 0 { - fmt.Println("No encoding repairs needed.") + fmt.Println("No text fields needed repair.") return nil, nil } @@ -277,14 +302,16 @@ func repairEncoding(s *store.Store) (reembedNeededIDs []int64, err error) { return reembedNeededIDs, nil } -func repairMessageFields(s *store.Store, stats *repairStats) (reembedNeededIDs []int64, err error) { +func repairMessageFields(s *store.Store, stats *repairStats, logger *slog.Logger) (reembedNeededIDs []int64, err error) { + logger = repairLogger(logger) fmt.Println("Scanning messages for invalid UTF-8...") db := s.DB() // Query all messages with their raw data rows, err := db.Query(` - SELECT m.id, m.message_type, m.subject, mb.body_text, mb.body_html, m.snippet, + SELECT m.id, m.message_type, m.subject, m.rfc822_message_id, + mb.body_text, mb.body_html, m.snippet, mr.raw_data, mr.compression FROM messages m LEFT JOIN message_bodies mb ON mb.message_id = m.id @@ -384,11 +411,11 @@ func repairMessageFields(s *store.Store, stats *repairStats) (reembedNeededIDs [ for rows.Next() { var id int64 var messageType string - var subject, bodyText, bodyHTML, snippet sql.NullString + var subject, messageID, bodyText, bodyHTML, snippet sql.NullString var rawData []byte var compression sql.NullString - if err := rows.Scan(&id, &messageType, &subject, &bodyText, &bodyHTML, &snippet, &rawData, &compression); err != nil { + if err := rows.Scan(&id, &messageType, &subject, &messageID, &bodyText, &bodyHTML, &snippet, &rawData, &compression); err != nil { logger.Warn("skipping message row with scan error", "error", err) stats.skippedRows++ continue @@ -405,6 +432,12 @@ func repairMessageFields(s *store.Store, stats *repairStats) (reembedNeededIDs [ var parsed *mime.Message needsRepair := false + // Message-ID is an identity key. Preserve its original bytes: both + // charset decoding and replacement characters can create collisions. + if messageID.Valid && !utf8.ValidString(messageID.String) { + stats.unrepairedIDs++ + } + // Subject if subject.Valid && !utf8.ValidString(subject.String) { if parsed == nil { @@ -486,14 +519,15 @@ func repairMessageFields(s *store.Store, stats *repairStats) (reembedNeededIDs [ if totalRepaired > 0 { fmt.Printf("Repaired %d messages\n", totalRepaired) } else { - fmt.Println("No messages needed repair") + fmt.Println("No message text fields needed repair") } return reembedNeededIDs, nil } const participantDisplayNameRepairSQL = "UPDATE participants SET display_name = ? WHERE id = ?" -func repairDisplayNames(s *store.Store, stats *repairStats) error { +func repairDisplayNames(s *store.Store, stats *repairStats, logger *slog.Logger) error { + logger = repairLogger(logger) // Repair display names in both message_recipients and participants tables tables := []struct { name string @@ -515,7 +549,7 @@ func repairDisplayNames(s *store.Store, stats *repairStats) error { for _, table := range tables { fmt.Printf("Scanning %s display names for invalid UTF-8...\n", table.name) - totalRepaired, err := repairDisplayNameTable(s, table.name, table.query, table.updateStmt, stats) + totalRepaired, err := repairDisplayNameTable(s, table.name, table.query, table.updateStmt, stats, logger) if err != nil { return err } @@ -545,7 +579,7 @@ type stringRepair struct { // still open deadlocks waiting for the connection the cursor holds. const participantEmailRepairSQL = "UPDATE participants SET email_address = ? WHERE id = ?" -func applyStringRepairs(s *store.Store, updateStmt, tableName string, batch []stringRepair) error { +func applyStringRepairs(s *store.Store, updateStmt, tableName string, batch []stringRepair, logger *slog.Logger) error { if updateStmt == participantEmailRepairSQL { // The email is an ownership surface: the store applies the rewrite // and settles attribution plus the identity revisions in ONE @@ -599,7 +633,7 @@ func applyStringRepairs(s *store.Store, updateStmt, tableName string, batch []st return nil } -func repairDisplayNameTable(s *store.Store, tableName, query, updateStmt string, stats *repairStats) (int, error) { +func repairDisplayNameTable(s *store.Store, tableName, query, updateStmt string, stats *repairStats, logger *slog.Logger) (int, error) { db := s.DB() // Read phase: collect repairs, then release the cursor before any write. @@ -643,7 +677,7 @@ func repairDisplayNameTable(s *store.Store, tableName, query, updateStmt string, totalRepaired := 0 for start := 0; start < len(repairs); start += batchSize { end := min(start+batchSize, len(repairs)) - if err := applyStringRepairs(s, updateStmt, tableName, repairs[start:end]); err != nil { + if err := applyStringRepairs(s, updateStmt, tableName, repairs[start:end], logger); err != nil { return totalRepaired, err } totalRepaired += end - start @@ -655,7 +689,8 @@ func repairDisplayNameTable(s *store.Store, tableName, query, updateStmt string, // repairConversationPreviews recomputes invalid denormalized previews from // the final message state. The compare-and-set store update preserves a // preview changed after this scan and makes a failed run safe to retry. -func repairConversationPreviews(s *store.Store, stats *repairStats) error { +func repairConversationPreviews(s *store.Store, stats *repairStats, logger *slog.Logger) error { + logger = repairLogger(logger) fmt.Println("Scanning conversations.last_message_preview for invalid UTF-8...") type previewRepair struct { @@ -710,7 +745,8 @@ func repairConversationPreviews(s *store.Store, stats *repairStats) error { } // repairOtherStrings repairs other string fields that could have encoding issues. -func repairOtherStrings(s *store.Store, stats *repairStats) error { +func repairOtherStrings(s *store.Store, stats *repairStats, logger *slog.Logger) error { + logger = repairLogger(logger) // Tables and columns to repair tables := []struct { name string @@ -768,7 +804,7 @@ func repairOtherStrings(s *store.Store, stats *repairStats) error { totalRepaired, err := repairOtherStringColumn( s, table.name, table.column, table.query, table.updateStmt, - table.counter, stats, + table.counter, stats, logger, ) if err != nil { return err @@ -790,7 +826,7 @@ func repairOtherStrings(s *store.Store, stats *repairStats) error { // applyStringRepairs (see the participantEmailRepairSQL special case), so a // committed batch never depends on later batches or a follow-up step. // It returns the number of rows repaired. -func repairOtherStringColumn(s *store.Store, tableName, column, query, updateStmt string, counter *int, stats *repairStats) (int, error) { +func repairOtherStringColumn(s *store.Store, tableName, column, query, updateStmt string, counter *int, stats *repairStats, logger *slog.Logger) (int, error) { db := s.DB() // Read phase: collect repairs, then release the cursor before any write @@ -833,7 +869,7 @@ func repairOtherStringColumn(s *store.Store, tableName, column, query, updateStm totalRepaired := 0 for start := 0; start < len(repairs); start += batchSize { end := min(start+batchSize, len(repairs)) - if err := applyStringRepairs(s, updateStmt, tableName, repairs[start:end]); err != nil { + if err := applyStringRepairs(s, updateStmt, tableName, repairs[start:end], logger); err != nil { return totalRepaired, err } totalRepaired += end - start diff --git a/cmd/msgvault/cmd/repair_encoding_http_test.go b/cmd/msgvault/cmd/repair_encoding_http_test.go index ab198d487..1e4b0a6af 100644 --- a/cmd/msgvault/cmd/repair_encoding_http_test.go +++ b/cmd/msgvault/cmd/repair_encoding_http_test.go @@ -28,11 +28,12 @@ func TestRepairEncodingUsesConfiguredRemoteHTTPAndPreservesOutput(t *testing.T) })) t.Cleanup(server.Close) - configureRemoteSyncTest(t, server.URL) + testCtx := configureRemoteSyncTest(t, server.URL) var stdout bytes.Buffer var stderr bytes.Buffer cmd := &cobra.Command{Use: repairEncodingCmd.Use, Args: repairEncodingCmd.Args, RunE: repairEncodingCmd.RunE} + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetErr(&stderr) diff --git a/cmd/msgvault/cmd/repair_encoding_test.go b/cmd/msgvault/cmd/repair_encoding_test.go index 3ff9fd9dd..432fab63c 100644 --- a/cmd/msgvault/cmd/repair_encoding_test.go +++ b/cmd/msgvault/cmd/repair_encoding_test.go @@ -2,6 +2,7 @@ package cmd import ( "fmt" + "strconv" "strings" "testing" "unicode/utf8" @@ -29,7 +30,7 @@ func TestRepairDisplayNamesBumpsParticipantRevisionWithTheRepair(t *testing.T) { require.NoError(err) stats := &repairStats{} - require.NoError(repairDisplayNames(st, stats)) + require.NoError(repairDisplayNames(st, stats, testDiscardLogger())) after, err := st.ParticipantDisplayNameRevision() require.NoError(err) assert.Equal(before+1, after, @@ -73,7 +74,7 @@ func TestRepairOtherStrings_LogsScanErrors(t *testing.T) { require.NoError(err, "insert bad label") stats := &repairStats{} - require.NoError(repairOtherStrings(st, stats), "repairOtherStrings") + require.NoError(repairOtherStrings(st, stats, testDiscardLogger()), "repairOtherStrings") // Before fix: skippedRows == 0 (scan error silently swallowed) // After fix: skippedRows == 1 (scan error counted) @@ -108,7 +109,7 @@ func TestRepairDisplayNames_LogsScanErrors(t *testing.T) { require.NoError(err, "insert bad participant") stats := &repairStats{} - require.NoError(repairDisplayNames(st, stats), "repairDisplayNames") + require.NoError(repairDisplayNames(st, stats, testDiscardLogger()), "repairDisplayNames") // Before fix: skippedRows == 0 (scan error silently swallowed) // After fix: skippedRows == 1 (scan error counted) @@ -123,14 +124,53 @@ func TestRepairEncoding_NoScanErrors(t *testing.T) { stats := &repairStats{} - _, err := repairMessageFields(st, stats) + _, err := repairMessageFields(st, stats, testDiscardLogger()) require.NoError(err, "repairMessageFields") - require.NoError(repairDisplayNames(st, stats), "repairDisplayNames") - require.NoError(repairOtherStrings(st, stats), "repairOtherStrings") + require.NoError(repairDisplayNames(st, stats, testDiscardLogger()), "repairDisplayNames") + require.NoError(repairOtherStrings(st, stats, testDiscardLogger()), "repairOtherStrings") assert.Zero(t, stats.skippedRows, "skippedRows should be 0 for valid data") } +func TestRepairEncodingPreservesAndReportsInvalidMessageIDs(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + testutil.SkipIfPostgres(t, + "inserts invalid UTF-8 bytes into a TEXT column; PostgreSQL rejects them") + st := testutil.NewTestStore(t) + db := st.DB() + + _, err := db.Exec(`INSERT INTO sources + (id, source_type, identifier, created_at, updated_at) + VALUES (1, 'test', 'test@example.com', datetime('now'), datetime('now'))`) + require.NoError(err, "insert source") + _, err = db.Exec(`INSERT INTO conversations + (id, source_id, source_conversation_id, conversation_type, title, created_at, updated_at) + VALUES (1, 1, 'conv-1', 'email_thread', 'title', datetime('now'), datetime('now'))`) + require.NoError(err, "insert conversation") + messageIDs := []string{"broken-\xff@example.test", "broken-\xfe@example.test", "broken-\uFFFD@example.test"} + for i, messageID := range messageIDs { + _, err = db.Exec(`INSERT INTO messages + (id, conversation_id, source_id, source_message_id, rfc822_message_id, + message_type, sent_at, size_estimate) + VALUES (?, 1, 1, ?, ?, 'email', datetime('now'), 1000)`, + i+1, strconv.Itoa(i+1), messageID) + require.NoError(err, "insert message") + } + + var reembedNeededIDs []int64 + stderr := captureStderrDuring(t, func() { reembedNeededIDs, err = repairEncoding(st, testDiscardLogger()) }) + require.NoError(err, "repair encoding") + for i, messageID := range messageIDs { + var got string + require.NoError(db.QueryRow(`SELECT rfc822_message_id FROM messages WHERE id = ?`, i+1).Scan(&got)) + assert.Equal(messageID, got, "repair must preserve distinct identifiers") + } + assert.Contains(stderr, "2 RFC 822 Message-ID") + assert.Contains(stderr, "left unchanged") + assert.Empty(reembedNeededIDs, "Message-ID does not feed the message embedder") +} + // TestRepairMessageFields_ReturnsReembedNeededIDs guards the re-embedding // hook: when any field that feeds the embedder (subject, body_text, // body_html) is repaired, the affected message id must appear in the @@ -189,7 +229,7 @@ func TestRepairMessageFields_ReturnsReembedNeededIDs(t *testing.T) { } stats := &repairStats{} - ids, err := repairMessageFields(st, stats) + ids, err := repairMessageFields(st, stats, testDiscardLogger()) require.NoError(err, "repairMessageFields") gotSet := map[int64]bool{} @@ -236,7 +276,7 @@ func TestRepairOtherStrings_FixesNewColumns(t *testing.T) { require.NoError(err, "insert participant") stats := &repairStats{} - require.NoError(repairOtherStrings(st, stats), "repairOtherStrings") + require.NoError(repairOtherStrings(st, stats, testDiscardLogger()), "repairOtherStrings") assert.Equal(1, stats.convSourceIDs, "convSourceIDs") assert.Equal(1, stats.emailAddrs, "emailAddrs") @@ -268,14 +308,14 @@ func TestRepairConversationPreviews_RestoresPreviewStrandedByEarlierRepair(t *te require.NoError(err, "insert previously repaired message") stats := &repairStats{} - require.NoError(repairConversationPreviews(st, stats), "repair stranded conversation preview") + require.NoError(repairConversationPreviews(st, stats, testDiscardLogger()), "repair stranded conversation preview") var got string require.NoError(db.QueryRow(`SELECT last_message_preview FROM conversations WHERE id = 1`).Scan(&got), "read repaired conversation preview") assert.Equal(want, got, "preview is rederived from the previously repaired latest message") assert.Equal(1, stats.convPreviews, "convPreviews") - require.NoError(repairConversationPreviews(st, stats), "rerun preview repair") + require.NoError(repairConversationPreviews(st, stats, testDiscardLogger()), "rerun preview repair") assert.Equal(1, stats.convPreviews, "rerun is idempotent") } @@ -308,9 +348,9 @@ func TestRepairConversationPreviews_UsesLatestMessageAfterCollidingRepairs(t *te require.NoError(err, "insert message bodies") stats := &repairStats{} - _, err = repairMessageFields(st, stats) + _, err = repairMessageFields(st, stats, testDiscardLogger()) require.NoError(err, "repair colliding message snippets") - require.NoError(repairConversationPreviews(st, stats), "repair conversation preview") + require.NoError(repairConversationPreviews(st, stats, testDiscardLogger()), "repair conversation preview") var older, latest, preview string require.NoError(db.QueryRow(`SELECT snippet FROM messages WHERE id = 10`).Scan(&older), "read older snippet") @@ -377,9 +417,9 @@ func TestRepairMessageFields_RegeneratesOnlyInvalidCalendarSnippetFromCanonicalB require.NoError(err, "store unrelated conversation preview") stats := &repairStats{} - reembedNeededIDs, err := repairMessageFields(st, stats) + reembedNeededIDs, err := repairMessageFields(st, stats, testDiscardLogger()) require.NoError(err, "repair message fields") - require.NoError(repairConversationPreviews(st, stats), "repair copied conversation preview") + require.NoError(repairConversationPreviews(st, stats, testDiscardLogger()), "repair copied conversation preview") got := make(map[int64]string) resultRows, err := db.Query(`SELECT id, snippet FROM messages WHERE id IN (10, 20, 30, 40, 50, 60, 70)`) @@ -451,7 +491,7 @@ func TestRepairOtherStrings_RefreshesOwnershipAtomicallyPerBatch(t *testing.T) { revisionBefore, err := st.AccountIdentityRevision() require.NoError(err) stats := &repairStats{} - require.NoError(repairOtherStrings(st, stats), "repairOtherStrings") + require.NoError(repairOtherStrings(st, stats, testDiscardLogger()), "repairOtherStrings") assert.Equal(broken, stats.emailAddrs) revisionAfter, err := st.AccountIdentityRevision() diff --git a/cmd/msgvault/cmd/repair_encoding_vector.go b/cmd/msgvault/cmd/repair_encoding_vector.go index 3ba42c563..eb3ccdd6a 100644 --- a/cmd/msgvault/cmd/repair_encoding_vector.go +++ b/cmd/msgvault/cmd/repair_encoding_vector.go @@ -4,9 +4,11 @@ package cmd import ( "context" + "errors" "fmt" "path/filepath" + "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/store" "go.kenn.io/msgvault/internal/vector" "go.kenn.io/msgvault/internal/vector/pgvector" @@ -30,7 +32,10 @@ import ( // // This file is compiled only with a vector backend build tag; the no-tag build // uses the stub in repair_encoding_vector_stub.go. -func openVectorBackendForRepair(ctx context.Context, s *store.Store) (vector.Backend, func() error, error) { +func openVectorBackendForRepair(ctx context.Context, s *store.Store, cfg *config.Config) (vector.Backend, func() error, error) { + if cfg == nil { + return nil, nil, errors.New("configuration is unavailable") + } if !cfg.Vector.Enabled { // Vector search disabled: nothing to open. No-op. return nil, nil, nil diff --git a/cmd/msgvault/cmd/repair_encoding_vector_stub.go b/cmd/msgvault/cmd/repair_encoding_vector_stub.go index aa1fefbd8..560a75241 100644 --- a/cmd/msgvault/cmd/repair_encoding_vector_stub.go +++ b/cmd/msgvault/cmd/repair_encoding_vector_stub.go @@ -5,6 +5,7 @@ package cmd import ( "context" + "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/store" "go.kenn.io/msgvault/internal/vector" ) @@ -14,7 +15,7 @@ import ( // run. The real implementation lives in repair_encoding_vector.go (built with // sqlite_vec or pgvector). repair-encoding still resets embed_gen on the main // DB column, which is harmless when vector search is unavailable. -func openVectorBackendForRepair(_ context.Context, _ *store.Store) (vector.Backend, func() error, error) { +func openVectorBackendForRepair(_ context.Context, _ *store.Store, _ *config.Config) (vector.Backend, func() error, error) { return nil, nil, nil } diff --git a/cmd/msgvault/cmd/repair_encoding_vector_test.go b/cmd/msgvault/cmd/repair_encoding_vector_test.go index e341de9ab..afdc2e9cc 100644 --- a/cmd/msgvault/cmd/repair_encoding_vector_test.go +++ b/cmd/msgvault/cmd/repair_encoding_vector_test.go @@ -52,6 +52,8 @@ func readEmbedGen(t *testing.T, db *sql.DB, id int64) (val int64, isNull bool) { // This FAILS with the old reset-before-open ordering (the message ends stamped // =active) and PASSES with the open-before-reset fix. func TestRepairResetEmbeddings_OpensBackendBeforeResettingEmbedGen(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) require.NoError( @@ -123,13 +125,14 @@ VALUES (1, 1, 1, 'm1', 'email'), (2, 1, 1, 'm2', 'email'); savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = &config.Config{} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) cfg.Data.DataDir = dir cfg.Vector.Enabled = true cfg.Vector.DBPath = vecPath cfg.Vector.Embeddings.Dimension = 4 require.NoError( - repairResetEmbeddings(ctx, s, []int64{1}), + repairResetEmbeddings(testCtx, s, []int64{1}, cfg), "repairResetEmbeddings") // FIX A assertion: the repaired message must end embed_gen IS NULL — the @@ -151,9 +154,10 @@ VALUES (1, 1, 1, 'm1', 'email'), (2, 1, 1, 'm2', 'email'); // with vector search disabled, repairResetEmbeddings opens no backend (no // backfill) but still clears embed_gen so the column is consistent. No error. func TestRepairResetEmbeddings_VectorDisabledStillResets(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) - ctx := context.Background() dir := t.TempDir() mainPath := filepath.Join(dir, "msgvault.db") @@ -177,10 +181,11 @@ VALUES (1, 1, 1, 'm1', 'email', 7); savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = &config.Config{} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) cfg.Data.DataDir = dir cfg.Vector.Enabled = false require.NoError( - repairResetEmbeddings(ctx, s, []int64{1}), + repairResetEmbeddings(testCtx, s, []int64{1}, cfg), "repairResetEmbeddings (vector disabled)") _, isNull := readEmbedGen(t, s.DB(), 1) diff --git a/cmd/msgvault/cmd/repair_identity.go b/cmd/msgvault/cmd/repair_identity.go index 46a73dc31..cc7033cff 100644 --- a/cmd/msgvault/cmd/repair_identity.go +++ b/cmd/msgvault/cmd/repair_identity.go @@ -42,10 +42,15 @@ Examples: msgvault repair-identity --type imap`, Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg if !isDaemonCLISubprocess() { return runDaemonCLICommandHTTPFromCobra(cmd, args) } - s, cleanup, err := openWritableStoreAndInitForIngest() + s, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) if err != nil { return err } @@ -56,7 +61,7 @@ Examples: only = strings.TrimSpace(args[0]) } repaired, rerr := repairIdentities(cmd.Context(), s, repairIdentityType, only, cmd.OutOrStdout()) - cacheErr := rebuildCacheAfterWrite(cfg.DatabaseDSN()) + cacheErr := rebuildCacheAfterWrite(cfg.DatabaseDSN(), state) if rerr != nil { return errors.Join(rerr, cacheErr) } diff --git a/cmd/msgvault/cmd/repair_labels.go b/cmd/msgvault/cmd/repair_labels.go index b609edf45..fbafc861e 100644 --- a/cmd/msgvault/cmd/repair_labels.go +++ b/cmd/msgvault/cmd/repair_labels.go @@ -49,12 +49,17 @@ Examples: } func runRepairLabelsLocal(cmd *cobra.Command, only string, apply bool) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg // A dry run still needs a writable connection: RepairIMAPSourceLabels // plans by writing inside a transaction and rolling it back, the same // idiom the store package uses elsewhere for a dry-run plan (see // errAttributeDryRun) — a read-only connection would reject the writes // outright before rollback ever came into it. - st, cleanup, err := openWritableStoreAndInit() + st, cleanup, err := openWritableStoreAndInitForInvocation(state) if err != nil { return fmt.Errorf("open archive for label repair: %w", err) } @@ -122,7 +127,7 @@ func runRepairLabelsLocal(cmd *cobra.Command, only string, apply bool) error { if !apply { return runErr } - if cacheErr := rebuildCacheAfterWrite(cfg.DatabaseDSN()); cacheErr != nil { + if cacheErr := rebuildCacheAfterWrite(cfg.DatabaseDSN(), state); cacheErr != nil { return errors.Join(runErr, cacheErr) } return runErr diff --git a/cmd/msgvault/cmd/repair_labels_test.go b/cmd/msgvault/cmd/repair_labels_test.go index 2e57d45e8..9cb74a231 100644 --- a/cmd/msgvault/cmd/repair_labels_test.go +++ b/cmd/msgvault/cmd/repair_labels_test.go @@ -2,7 +2,6 @@ package cmd import ( "bytes" - "context" "errors" "testing" @@ -20,24 +19,28 @@ import ( // issue #748 gap directly: an add-only label merge leaves a label no // imap_message_memberships row backs. func TestRunRepairLabelsLocalDryRunApplyAndNoop(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) dataDir := t.TempDir() savedCfg := cfg cfg = &config.Config{HomeDir: dataDir, Data: config.DataConfig{DataDir: dataDir}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx t.Cleanup(func() { cfg = savedCfg }) - messageID := newLabelRepairArchive(t, "labels@example.test") + messageID := newLabelRepairArchive(t, "labels@example.test", cfg) _, err := buildCache(cfg.DatabaseDSN(), cfg.AnalyticsDir(), true) require.NoError(err) - beforeRevision := labelRepairArchiveRevision(t) + beforeRevision := labelRepairArchiveRevision(t, cfg) beforeCacheState, err := query.ReadCacheSyncState(cfg.AnalyticsDir()) require.NoError(err) assert.Equal(beforeRevision, beforeCacheState.DerivedDataRevision) var dryRunOut bytes.Buffer dryRunCmd := &cobra.Command{} - dryRunCmd.SetContext(context.Background()) + dryRunCmd.SetContext(testCtx) dryRunCmd.SetOut(&dryRunOut) require.NoError(runRepairLabelsLocal(dryRunCmd, "", false)) assert.Equal( @@ -45,50 +48,54 @@ func TestRunRepairLabelsLocalDryRunApplyAndNoop(t *testing.T) { "Label repair dry run: scanned=2 changed=1\n"+ "Dry run: no rows were modified. Re-run with --apply to write repairs.\n", dryRunOut.String()) - assert.Equal([]string{"INBOX", "Stray"}, labelRepairArchiveLabels(t, messageID)) - assert.Equal(beforeRevision, labelRepairArchiveRevision(t)) + assert.Equal([]string{"INBOX", "Stray"}, labelRepairArchiveLabels(t, messageID, cfg)) + assert.Equal(beforeRevision, labelRepairArchiveRevision(t, cfg)) var applyOut bytes.Buffer applyCmd := &cobra.Command{} - applyCmd.SetContext(context.Background()) + applyCmd.SetContext(testCtx) applyCmd.SetOut(&applyOut) require.NoError(runRepairLabelsLocal(applyCmd, "", true)) assert.Equal( " labels@example.test: scanned=2 changed=1\n"+ "Label repair applied: scanned=2 changed=1\n", applyOut.String()) - assert.Equal([]string{"INBOX"}, labelRepairArchiveLabels(t, messageID)) - assert.Equal(beforeRevision+1, labelRepairArchiveRevision(t)) + assert.Equal([]string{"INBOX"}, labelRepairArchiveLabels(t, messageID, cfg)) + assert.Equal(beforeRevision+1, labelRepairArchiveRevision(t, cfg)) afterCacheState, err := query.ReadCacheSyncState(cfg.AnalyticsDir()) require.NoError(err) assert.Equal(beforeRevision+1, afterCacheState.DerivedDataRevision) var noChangeOut bytes.Buffer noChangeCmd := &cobra.Command{} - noChangeCmd.SetContext(context.Background()) + noChangeCmd.SetContext(testCtx) noChangeCmd.SetOut(&noChangeOut) require.NoError(runRepairLabelsLocal(noChangeCmd, "", true)) assert.Equal( " labels@example.test: scanned=2 changed=0\n"+ "Label repair applied: scanned=2 changed=0\n", noChangeOut.String()) - assert.Equal(beforeRevision+1, labelRepairArchiveRevision(t)) + assert.Equal(beforeRevision+1, labelRepairArchiveRevision(t, cfg)) } // TestRunRepairLabelsLocalUnknownIdentifierErrors catches a repair command // that silently matches no source instead of failing loudly when the given // identifier does not resolve to one. func TestRunRepairLabelsLocalUnknownIdentifierErrors(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) dataDir := t.TempDir() savedCfg := cfg cfg = &config.Config{HomeDir: dataDir, Data: config.DataConfig{DataDir: dataDir}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx t.Cleanup(func() { cfg = savedCfg }) - newLabelRepairArchive(t, "one@example.test") + newLabelRepairArchive(t, "one@example.test", cfg) repairCmd := &cobra.Command{} - repairCmd.SetContext(context.Background()) + repairCmd.SetContext(testCtx) repairCmd.SetOut(&bytes.Buffer{}) err := runRepairLabelsLocal(repairCmd, "someone-else@example.test", true) require.Error(err) @@ -100,11 +107,15 @@ func TestRunRepairLabelsLocalUnknownIdentifierErrors(t *testing.T) { // its imaps://user@host:port connection string, not the email a person types // on the command line — the display name carries that email. func TestRunRepairLabelsLocalIdentifierScopesByDisplayName(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) dataDir := t.TempDir() savedCfg := cfg cfg = &config.Config{HomeDir: dataDir, Data: config.DataConfig{DataDir: dataDir}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx t.Cleanup(func() { cfg = savedCfg }) st, err := store.OpenForTest(cfg.DatabaseDSN()) @@ -120,7 +131,7 @@ func TestRunRepairLabelsLocalIdentifierScopesByDisplayName(t *testing.T) { var out bytes.Buffer repairCmd := &cobra.Command{} - repairCmd.SetContext(context.Background()) + repairCmd.SetContext(testCtx) repairCmd.SetOut(&out) require.NoError(runRepairLabelsLocal(repairCmd, "scoped@example.test", true)) assert.Equal( @@ -158,25 +169,29 @@ func (w *errAfterNWriter) Write(p []byte) (int, error) { // is that a real, already-committed change is not lost from the cache just // because the command as a whole reports an error. func TestRunRepairLabelsLocalRebuildsCacheDespitePartialFailure(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) dataDir := t.TempDir() savedCfg := cfg cfg = &config.Config{HomeDir: dataDir, Data: config.DataConfig{DataDir: dataDir}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx t.Cleanup(func() { cfg = savedCfg }) - newLabelRepairArchive(t, "one@example.test") - newLabelRepairArchive(t, "two@example.test") + newLabelRepairArchive(t, "one@example.test", cfg) + newLabelRepairArchive(t, "two@example.test", cfg) _, err := buildCache(cfg.DatabaseDSN(), cfg.AnalyticsDir(), true) require.NoError(err) - beforeRevision := labelRepairArchiveRevision(t) + beforeRevision := labelRepairArchiveRevision(t, cfg) // The first source's per-source line is the only Write call allowed to // succeed; the second source's own repair still runs and commits before // its line fails to print. out := &errAfterNWriter{n: 1} repairCmd := &cobra.Command{} - repairCmd.SetContext(context.Background()) + repairCmd.SetContext(testCtx) repairCmd.SetOut(out) err = runRepairLabelsLocal(repairCmd, "", true) require.ErrorContains(err, "write label repair line") @@ -184,7 +199,7 @@ func TestRunRepairLabelsLocalRebuildsCacheDespitePartialFailure(t *testing.T) { // Both sources actually committed (the stray label from each is gone), // bumping the revision twice, and the cache rebuild ran anyway and // caught up to it — despite the command itself returning an error. - assert.Equal(beforeRevision+2, labelRepairArchiveRevision(t)) + assert.Equal(beforeRevision+2, labelRepairArchiveRevision(t, cfg)) cacheState, err := query.ReadCacheSyncState(cfg.AnalyticsDir()) require.NoError(err) assert.Equal(beforeRevision+2, cacheState.DerivedDataRevision) @@ -200,10 +215,12 @@ func TestRepairLabelsCommandRoutesThroughDaemonCLIRunner(t *testing.T) { `{"type":"stdout","data":"Label repair applied: scanned=1 changed=1\n"}`, `{"type":"complete"}`, ) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx t.Setenv(daemonCLISubprocessEnv, "") cmd := newRepairLabelsCmd() + cmd.SetContext(testCtx) var stdout bytes.Buffer cmd.SetOut(&stdout) cmd.SetArgs([]string{"--apply"}) @@ -239,8 +256,13 @@ func TestRepairLabelsCommandHelpAndFlagValidation(t *testing.T) { // then reproduces the issue #748 gap: an add-only label merge on the first // message leaves a "Stray" label no membership row backs. Returns that // message's ID. -func newLabelRepairArchive(t *testing.T, identifier string) int64 { +func newLabelRepairArchive(t *testing.T, identifier string, configs ...*config.Config) int64 { t.Helper() + cfg := testConfigValue() + if len(configs) > 0 && configs[0] != nil { + cfg = configs[0] + } + st, err := store.OpenForTest(cfg.DatabaseDSN()) require.NoError(t, err) require.NoError(t, st.InitSchema()) @@ -280,8 +302,13 @@ func newLabelRepairArchive(t *testing.T, identifier string) int64 { return messageID1 } -func labelRepairArchiveLabels(t *testing.T, messageID int64) []string { +func labelRepairArchiveLabels(t *testing.T, messageID int64, configs ...*config.Config) []string { t.Helper() + cfg := testConfigValue() + if len(configs) > 0 && configs[0] != nil { + cfg = configs[0] + } + st, err := store.OpenForTest(cfg.DatabaseDSN()) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, st.Close()) }) @@ -304,8 +331,13 @@ func labelRepairArchiveLabels(t *testing.T, messageID int64) []string { return labels } -func labelRepairArchiveRevision(t *testing.T) int64 { +func labelRepairArchiveRevision(t *testing.T, configs ...*config.Config) int64 { t.Helper() + cfg := testConfigValue() + if len(configs) > 0 && configs[0] != nil { + cfg = configs[0] + } + st, err := store.OpenForTest(cfg.DatabaseDSN()) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, st.Close()) }) diff --git a/cmd/msgvault/cmd/repair_list_ids.go b/cmd/msgvault/cmd/repair_list_ids.go index 4511ea31f..76a979213 100644 --- a/cmd/msgvault/cmd/repair_list_ids.go +++ b/cmd/msgvault/cmd/repair_list_ids.go @@ -1,6 +1,7 @@ package cmd import ( + "errors" "fmt" "github.com/spf13/cobra" @@ -30,13 +31,18 @@ works entirely offline and never contacts a provider.`, } func runRepairListIDsLocal(cmd *cobra.Command, apply bool) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg var ( st *store.Store cleanup func() err error ) if apply { - st, cleanup, err = openWritableStoreAndInit() + st, cleanup, err = openWritableStoreAndInitForInvocation(state) } else { st, err = store.OpenReadOnly(cfg.DatabaseDSN()) cleanup = func() { _ = st.Close() } @@ -67,7 +73,7 @@ func runRepairListIDsLocal(cmd *cobra.Command, apply bool) error { } } if apply && summary.Changed > 0 { - if err := rebuildCacheAfterWrite(cfg.DatabaseDSN()); err != nil { + if err := rebuildCacheAfterWrite(cfg.DatabaseDSN(), state); err != nil { return err } } diff --git a/cmd/msgvault/cmd/repair_list_ids_test.go b/cmd/msgvault/cmd/repair_list_ids_test.go index 583c4af20..42ce40b1f 100644 --- a/cmd/msgvault/cmd/repair_list_ids_test.go +++ b/cmd/msgvault/cmd/repair_list_ids_test.go @@ -2,7 +2,6 @@ package cmd import ( "bytes" - "context" "database/sql" "os" "testing" @@ -20,44 +19,48 @@ import ( // cache revision for an already-current archive. The fixture uses the archive's // real zlib MIME rows rather than a repair stub. func TestRunRepairListIDsLocalDryRunApplyAndNoop(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) dataDir := t.TempDir() savedCfg := cfg cfg = &config.Config{HomeDir: dataDir, Data: config.DataConfig{DataDir: dataDir}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx t.Cleanup(func() { cfg = savedCfg }) - messageIDs := newListIDRepairArchive(t) + messageIDs := newListIDRepairArchive(t, cfg) _, err := buildCache(cfg.DatabaseDSN(), cfg.AnalyticsDir(), true) require.NoError(err) const pendingMigration = "person_sweep_change_triggers_v5" - markListIDRepairMigrationPending(t, pendingMigration) - assert.False(listIDRepairMigrationApplied(t, pendingMigration)) - beforeRevision := listIDRepairArchiveRevision(t) + markListIDRepairMigrationPending(t, pendingMigration, cfg) + assert.False(listIDRepairMigrationApplied(t, pendingMigration, cfg)) + beforeRevision := listIDRepairArchiveRevision(t, cfg) beforeCacheState, err := query.ReadCacheSyncState(cfg.AnalyticsDir()) require.NoError(err) assert.Equal(beforeRevision, beforeCacheState.DerivedDataRevision) var dryRunOut bytes.Buffer dryRunCmd := &cobra.Command{} - dryRunCmd.SetContext(context.Background()) + dryRunCmd.SetContext(testCtx) dryRunCmd.SetOut(&dryRunOut) require.NoError(runRepairListIDsLocal(dryRunCmd, false)) assert.Equal( "List-Id repair dry run: scanned=3 found=1 changed=2 undecodable=1\n"+ "Dry run: no rows were modified. Re-run with --apply to write repairs.\n", dryRunOut.String()) - assert.Equal(sql.NullString{}, listIDRepairArchiveValue(t, messageIDs.missing)) + assert.Equal(sql.NullString{}, listIDRepairArchiveValue(t, messageIDs.missing, cfg)) assert.Equal( sql.NullString{String: "", Valid: true}, - listIDRepairArchiveValue(t, messageIDs.stale)) - assert.Equal(beforeRevision, listIDRepairArchiveRevision(t)) - assert.False(listIDRepairMigrationApplied(t, pendingMigration), + listIDRepairArchiveValue(t, messageIDs.stale, cfg)) + assert.Equal(beforeRevision, listIDRepairArchiveRevision(t, cfg)) + assert.False(listIDRepairMigrationApplied(t, pendingMigration, cfg), "dry run must not initialize schema or apply pending migrations") var applyOut bytes.Buffer applyCmd := &cobra.Command{} - applyCmd.SetContext(context.Background()) + applyCmd.SetContext(testCtx) applyCmd.SetOut(&applyOut) require.NoError(runRepairListIDsLocal(applyCmd, true)) assert.Equal( @@ -65,48 +68,58 @@ func TestRunRepairListIDsLocalDryRunApplyAndNoop(t *testing.T) { applyOut.String()) assert.Equal( sql.NullString{String: "", Valid: true}, - listIDRepairArchiveValue(t, messageIDs.missing)) - assert.Equal(sql.NullString{}, listIDRepairArchiveValue(t, messageIDs.stale)) - assert.Equal(beforeRevision+1, listIDRepairArchiveRevision(t)) + listIDRepairArchiveValue(t, messageIDs.missing, cfg)) + assert.Equal(sql.NullString{}, listIDRepairArchiveValue(t, messageIDs.stale, cfg)) + assert.Equal(beforeRevision+1, listIDRepairArchiveRevision(t, cfg)) afterCacheState, err := query.ReadCacheSyncState(cfg.AnalyticsDir()) require.NoError(err) assert.Equal(beforeRevision+1, afterCacheState.DerivedDataRevision) var noChangeOut bytes.Buffer noChangeCmd := &cobra.Command{} - noChangeCmd.SetContext(context.Background()) + noChangeCmd.SetContext(testCtx) noChangeCmd.SetOut(&noChangeOut) require.NoError(runRepairListIDsLocal(noChangeCmd, true)) assert.Equal( "List-Id repair applied: scanned=3 found=1 changed=0 undecodable=1\n", noChangeOut.String()) - assert.Equal(beforeRevision+1, listIDRepairArchiveRevision(t)) + assert.Equal(beforeRevision+1, listIDRepairArchiveRevision(t, cfg)) } func TestRunRepairListIDsLocalApplySurfacesCacheRefreshFailure(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) dataDir := t.TempDir() savedCfg := cfg cfg = &config.Config{HomeDir: dataDir, Data: config.DataConfig{DataDir: dataDir}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx t.Cleanup(func() { cfg = savedCfg }) - messageIDs := newListIDRepairArchive(t) + messageIDs := newListIDRepairArchive(t, cfg) require.NoError(os.WriteFile(cfg.AnalyticsDir(), []byte("not a directory"), 0o600)) cmd := &cobra.Command{} - cmd.SetContext(context.Background()) + cmd.SetContext(testCtx) + cmd.SetContext(testCtx) cmd.SetOut(&bytes.Buffer{}) err := runRepairListIDsLocal(cmd, true) require.Error(err) require.ErrorContains(err, "refresh analytics cache") assert.Equal( sql.NullString{String: "", Valid: true}, - listIDRepairArchiveValue(t, messageIDs.missing)) + listIDRepairArchiveValue(t, messageIDs.missing, cfg)) } -func markListIDRepairMigrationPending(t *testing.T, migration string) { +func markListIDRepairMigrationPending(t *testing.T, migration string, configs ...*config.Config) { t.Helper() + cfg := testConfigValue() + if len(configs) > 0 && configs[0] != nil { + cfg = configs[0] + } + st, err := store.OpenForTest(cfg.DatabaseDSN()) require.NoError(t, err) _, err = st.DB().Exec(st.Rebind(`DELETE FROM applied_migrations WHERE name = ?`), migration) @@ -114,8 +127,13 @@ func markListIDRepairMigrationPending(t *testing.T, migration string) { require.NoError(t, st.Close()) } -func listIDRepairMigrationApplied(t *testing.T, migration string) bool { +func listIDRepairMigrationApplied(t *testing.T, migration string, configs ...*config.Config) bool { t.Helper() + cfg := testConfigValue() + if len(configs) > 0 && configs[0] != nil { + cfg = configs[0] + } + st, err := store.OpenForTest(cfg.DatabaseDSN()) require.NoError(t, err) applied, err := st.IsMigrationApplied(migration) @@ -134,10 +152,12 @@ func TestRepairListIDsCommandRoutesThroughDaemonCLIRunner(t *testing.T) { `{"type":"stdout","data":"List-Id repair applied: scanned=1 found=1 changed=1 undecodable=0\n"}`, `{"type":"complete"}`, ) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx t.Setenv(daemonCLISubprocessEnv, "") cmd := newRepairListIDsCmd() + cmd.SetContext(testCtx) var stdout bytes.Buffer cmd.SetOut(&stdout) cmd.SetArgs([]string{"--apply"}) @@ -174,8 +194,13 @@ type listIDRepairArchiveMessageIDs struct { stale int64 } -func newListIDRepairArchive(t *testing.T) listIDRepairArchiveMessageIDs { +func newListIDRepairArchive(t *testing.T, configs ...*config.Config) listIDRepairArchiveMessageIDs { t.Helper() + cfg := testConfigValue() + if len(configs) > 0 && configs[0] != nil { + cfg = configs[0] + } + st, err := store.OpenForTest(cfg.DatabaseDSN()) require.NoError(t, err) require.NoError(t, st.InitSchema()) @@ -211,8 +236,13 @@ func newListIDRepairArchive(t *testing.T) listIDRepairArchiveMessageIDs { return listIDRepairArchiveMessageIDs{missing: missing, stale: stale} } -func listIDRepairArchiveValue(t *testing.T, messageID int64) sql.NullString { +func listIDRepairArchiveValue(t *testing.T, messageID int64, configs ...*config.Config) sql.NullString { t.Helper() + cfg := testConfigValue() + if len(configs) > 0 && configs[0] != nil { + cfg = configs[0] + } + st, err := store.OpenForTest(cfg.DatabaseDSN()) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, st.Close()) }) @@ -221,8 +251,13 @@ func listIDRepairArchiveValue(t *testing.T, messageID int64) sql.NullString { return value } -func listIDRepairArchiveRevision(t *testing.T) int64 { +func listIDRepairArchiveRevision(t *testing.T, configs ...*config.Config) int64 { t.Helper() + cfg := testConfigValue() + if len(configs) > 0 && configs[0] != nil { + cfg = configs[0] + } + st, err := store.OpenForTest(cfg.DatabaseDSN()) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, st.Close()) }) diff --git a/cmd/msgvault/cmd/repair_message.go b/cmd/msgvault/cmd/repair_message.go index 19084b78d..5c8f6c28a 100644 --- a/cmd/msgvault/cmd/repair_message.go +++ b/cmd/msgvault/cmd/repair_message.go @@ -7,11 +7,13 @@ import ( "errors" "fmt" "io" + "log/slog" "sort" "strconv" "strings" "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/daemonclient" "go.kenn.io/msgvault/internal/gmail" "go.kenn.io/msgvault/internal/store" @@ -20,6 +22,7 @@ import ( ) type repairMessageCommandDeps struct { + bind func(context.Context) repairMessageCommandDeps isDaemonSubprocess func() bool openHTTPStore func(context.Context) (*daemonclient.Client, HTTPStoreInfo, error) preflightReauth func(context.Context, *daemonclient.Client, HTTPStoreInfo, int64) error @@ -30,29 +33,78 @@ type repairMessageCommandDeps struct { attachmentsDir string } -func defaultRepairMessageCommandDeps() repairMessageCommandDeps { +func defaultRepairMessageCommandDeps(contexts ...context.Context) repairMessageCommandDeps { + if len(contexts) > 0 { + deps := defaultRepairMessageCommandDeps() + if deps.bind != nil { + return deps.bind(contexts[0]) + } + } return repairMessageCommandDeps{ + bind: defaultRepairMessageCommandDepsForContext, isDaemonSubprocess: isDaemonCLISubprocess, openHTTPStore: OpenHTTPStore, preflightReauth: func( ctx context.Context, client *daemonclient.Client, info HTTPStoreInfo, sourceID int64, ) error { - return preflightReauth(ctx, buildSyncPreflight(client, info), "", sourceID) + return preflightReauth(ctx, buildSyncPreflight(client, info, invocationFromContext(ctx)), "", sourceID) + }, + openWritableStore: func() (*store.Store, func(), error) { + return nil, nil, errors.New("configuration is unavailable") }, - openWritableStore: openWritableStoreAndInit, openReadOnlyStore: func() (*store.Store, func(), error) { - st, err := store.OpenReadOnly(cfg.DatabaseDSN()) + return nil, nil, errors.New("configuration is unavailable") + }, + newGmailClient: func(ctx context.Context, source *store.Source) (gmail.API, error) { + return buildAPIClient(ctx, source, oauthManagerCache(invocationFromContext(ctx)), nil) + }, + refreshCache: func() error { return errors.New("configuration is unavailable") }, + } +} + +func defaultRepairMessageCommandDepsForContext(ctx context.Context) repairMessageCommandDeps { + state := invocationFromContext(ctx) + var currentCfg *config.Config + if state != nil && state.cfg != nil { + currentCfg = state.cfg + } + return repairMessageCommandDeps{ + bind: defaultRepairMessageCommandDepsForContext, + isDaemonSubprocess: isDaemonCLISubprocess, + openHTTPStore: OpenHTTPStore, + preflightReauth: func( + ctx context.Context, client *daemonclient.Client, info HTTPStoreInfo, sourceID int64, + ) error { + return preflightReauth(ctx, buildSyncPreflight(client, info, state), "", sourceID) + }, + openWritableStore: func() (*store.Store, func(), error) { + return openWritableStoreAndInitForInvocation(state) + }, + openReadOnlyStore: func() (*store.Store, func(), error) { + if currentCfg == nil { + return nil, nil, errors.New("configuration is unavailable") + } + st, err := store.OpenReadOnly(currentCfg.DatabaseDSN()) if err != nil { return nil, nil, fmt.Errorf("open database read-only: %w", err) } return st, func() { _ = st.Close() }, nil }, newGmailClient: func(ctx context.Context, source *store.Source) (gmail.API, error) { - return buildAPIClient(ctx, source, oauthManagerCache(), nil) + return buildAPIClient(ctx, source, oauthManagerCache(state), nil) }, refreshCache: func() error { - return rebuildCacheAfterWrite(cfg.DatabaseDSN()) + if currentCfg == nil { + return errors.New("configuration is unavailable") + } + return rebuildCacheAfterWrite(currentCfg.DatabaseDSN(), state) }, + attachmentsDir: func() string { + if currentCfg == nil { + return "" + } + return currentCfg.AttachmentsDir() + }(), } } @@ -71,6 +123,9 @@ func newRepairMessageCmd(deps repairMessageCommandDeps) *cobra.Command { Short: "Repair one Gmail message snapshot or audit stored Gmail MIME", Args: cobra.ArbitraryArgs, RunE: func(cmd *cobra.Command, args []string) error { + if invocationFromContext(cmd.Context()) != nil && deps.bind != nil { + deps = deps.bind(cmd.Context()) + } if audit { if len(args) != 0 { return usageErr(cmd, errors.New("--audit does not accept a message reference")) @@ -206,10 +261,18 @@ func runRepairMessageLocal( defer func() { _ = closer.Close() }() } attachmentsDir := deps.attachmentsDir - if attachmentsDir == "" && cfg != nil { - attachmentsDir = cfg.AttachmentsDir() + state := invocationFromCommand(cmd) + var logger *slog.Logger + if state != nil { + logger = state.logger + } + if attachmentsDir == "" && state != nil && state.cfg != nil { + attachmentsDir = state.cfg.AttachmentsDir() + } + service := syncer.New(client, st, &syncer.Options{AttachmentsDir: attachmentsDir}) + if logger != nil { + service = service.WithLogger(logger) } - service := syncer.New(client, st, &syncer.Options{AttachmentsDir: attachmentsDir}).WithLogger(logger) result, err := service.RepairMessage(cmd.Context(), syncer.RepairRequest{ Reference: reference, SourceID: sourceID, diff --git a/cmd/msgvault/cmd/repair_message_test.go b/cmd/msgvault/cmd/repair_message_test.go index 032c0a665..ae46d642b 100644 --- a/cmd/msgvault/cmd/repair_message_test.go +++ b/cmd/msgvault/cmd/repair_message_test.go @@ -221,7 +221,7 @@ func TestRepairMessageCommandUsesDedicatedRemoteEndpoint(t *testing.T) { } })) t.Cleanup(server.Close) - configureRemoteSyncTest(t, server.URL) + testCtx := configureRemoteSyncTest(t, server.URL) deps := defaultRepairMessageCommandDeps() deps.preflightReauth = func(context.Context, *daemonclient.Client, HTTPStoreInfo, int64) error { preflightCalls.Add(1) @@ -235,6 +235,7 @@ func TestRepairMessageCommandUsesDedicatedRemoteEndpoint(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer command := newRepairMessageCmd(deps) + command.SetContext(testCtx) command.SetOut(&stdout) command.SetErr(&stderr) command.SetArgs(test.args) diff --git a/cmd/msgvault/cmd/repair_senders.go b/cmd/msgvault/cmd/repair_senders.go index f94a0972d..4c4b7b97a 100644 --- a/cmd/msgvault/cmd/repair_senders.go +++ b/cmd/msgvault/cmd/repair_senders.go @@ -5,6 +5,7 @@ import ( "crypto/sha256" "errors" "fmt" + "log/slog" "github.com/spf13/cobra" "go.kenn.io/msgvault/internal/mime" @@ -49,6 +50,11 @@ type senderRepairPlan struct { const senderRepairScanBatchSize = 100 func runRepairSendersLocal(cmd *cobra.Command, apply bool) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg ctx := cmd.Context() var ( st *store.Store @@ -56,7 +62,7 @@ func runRepairSendersLocal(cmd *cobra.Command, apply bool) error { err error ) if apply { - st, cleanup, err = openWritableStoreAndInit() + st, cleanup, err = openWritableStoreAndInitForInvocation(state) } else { st, err = store.OpenReadOnly(cfg.DatabaseDSN()) cleanup = func() { _ = st.Close() } @@ -66,7 +72,7 @@ func runRepairSendersLocal(cmd *cobra.Command, apply bool) error { } defer cleanup() - plan, err := scanAndPlanSenderRepairs(ctx, st) + plan, err := scanAndPlanSenderRepairs(ctx, st, state.logger) if err != nil { return err } @@ -102,7 +108,7 @@ func runRepairSendersLocal(cmd *cobra.Command, apply bool) error { failures = append(failures, fmt.Errorf("write sender repair result: %w", err)) } if repaired > 0 { - if err := rebuildCacheAfterWrite(cfg.DatabaseDSN()); err != nil { + if err := rebuildCacheAfterWrite(cfg.DatabaseDSN(), state); err != nil { failures = append(failures, err) } } @@ -112,7 +118,9 @@ func runRepairSendersLocal(cmd *cobra.Command, apply bool) error { func scanAndPlanSenderRepairs( ctx context.Context, st *store.Store, + logger *slog.Logger, ) (*senderRepairPlan, error) { + logger = repairLogger(logger) plan := &senderRepairPlan{} var afterMessageID int64 for { diff --git a/cmd/msgvault/cmd/repair_senders_test.go b/cmd/msgvault/cmd/repair_senders_test.go index 111bf5c2e..45ee17ca7 100644 --- a/cmd/msgvault/cmd/repair_senders_test.go +++ b/cmd/msgvault/cmd/repair_senders_test.go @@ -20,11 +20,13 @@ func TestRepairSendersAlwaysProxiesThroughDaemonCLIRunner(t *testing.T) { `{"type":"stdout","data":"Repaired: 1\n"}`, `{"type":"complete"}`, ) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx t.Setenv(daemonCLISubprocessEnv, "") var stdout bytes.Buffer cmd := newRepairSendersCmd() + cmd.SetContext(testCtx) cmd.SetArgs([]string{"--apply"}) cmd.SetOut(&stdout) @@ -34,6 +36,8 @@ func TestRepairSendersAlwaysProxiesThroughDaemonCLIRunner(t *testing.T) { } func TestRunRepairSendersLocalDryRunAndApply(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) dataDir := t.TempDir() @@ -42,6 +46,8 @@ func TestRunRepairSendersLocalDryRunAndApply(t *testing.T) { HomeDir: dataDir, Data: config.DataConfig{DataDir: dataDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx t.Cleanup(func() { cfg = savedCfg }) st, err := store.OpenForTest(cfg.DatabaseDSN()) @@ -81,32 +87,32 @@ func TestRunRepairSendersLocalDryRunAndApply(t *testing.T) { var dryRunOut bytes.Buffer dryRunCmd := &cobra.Command{} - dryRunCmd.SetContext(t.Context()) + dryRunCmd.SetContext(testCtx) dryRunCmd.SetOut(&dryRunOut) require.NoError(runRepairSendersLocal(dryRunCmd, false)) assert.Contains(dryRunOut.String(), "Candidates: 3") assert.Contains(dryRunOut.String(), "Repairable: 1") assert.Contains(dryRunOut.String(), "Unresolved: 2") assert.Contains(dryRunOut.String(), "Dry run: no rows were modified") - assert.False(readRepairSenderID(t, repairable).Valid, + assert.False(readRepairSenderID(t, repairable, cfg).Valid, "dry run must not set sender_id") var applyOut bytes.Buffer applyCmd := &cobra.Command{} - applyCmd.SetContext(t.Context()) + applyCmd.SetContext(testCtx) applyCmd.SetOut(&applyOut) require.NoError(runRepairSendersLocal(applyCmd, true)) assert.Contains(applyOut.String(), "Candidates: 3") assert.Contains(applyOut.String(), "Repairable: 1") assert.Contains(applyOut.String(), "Unresolved: 2") assert.Contains(applyOut.String(), "Repaired: 1") - require.True(readRepairSenderID(t, repairable).Valid, + require.True(readRepairSenderID(t, repairable, cfg).Valid, "apply must set sender_id") - assert.False(readRepairSenderID(t, headerless).Valid, + assert.False(readRepairSenderID(t, headerless, cfg).Valid, "headerless MIME must remain unresolved") - assert.False(readRepairSenderID(t, uninstallable).Valid, + assert.False(readRepairSenderID(t, uninstallable, cfg).Valid, "a recovered but invalid address must stay unresolved instead of failing apply") - assert.False(readRepairSenderID(t, chat).Valid, + assert.False(readRepairSenderID(t, chat, cfg).Valid, "non-email messages must remain untouched") check, err := store.OpenForTest(cfg.DatabaseDSN()) @@ -123,8 +129,13 @@ func TestRunRepairSendersLocalDryRunAndApply(t *testing.T) { assert.Equal(1, fromCount) } -func readRepairSenderID(t *testing.T, messageID int64) sql.NullInt64 { +func readRepairSenderID(t *testing.T, messageID int64, configs ...*config.Config) sql.NullInt64 { t.Helper() + cfg := testConfigValue() + if len(configs) > 0 && configs[0] != nil { + cfg = configs[0] + } + st, err := store.OpenForTest(cfg.DatabaseDSN()) require.NoError(t, err, "open sender check store") defer func() { _ = st.Close() }() diff --git a/cmd/msgvault/cmd/reply_draft.go b/cmd/msgvault/cmd/reply_draft.go index d0614999a..ed2b08c11 100644 --- a/cmd/msgvault/cmd/reply_draft.go +++ b/cmd/msgvault/cmd/reply_draft.go @@ -6,23 +6,25 @@ func init() { rootCmd.AddCommand(newDraftReplyCommand()) } +const draftFromFlag = "from" + func newDraftReplyCommand() *cobra.Command { command := &cobra.Command{ Use: "draft-reply ", - Short: "Create an IMAP reply draft from an archived message", + Short: "Create an IMAP or Gmail reply draft from an archived message", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - if !cmd.Flags().Changed("from") { - return usageErr(cmd, errDraftReplyFromRequired) - } if !cmd.Flags().Changed("body") { return usageErr(cmd, errDraftReplyBodyRequired) } return runDaemonCLICommandHTTPFromCobra(cmd, args) }, } - command.Flags().String("from", "", "confirmed source identity for the draft") + command.Flags().String(draftFromFlag, "", "confirmed source identity for the draft") command.Flags().String("body", "", "reply body") + command.Flags().Bool("all", false, "reply to the parent sender and visible recipients") + command.Flags().String("account", "", "destination source account or display name") + command.Flags().Int64("source-id", 0, "exact destination source ID") command.Flags().Bool("json", false, "emit one JSON result") return command } diff --git a/cmd/msgvault/cmd/reply_draft_test.go b/cmd/msgvault/cmd/reply_draft_test.go index ab9de241e..c7a61e9c8 100644 --- a/cmd/msgvault/cmd/reply_draft_test.go +++ b/cmd/msgvault/cmd/reply_draft_test.go @@ -38,6 +38,18 @@ func TestDraftReplyArgsRejectsDuplicateJSON(t *testing.T) { requirements.ErrorContains(err, "invalid_args") } +func TestDraftReplyArgsAcceptsSourceAndReplyAll(t *testing.T) { + assertions := assert.New(t) + requirements := require.New(t) + intent, err := parseDraftReplyArgs([]string{ + "draft-reply", "42", "--source-id", "7", "--all", "--from", "alice@example.com", "--body", "body", + }) + requirements.NoError(err) + assertions.Equal(int64(7), intent.SourceID) + assertions.True(intent.SourceIDSet) + assertions.True(intent.ReplyAll) +} + func TestAuthorizeIMAPDraftUsesExactSource(t *testing.T) { assertions := assert.New(t) requirements := require.New(t) diff --git a/cmd/msgvault/cmd/root.go b/cmd/msgvault/cmd/root.go index 7f5b77c19..84b93c295 100644 --- a/cmd/msgvault/cmd/root.go +++ b/cmd/msgvault/cmd/root.go @@ -21,203 +21,194 @@ import ( "golang.org/x/oauth2" ) -var ( - cfgFile string - homeDir string - verbose bool - useLocal bool // Use local daemon even when remote is configured - logFile string - logLevel string - noLogFile bool - logSQL bool - logSQLSlow int64 - cfg *config.Config - // logger is always non-nil so code paths outside the normal - // PersistentPreRunE flow (tests, library embeds) don't have - // to nil-check before calling logger.Info. PersistentPreRunE - // replaces this with a properly configured multi-handler at - // CLI startup. - logger = slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo})) - logResult *logging.Result // non-nil after PersistentPreRunE runs -) +var rootCmd = newRootCommand() -var rootCmd = &cobra.Command{ - Use: daemonService, - Short: "Offline email, chat, and meeting archive tool", - Long: `msgvault is an offline archive tool that exports and stores email, +func newRootCommand() *cobra.Command { + root := &cobra.Command{ + Use: daemonService, + Short: "Offline email, chat, and meeting archive tool", + Long: `msgvault is an offline archive tool that exports and stores email, chat, and meeting data locally with full-text search capabilities. This is the Go implementation providing sync, search, and TUI functionality in a single binary.`, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - // Cobra's command lifecycle (v1.10.x) is: - // 1. ParseFlags + ValidateArgs (Args:) - // 2. PersistentPreRunE ← we are here - // 3. PreRunE - // 4. ValidateRequiredFlags ← MarkFlagRequired - // 5. ValidateFlagGroups ← MarkFlagsMutuallyExclusive - // 6. RunE - // - // Errors from (2) (config load, logger setup) are runtime - // failures: hide the usage block. Errors from (4)/(5) are - // invocation-contract failures: keep the usage block. To get - // both, silence usage on entry and clear it before a successful - // return so the subsequent built-in validators see the default - // (usage on). Each command's RunE is wrapped separately (see - // silenceUsageInRunE) to re-silence usage once those validators - // have run; usageErr() flips it back on for RunE-internal - // invocation-contract violations. - cmd.SilenceUsage = true - - // Agent-delegated mode: detect before any local owner lifecycle. - // Only commands in agentDelegatedCapable's set may run this way. - // Reject flags that are meaningless in delegated mode, then skip - // config.Load, EnsureHomeDir, and logging init entirely — a - // delegated invocation must not depend on local configuration or - // writable local storage. - if isAgentMode() { - if !agentDelegatedCapable(cmd) { - return fmt.Errorf("%s is not available in agent-delegated mode", cmd.Name()) - } - if cfgFile != "" { - return errors.New("--config is not allowed in agent-delegated mode") + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + inv := prepareInvocation(cmd) + if inv == nil { + return errors.New("missing invocation state") } - if homeDir != "" { - return errors.New("--home is not allowed in agent-delegated mode") + // Cobra's command lifecycle (v1.10.x) is: + // 1. ParseFlags + ValidateArgs (Args:) + // 2. PersistentPreRunE ← we are here + // 3. PreRunE + // 4. ValidateRequiredFlags ← MarkFlagRequired + // 5. ValidateFlagGroups ← MarkFlagsMutuallyExclusive + // 6. RunE + // + // Errors from (2) (config load, logger setup) are runtime + // failures: hide the usage block. Errors from (4)/(5) are + // invocation-contract failures: keep the usage block. To get + // both, silence usage on entry and clear it before a successful + // return so the subsequent built-in validators see the default + // (usage on). Each command's RunE is wrapped separately (see + // silenceUsageInRunE) to re-silence usage once those validators + // have run; usageErr() flips it back on for RunE-internal + // invocation-contract violations. + cmd.SilenceUsage = true + + // Agent-delegated mode: detect before any local owner lifecycle. + // Only commands in agentDelegatedCapable's set may run this way. + // Reject flags that are meaningless in delegated mode, then skip + // config.Load, EnsureHomeDir, and logging init entirely — a + // delegated invocation must not depend on local configuration or + // writable local storage. + if isAgentMode(inv) { + if !agentDelegatedCapable(cmd) { + return fmt.Errorf("%s is not available in agent-delegated mode", cmd.Name()) + } + if inv.options.cfgFile != "" { + return errors.New("--config is not allowed in agent-delegated mode") + } + if inv.options.homeDir != "" { + return errors.New("--home is not allowed in agent-delegated mode") + } + cmd.SilenceUsage = false + return nil } - cmd.SilenceUsage = false - return nil - } - // Skip config loading (and therefore logging setup) for - // commands that must run without touching disk or config. - if skipsConfigLoad(cmd) { - cmd.SilenceUsage = false - return nil - } + // Skip config loading (and therefore logging setup) for + // commands that must run without touching disk or config. + if skipsConfigLoad(cmd) { + cmd.SilenceUsage = false + return nil + } - // Load config first; logging options live under [log]. - var err error - cfg, err = config.Load(cfgFile, homeDir) - if err != nil { - return fmt.Errorf("load config: %w", err) - } - if err := cfg.EnsureHomeDir(); err != nil { - return fmt.Errorf( - "create data directory %s: %w", - cfg.HomeDir, err, - ) - } + // Load config first; logging options live under [log]. + var err error + inv.cfg, err = config.LoadWithOverrides(inv.options.cfgFile, inv.options.homeDir, serveRuntimeOverrides(cmd)) + if err != nil { + return fmt.Errorf("load config: %w", err) + } + if err := inv.cfg.EnsureHomeDir(); err != nil { + return fmt.Errorf( + "create data directory %s: %w", + inv.cfg.HomeDir, err, + ) + } - // Resolve logging options. CLI flags override config; - // --verbose forces debug level regardless of other - // settings. - var levelOverride *slog.Level - if verbose { - lv := slog.LevelDebug - levelOverride = &lv - } - levelString := logLevel - if levelString == "" { - levelString = cfg.Log.Level - } - if err := logging.ValidateLevel(levelString); err != nil { - return err - } - logsDir := cfg.LogsDir() - // File logging is opt-in: requires [log].enabled, - // [log].dir, or --log-file. --no-log-file overrides. - fileDisabled := noLogFile || (logFile == "" && !cfg.Log.Enabled && cfg.Log.Dir == "") - - // SQL tracing (--log-sql or [log].sql_trace) emits at INFO, so treat - // it as an implicit request for info-level logging: skip the - // interactive-terminal quieting below that would otherwise raise the - // console level to WARN and suppress the very output the user asked for. - sqlTrace := logSQL || cfg.Log.SQLTrace - - // When the stderr fallback is the only sink and the user - // hasn't asked for a level, quiet routine INFO noise on an - // interactive terminal. The terminal check preserves INFO - // for the background daemon child (stderr → serve.log). - // The same condition means a person is reading stderr, so - // render records human-style (no timestamps or run_id) - // instead of logfmt. - humanConsole := false - if levelOverride == nil && !sqlTrace { - stderrIsTerminal := isatty.IsTerminal(os.Stderr.Fd()) || - isatty.IsCygwinTerminal(os.Stderr.Fd()) - if consoleLevel := logging.ResolveConsoleLevel( - levelString, verbose, fileDisabled, stderrIsTerminal, isDaemonConsoleSubprocess(), - ); consoleLevel != nil { - levelOverride = consoleLevel - humanConsole = true + // Resolve logging options. CLI flags override config; + // --verbose forces debug level regardless of other + // settings. + var levelOverride *slog.Level + if inv.options.verbose { + lv := slog.LevelDebug + levelOverride = &lv + } + levelString := inv.options.logLevel + if levelString == "" { + levelString = inv.cfg.Log.Level + } + if err := logging.ValidateLevel(levelString); err != nil { + return err + } + logsDir := inv.cfg.LogsDir() + // File logging is opt-in: requires [log].enabled, + // [log].dir, or --log-file. --no-log-file overrides. + fileDisabled := inv.options.noLogFile || (inv.options.logFile == "" && !inv.cfg.Log.Enabled && inv.cfg.Log.Dir == "") + + // SQL tracing (--log-sql or [log].sql_trace) emits at INFO, so treat + // it as an implicit request for info-level logging: skip the + // interactive-terminal quieting below that would otherwise raise the + // console level to WARN and suppress the very output the user asked for. + sqlTrace := inv.options.logSQL || inv.cfg.Log.SQLTrace + + // When the stderr fallback is the only sink and the user + // hasn't asked for a level, quiet routine INFO noise on an + // interactive terminal. The terminal check preserves INFO + // for the background daemon child (stderr → serve.log). + // The same condition means a person is reading stderr, so + // render records human-style (no timestamps or run_id) + // instead of logfmt. + humanConsole := false + if levelOverride == nil && !sqlTrace { + stderrIsTerminal := isatty.IsTerminal(os.Stderr.Fd()) || + isatty.IsCygwinTerminal(os.Stderr.Fd()) + if consoleLevel := logging.ResolveConsoleLevel( + levelString, inv.options.verbose, fileDisabled, stderrIsTerminal, isDaemonConsoleSubprocess(), + ); consoleLevel != nil { + levelOverride = consoleLevel + humanConsole = true + } } - } - // Close a previous log handler if tests re-enter - // PersistentPreRunE without going through ExecuteContext. - if logResult != nil { - logResult.Close() - logResult = nil - } + // Close a previous log handler if tests re-enter + // PersistentPreRunE without going through ExecuteContext. + if inv.logResult != nil { + inv.logResult.Close() + inv.logResult = nil + } - logResult, err = logging.BuildHandler(logging.Options{ - LogsDir: logsDir, - FilePath: logFile, - FileDisabled: fileDisabled, - LevelOverride: levelOverride, - LevelString: levelString, - HumanConsole: humanConsole, - }) - if err != nil { - return fmt.Errorf("build logger: %w", err) - } - logger = slog.New(logResult.Handler) - // logResult.RunID is available for any command that needs it. - slog.SetDefault(logger) - - // Configure the store's SQL logging adapter now that - // slog.Default is set. Flag overrides config; a zero - // SlowMs falls back to the built-in default (100 ms). - slowMs := logSQLSlow - if slowMs == 0 { - slowMs = cfg.Log.SQLSlowMs - } - store.ConfigureSQLLogging(store.SQLLogOptions{ - SlowMs: slowMs, - FullTrace: sqlTrace, - }) - - // Startup header: one structured line per run that - // captures everything you'd want to correlate later. - // Positional args may contain email addresses, search - // queries, or other PII — log only the count at info - // level and the full (sanitized) values at debug. - logger.Info("msgvault startup", - "command", cmd.CommandPath(), - "argc", len(args), - "version", Version, - "go_version", runtime.Version(), - "os", runtime.GOOS, - "arch", runtime.GOARCH, - "config_path", cfg.ConfigFilePath(), - "data_dir", cfg.Data.DataDir, - "log_file", logResult.FilePath, - "level", logResult.Level.String(), - ) - logger.Debug("msgvault startup args", - "args", sanitizeArgs(args), - ) - // Restore the default so cobra's required-flag and - // mutually-exclusive-flag validators (steps 4/5 above) print - // usage if they fail. - cmd.SilenceUsage = false - return nil - }, - // Note: log file closing is handled by ExecuteContext's deferred - // shutdown, which runs after the exit record is written. Do not - // close logResult in PersistentPostRunE — doing so drops the - // "msgvault exit" log line on successful runs. + inv.logResult, err = logging.BuildHandler(logging.Options{ + LogsDir: logsDir, + FilePath: inv.options.logFile, + FileDisabled: fileDisabled, + LevelOverride: levelOverride, + LevelString: levelString, + HumanConsole: humanConsole, + }) + if err != nil { + return fmt.Errorf("build logger: %w", err) + } + inv.logger = slog.New(inv.logResult.Handler) + // logResult.RunID is available for any command that needs it. + slog.SetDefault(inv.logger) + + // Configure the store's SQL logging adapter now that + // slog.Default is set. Flag overrides config; a zero + // SlowMs falls back to the built-in default (100 ms). + slowMs := inv.options.logSQLSlow + if slowMs == 0 { + slowMs = inv.cfg.Log.SQLSlowMs + } + store.ConfigureSQLLogging(store.SQLLogOptions{ + SlowMs: slowMs, + FullTrace: sqlTrace, + }) + + // Startup header: one structured line per run that + // captures everything you'd want to correlate later. + // Positional args may contain email addresses, search + // queries, or other PII — log only the count at info + // level and the full (sanitized) values at debug. + inv.logger.Info("msgvault startup", + "command", cmd.CommandPath(), + "argc", len(args), + "version", Version, + "go_version", runtime.Version(), + "os", runtime.GOOS, + "arch", runtime.GOARCH, + "config_path", inv.cfg.ConfigFilePath(), + "data_dir", inv.cfg.Data.DataDir, + "log_file", inv.logResult.FilePath, + "level", inv.logResult.Level.String(), + ) + inv.logger.Debug("msgvault startup args", + "args", sanitizeArgs(args), + ) + // Restore the default so cobra's required-flag and + // mutually-exclusive-flag validators (steps 4/5 above) print + // usage if they fail. + cmd.SilenceUsage = false + return nil + }, + // Note: log file closing is handled by ExecuteContext's deferred + // shutdown, which runs after the exit record is written. Do not + // close logResult in PersistentPostRunE — doing so drops the + // "msgvault exit" log line on successful runs. + } + registerRootFlags(root) + registerAgentFlags(root) + return root } // skipsConfigLoad reports whether cmd must run without loading config @@ -226,6 +217,7 @@ in a single binary.`, func skipsConfigLoad(cmd *cobra.Command) bool { switch cmd.Name() { case "version", "update", "quickstart", "openapi", "completion", + embeddingsOptimizeWorkerName, cobra.ShellCompRequestCmd, cobra.ShellCompNoDescRequestCmd: return true } @@ -242,8 +234,13 @@ func skipsConfigLoad(cmd *cobra.Command) bool { // work without local configuration or a local daemon are permitted; all others // must be run by the owner. func agentDelegatedCapable(cmd *cobra.Command) bool { + for ancestor := cmd; ancestor != nil; ancestor = ancestor.Parent() { + if ancestor.Name() == "calendar" { + return true + } + } switch cmd.Name() { - case "draft-reply": + case "draft-reply", "draft-compose", "draft-get", "draft-edit", "draft-delete", "draft-recover", "mcp": return true } return false @@ -293,13 +290,13 @@ func sanitizeArgs(args []string) []string { // structured log line with a stack trace before re-raising the // process exit. Called in a deferred statement at the top of // Execute/ExecuteContext so crashes always leave a trail on disk. -func recoverAndLogPanic() { +func recoverAndLogPanic(inv *invocation) { r := recover() if r == nil { return } - if logger != nil { - logger.Error("msgvault panic", + if inv != nil && inv.logger != nil { + inv.logger.Error("msgvault panic", "panic", fmt.Sprint(r), "stack", string(debug.Stack()), ) @@ -308,8 +305,8 @@ func recoverAndLogPanic() { "msgvault panic: %v\n%s\n", r, debug.Stack(), ) } - if logResult != nil { - logResult.Close() + if inv != nil && inv.logResult != nil { + inv.logResult.Close() } os.Exit(2) } @@ -325,30 +322,46 @@ func Execute() error { // Installs a panic recovery and closes the log file handler on // return so every run ends cleanly in the log. func ExecuteContext(ctx context.Context) error { - silenceUsageOnce.Do(func() { silenceUsageInRunE(rootCmd) }) + ensureSilenceUsageWrapped(rootCmd) + return executeRootContext(ctx, rootCmd) +} + +// executeRootContext gives one root execution a private owner for parsed +// options, loaded configuration and cleanup resources. The Cobra registry is +// shared by the process, so callers still serialize full tree executions. +func executeRootContext(ctx context.Context, root *cobra.Command) error { + if root == nil { + return errors.New("nil root command") + } + ensureSilenceUsageWrapped(root) + inv := newInvocation() + root.SetContext(withInvocation(ctx, inv)) // Defer ordering is load-bearing. LIFO means recoverAndLogPanic // runs before the log-file close. Because recoverAndLogPanic calls // os.Exit (which skips remaining defers), it closes logResult // itself before exiting. Do not reorder these defers. defer func() { - if logResult != nil { - logResult.Close() + if inv.logResult != nil { + inv.logResult.Close() + inv.logResult = nil } + inv.cfg = nil + clearInvocationFlags(root) }() - defer recoverAndLogPanic() + defer recoverAndLogPanic(inv) - err := rootCmd.ExecuteContext(ctx) + err := root.ExecuteContext(root.Context()) // Record the exit outcome so users can see the per-run // result in the log without parsing error messages. - if logResult != nil && logger != nil { + if inv.logResult != nil && inv.logger != nil { if err != nil { - logger.Info("msgvault exit", + inv.logger.Info("msgvault exit", "outcome", "error", "error", err.Error(), ) } else { - logger.Info("msgvault exit", "outcome", "ok") + inv.logger.Info("msgvault exit", "outcome", "ok") } } if err != nil { @@ -371,10 +384,16 @@ func usageErr(cmd *cobra.Command, err error) error { return err } -// silenceUsageOnce guards silenceUsageInRunE so each command's RunE is -// only wrapped once, even if Execute is called more than once (e.g. in -// tests that drive rootCmd repeatedly). -var silenceUsageOnce sync.Once +var silencedRoots sync.Map + +func ensureSilenceUsageWrapped(root *cobra.Command) { + if root == nil { + return + } + if _, loaded := silencedRoots.LoadOrStore(root, struct{}{}); !loaded { + silenceUsageInRunE(root) + } +} // silenceUsageInRunE walks cmd's subtree and replaces each RunE with a // wrapper that sets SilenceUsage = true on entry, then delegates to the @@ -400,13 +419,13 @@ func silenceUsageInRunE(cmd *cobra.Command) { // oauthSetupHint returns help text for OAuth configuration issues, // using the actual config file path so it's clear on all platforms. -func oauthSetupHint() string { +func oauthSetupHint(cfg *config.Config) string { configPath := "" if cfg != nil { configPath = cfg.ConfigFilePath() } hint := fmt.Sprintf(` -To use msgvault, you need a Google Cloud OAuth credential: +Gmail and Google Calendar need a Google Cloud OAuth credential: 1. Follow the setup guide: https://msgvault.io/guides/oauth-setup/ 2. Download the client_secret.json file 3. Create or edit %s: @@ -421,18 +440,18 @@ To use msgvault, you need a Google Cloud OAuth credential: // errOAuthNotConfigured returns a helpful error when OAuth client secrets are missing. // It also searches for client_secret*.json files in common locations. -func errOAuthNotConfigured() error { +func errOAuthNotConfigured(cfg *config.Config) error { // Check common locations for client_secret*.json - hint := tryFindClientSecrets() + hint := tryFindClientSecrets(cfg) if hint != "" { return fmt.Errorf("OAuth client secrets not configured.%s", hint) } - return fmt.Errorf("OAuth client secrets not configured.%s", oauthSetupHint()) + return fmt.Errorf("OAuth client secrets not configured.%s", oauthSetupHint(cfg)) } // tryFindClientSecrets looks for client_secret*.json in common locations // and returns a hint if found. -func tryFindClientSecrets() string { +func tryFindClientSecrets(cfg *config.Config) string { home, _ := os.UserHomeDir() candidates := []string{ filepath.Join(home, "Downloads", "client_secret*.json"), @@ -466,9 +485,9 @@ Or copy the file to your msgvault home directory: // wrapOAuthError wraps an oauth/client-secrets error with setup instructions // if the root cause is a missing or unreadable secrets file. -func wrapOAuthError(err error) error { +func wrapOAuthError(err error, cfg *config.Config) error { if errors.Is(err, os.ErrNotExist) || errors.Is(err, os.ErrPermission) { - return fmt.Errorf("OAuth client secrets file not accessible.%s", oauthSetupHint()) + return fmt.Errorf("OAuth client secrets file not accessible.%s", oauthSetupHint(cfg)) } return err } @@ -652,7 +671,7 @@ func authorizeManualForReauth(ctx context.Context, mgr tokenReauthorizer, email // oauthManagerCache returns a resolver function that lazily creates and // caches oauth.Manager instances keyed by app name. The cache is safe // for concurrent use (serve runs scheduled syncs in goroutines). -func oauthManagerCache() func(appName string) (*oauth.Manager, error) { +func oauthManagerCache(state *invocation) func(appName string) (*oauth.Manager, error) { var mu sync.Mutex managers := map[string]*oauth.Manager{} return func(appName string) (*oauth.Manager, error) { @@ -661,13 +680,18 @@ func oauthManagerCache() func(appName string) (*oauth.Manager, error) { if mgr, ok := managers[appName]; ok { return mgr, nil } - secretsPath, err := cfg.OAuth.ClientSecretsFor(appName) + if state == nil || state.cfg == nil || state.logger == nil { + return nil, errors.New("configuration is unavailable") + } + currentCfg := state.cfg + currentLogger := state.logger + secretsPath, err := currentCfg.OAuth.ClientSecretsFor(appName) if err != nil { return nil, err } - mgr, err := oauth.NewManager(secretsPath, cfg.TokensDir(), logger) + mgr, err := oauth.NewManager(secretsPath, currentCfg.TokensDir(), currentLogger) if err != nil { - return nil, wrapOAuthError(fmt.Errorf("create oauth manager: %w", err)) + return nil, wrapOAuthError(fmt.Errorf("create oauth manager: %w", err), currentCfg) } managers[appName] = mgr return mgr, nil @@ -683,20 +707,21 @@ func sourceOAuthApp(src *store.Source) string { return "" } -func init() { - rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default: ~/.msgvault/config.toml)") - rootCmd.PersistentFlags().StringVar(&homeDir, "home", "", "home directory (overrides MSGVAULT_HOME)") - rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "verbose output (implies --log-level=debug)") - rootCmd.PersistentFlags().BoolVar(&useLocal, localValue, false, "use local daemon instead of configured remote") - rootCmd.PersistentFlags().StringVar(&logFile, "log-file", "", +func registerRootFlags(root *cobra.Command) { + flags := root.PersistentFlags() + flags.String("config", "", "config file (default: ~/.msgvault/config.toml)") + flags.String("home", "", "home directory (overrides MSGVAULT_HOME)") + flags.BoolP("verbose", "v", false, "verbose output (implies --log-level=debug)") + flags.Bool(localValue, false, "use local daemon instead of configured remote") + flags.String("log-file", "", "override log file path (default: /logs/msgvault-YYYY-MM-DD.log)") - rootCmd.PersistentFlags().StringVar(&logLevel, "log-level", "", + flags.String("log-level", "", "log level: debug, info, warn, error (default: info)") - rootCmd.PersistentFlags().BoolVar(&noLogFile, "no-log-file", false, + flags.Bool("no-log-file", false, "disable the log file for this run (stderr output stays on)") - rootCmd.PersistentFlags().BoolVar(&logSQL, "log-sql", false, + flags.Bool("log-sql", false, "log every SQL query at info level (verbose; for debugging)") - rootCmd.PersistentFlags().Int64Var(&logSQLSlow, "log-sql-slow-ms", 0, + flags.Int64("log-sql-slow-ms", 0, "threshold in ms above which a SQL query is logged as slow "+ "(default 100; 0 uses the default)") } diff --git a/cmd/msgvault/cmd/root_test.go b/cmd/msgvault/cmd/root_test.go index f405c1526..b0da8d44c 100644 --- a/cmd/msgvault/cmd/root_test.go +++ b/cmd/msgvault/cmd/root_test.go @@ -14,6 +14,7 @@ import ( "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/oauth" extOAuth2 "golang.org/x/oauth2" ) @@ -24,7 +25,7 @@ func TestSanitizeArgsDraftBody(t *testing.T) { func TestErrOAuthNotConfigured(t *testing.T) { assert := assert.New(t) - err := errOAuthNotConfigured() + err := errOAuthNotConfigured(config.NewDefaultConfig()) require.Error(t, err, "errOAuthNotConfigured()") msg := err.Error() @@ -48,7 +49,7 @@ func TestErrOAuthNotConfigured(t *testing.T) { func TestWrapOAuthError_NotExist(t *testing.T) { originalErr := fmt.Errorf("open /path/to/secrets.json: %w", os.ErrNotExist) - wrapped := wrapOAuthError(originalErr) + wrapped := wrapOAuthError(originalErr, config.NewDefaultConfig()) msg := wrapped.Error() @@ -61,7 +62,7 @@ func TestWrapOAuthError_NotExist(t *testing.T) { func TestWrapOAuthError_Permission(t *testing.T) { originalErr := fmt.Errorf("open /path/to/secrets.json: %w", os.ErrPermission) - wrapped := wrapOAuthError(originalErr) + wrapped := wrapOAuthError(originalErr, config.NewDefaultConfig()) msg := wrapped.Error() @@ -74,7 +75,7 @@ func TestWrapOAuthError_Permission(t *testing.T) { func TestWrapOAuthError_OtherError(t *testing.T) { originalErr := errors.New("some other error") - wrapped := wrapOAuthError(originalErr) + wrapped := wrapOAuthError(originalErr, config.NewDefaultConfig()) // Should return the original error unchanged assert.Equal(t, originalErr, wrapped, "wrapOAuthError() changed unrelated error") @@ -85,7 +86,7 @@ func TestWrapOAuthError_NestedNotExist(t *testing.T) { innerErr := fmt.Errorf("file error: %w", os.ErrNotExist) outerErr := fmt.Errorf("oauth manager: %w", innerErr) - wrapped := wrapOAuthError(outerErr) + wrapped := wrapOAuthError(outerErr, config.NewDefaultConfig()) msg := wrapped.Error() @@ -99,6 +100,13 @@ func newTestRootCmd() *cobra.Command { return &cobra.Command{ Use: "msgvault", Short: "Offline email, chat, and meeting archive tool", + PersistentPreRunE: func(cmd *cobra.Command, _ []string) error { + if prepareInvocation(cmd) == nil { + return errors.New("missing invocation state") + } + cmd.SetContext(cmd.Root().Context()) + return nil + }, } } @@ -644,38 +652,59 @@ func TestGetTokenSourceWithReauthUsesScopePreservingReauth(t *testing.T) { assert.Equal(0, m.authorizeManualCount, "plain reauth call count") } -// withAgentFlags sets the global agent-mode flags and restores them on -// test cleanup. -func withAgentFlags(t *testing.T, url, tokenFile string) { +// withAgentFlags binds agent-mode options to the test invocation. +func withAgentFlags(t *testing.T) context.Context { t.Helper() - old := agentURL - oldTF := agentTokenFile - agentURL = url - agentTokenFile = tokenFile - t.Cleanup(func() { - agentURL = old - agentTokenFile = oldTF + return testInvocationContext(t.Context(), config.NewDefaultConfig(), invocationOptions{ + agentURL: "http://daemon.example:8080", + agentTokenFile: "/tmp/token", + agentURLChanged: true, + agentTokenChanged: true, }) } -// TestAgentDelegatedCapableCommandSucceeds verifies that a delegated-capable -// command (draft-reply) with agent flags passes the PersistentPreRunE -// early-return path (returns nil without loading config). +// TestAgentDelegatedCapableCommandSucceeds verifies that delegated-capable +// draft commands with agent flags pass the PersistentPreRunE early-return path. func TestAgentDelegatedCapableCommandSucceeds(t *testing.T) { - withAgentFlags(t, "http://daemon.example:8080", "/tmp/token") + for _, name := range []string{ + "draft-reply", "draft-compose", "draft-get", "draft-edit", "draft-delete", + } { + t.Run(name, func(t *testing.T) { + ctx := withAgentFlags(t) + cmd := &cobra.Command{Use: name} + cmd.SetContext(ctx) + err := rootCmd.PersistentPreRunE(cmd, nil) + require.NoError(t, err) + }) + } +} + +func TestAgentDelegatedSendAsCommandRefused(t *testing.T) { + ctx := withAgentFlags(t) + cmd := &cobra.Command{Use: "draft-send-as"} + cmd.SetContext(ctx) + err := rootCmd.PersistentPreRunE(cmd, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "not available in agent-delegated mode") +} + +func TestAgentDelegatedRecoveryCommandSucceeds(t *testing.T) { + ctx := withAgentFlags(t) - cmd := &cobra.Command{Use: "draft-reply"} + cmd := &cobra.Command{Use: "draft-recover"} + cmd.SetContext(ctx) err := rootCmd.PersistentPreRunE(cmd, nil) - require.NoError(t, err, "draft-reply with agent flags should succeed in PersistentPreRunE") + require.NoError(t, err) } // TestAgentDelegatedNonCapableCommandReturnsError verifies that a command // not in the delegated-capable set (serve) returns "not available in // agent-delegated mode" when agent flags are present. func TestAgentDelegatedNonCapableCommandReturnsError(t *testing.T) { - withAgentFlags(t, "http://daemon.example:8080", "/tmp/token") + ctx := withAgentFlags(t) cmd := &cobra.Command{Use: "serve"} + cmd.SetContext(ctx) err := rootCmd.PersistentPreRunE(cmd, nil) require.Error(t, err) assert.Contains(t, err.Error(), "not available in agent-delegated mode") diff --git a/cmd/msgvault/cmd/runtime_config_test.go b/cmd/msgvault/cmd/runtime_config_test.go new file mode 100644 index 000000000..1b6cbf66e --- /dev/null +++ b/cmd/msgvault/cmd/runtime_config_test.go @@ -0,0 +1,169 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "sync/atomic" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/fileutil" +) + +func TestRuntimeRemoteEnvironmentUsesSelectedFile(t *testing.T) { //nolint:paralleltest // process environment + assert := assert.New(t) + require := require.New(t) + keyFile := filepath.Join(t.TempDir(), "remote-key") + require.NoError(fileutil.SecureWriteFile(keyFile, []byte("remote-file-key\n"), 0o600)) + daemon := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal("remote-file-key", r.Header.Get("X-Api-Key")) + _ = json.NewEncoder(w).Encode(map[string]any{"status": "ok", "api_schema_version": api.APISchemaVersion}) + })) + t.Cleanup(daemon.Close) + t.Setenv("MSGVAULT_REMOTE_URL", daemon.URL) + t.Setenv("MSGVAULT_REMOTE_API_KEY_FILE", keyFile) + t.Setenv("MSGVAULT_REMOTE_ALLOW_INSECURE", "true") + t.Setenv("MSGVAULT_API_KEY_FILE", filepath.Join(t.TempDir(), "unused-missing-server-key")) + cfg, err := config.Load("", t.TempDir()) + require.NoError(err) + ctx := withStoreResolverConfig(t, cfg) + client, info, err := OpenHTTPStore(ctx) + require.NoError(err) + t.Cleanup(func() { assert.NoError(client.Close()) }) + assert.Equal(HTTPStoreConfiguredRemote, info.Kind) + assert.Equal(daemon.URL, info.URL) + _, err = client.Health(ctx) + require.NoError(err) +} + +func TestRuntimeLocalIgnoresUnusedRemoteSecret(t *testing.T) { //nolint:paralleltest // process environment and invocation options + assert := assert.New(t) + require := require.New(t) + t.Setenv("MSGVAULT_REMOTE_URL", "https://archive.example.test") + t.Setenv("MSGVAULT_REMOTE_API_KEY_FILE", filepath.Join(t.TempDir(), "unused-missing-remote-key")) + cfg, err := config.Load("", t.TempDir()) + require.NoError(err) + disabled := false + cfg.Server.DaemonAutoStart = &disabled + ctx := withStoreResolverConfig(t, cfg) + invocationFromContext(ctx).options.useLocal = true + _, _, err = OpenHTTPStore(ctx) + require.Error(err) + assert.NotContains(err.Error(), "remote API key") + assert.NotContains(err.Error(), "credential file") +} + +func TestExportTokenKeepsMountedKeyOutOfSavedConfig(t *testing.T) { //nolint:paralleltest // command flags are legacy globals + previousURL, previousKey, previousAllow := exportTokenTo, exportTokenAPIKey, exportAllowInsecure + t.Cleanup(func() { + exportTokenTo, exportTokenAPIKey, exportAllowInsecure = previousURL, previousKey, previousAllow + }) + for _, tt := range []struct { + name string + explicitURL bool + explicitInsecure *bool + persistedInsecure bool + tls bool + wantInsecure bool + }{ + {name: "explicit URL matches environment", explicitURL: true}, + {name: "explicit allow-insecure matches environment", explicitInsecure: new(true), wantInsecure: true}, + {name: "explicit false overrides environment and config", explicitInsecure: new(false), persistedInsecure: true, tls: true}, + {name: "environment only"}, + } { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + home := t.TempDir() + keyFile := filepath.Join(home, "key") + require.NoError(fileutil.SecureWriteFile(keyFile, []byte("mounted-export-key"), 0o600)) + path := filepath.Join(home, "config.toml") + require.NoError(fileutil.SecureWriteFile(path, fmt.Appendf(nil, "[remote]\nurl = \"http://old.example.test\"\napi_key_file = \"key\"\nallow_insecure = %t\n", tt.persistedInsecure), 0o600)) + daemon := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal("mounted-export-key", r.Header.Get("X-Api-Key")) + w.WriteHeader(http.StatusCreated) + })) + t.Cleanup(daemon.Close) + if tt.tls { + daemon.StartTLS() + } else { + daemon.Start() + } + t.Setenv("MSGVAULT_REMOTE_URL", daemon.URL) + t.Setenv("MSGVAULT_REMOTE_ALLOW_INSECURE", "true") + cfg, err := config.Load(path, home) + require.NoError(err) + require.NoError(os.MkdirAll(cfg.TokensDir(), 0o700)) + require.NoError(fileutil.SecureWriteFile(filepath.Join(cfg.TokensDir(), "account@example.test.json"), []byte(`{"access_token":"synthetic-access-token"}`), 0o600)) + exportTokenTo, exportTokenAPIKey = "", "" + wantURL := "http://old.example.test" + if tt.explicitURL { + exportTokenTo, wantURL = daemon.URL, daemon.URL + } + command := &cobra.Command{} + command.Flags().String("api-key", "", "") + command.Flags().BoolVar(&exportAllowInsecure, "allow-insecure", false, "") + if tt.explicitInsecure != nil { + require.NoError(command.Flags().Set("allow-insecure", strconv.FormatBool(*tt.explicitInsecure))) + } + command.SetContext(withStoreResolverConfig(t, cfg)) + require.NoError(runExportTokenWithClient(command, []string{"account@example.test"}, daemon.Client())) + snapshot, err := config.ReadConfigFile(path) + require.NoError(err) + saved, err := config.LoadConfigFile(snapshot, home) + require.NoError(err) + assert.Equal(wantURL, saved.Remote.URL) + assert.Equal(tt.wantInsecure, saved.Remote.AllowInsecure) + assert.Equal(keyFile, saved.Remote.APIKeyFile) + assert.Empty(saved.Remote.APIKey) + }) + } +} + +func TestExportTokenExplicitFalseRequiresHTTPS(t *testing.T) { //nolint:paralleltest // command globals and process environment + for _, source := range []string{"environment", "config"} { + t.Run(source, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + var requests atomic.Int32 + daemon := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + w.WriteHeader(http.StatusCreated) + })) + t.Cleanup(daemon.Close) + t.Setenv("MSGVAULT_REMOTE_ALLOW_INSECURE", "true") + if source == "config" { + require.NoError(os.Unsetenv("MSGVAULT_REMOTE_ALLOW_INSECURE")) + } + home := t.TempDir() + path := filepath.Join(home, "config.toml") + require.NoError(fileutil.SecureWriteFile(path, fmt.Appendf(nil, + "[remote]\nurl = %q\napi_key = 'synthetic-remote-key'\nallow_insecure = %t\n", daemon.URL, source == "config"), 0o600)) + cfg, err := config.Load(path, home) + require.NoError(err) + require.NoError(fileutil.SecureMkdirAll(cfg.TokensDir(), 0o700)) + require.NoError(fileutil.SecureWriteFile(filepath.Join(cfg.TokensDir(), "account@example.test.json"), []byte(`{"access_token":"synthetic-token"}`), 0o600)) + previousURL, previousKey, previousAllow := exportTokenTo, exportTokenAPIKey, exportAllowInsecure + t.Cleanup(func() { + exportTokenTo, exportTokenAPIKey, exportAllowInsecure = previousURL, previousKey, previousAllow + }) + exportTokenTo, exportTokenAPIKey = "", "" + command := &cobra.Command{} + command.Flags().BoolVar(&exportAllowInsecure, "allow-insecure", false, "") + require.NoError(command.Flags().Set("allow-insecure", "false")) + command.SetContext(withStoreResolverConfig(t, cfg)) + err = runExportToken(command, []string{"account@example.test"}) + require.ErrorContains(err, "HTTPS required") + assert.Zero(requests.Load(), "explicit false must prevent the token upload over HTTP") + }) + } +} diff --git a/cmd/msgvault/cmd/scope_escalation_test.go b/cmd/msgvault/cmd/scope_escalation_test.go index b868edf3a..0f6f5ef3f 100644 --- a/cmd/msgvault/cmd/scope_escalation_test.go +++ b/cmd/msgvault/cmd/scope_escalation_test.go @@ -2,7 +2,6 @@ package cmd import ( "context" - "log/slog" "os" "path/filepath" "testing" @@ -112,26 +111,19 @@ const gmailCalendarDriveTokenJSON = `{ func seedTokenEnv(t *testing.T, tokenJSON string) (tokenPath string, restore func()) { t.Helper() tmpDir := t.TempDir() + t.Setenv("MSGVAULT_HOME", tmpDir) secretsPath := filepath.Join(tmpDir, "client_secret.json") require.NoError(t, os.WriteFile(secretsPath, []byte(fakeClientSecrets), 0600), "write client secrets") + require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "config.toml"), + []byte("[oauth]\nclient_secrets = \""+filepath.ToSlash(secretsPath)+"\"\n"), 0600), "write test config") tokensDir := filepath.Join(tmpDir, "tokens") require.NoError(t, os.MkdirAll(tokensDir, 0700), "mkdir tokens") tokenPath = filepath.Join(tokensDir, scopeEscalationAccount+".json") require.NoError(t, os.WriteFile(tokenPath, []byte(tokenJSON), 0600), "write token") - savedCfg, savedLogger := cfg, logger - cfg = &config.Config{ - HomeDir: tmpDir, - Data: config.DataConfig{DataDir: tmpDir}, - OAuth: config.OAuthConfig{ClientSecrets: secretsPath}, - } - logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) - - return tokenPath, func() { - cfg, logger = savedCfg, savedLogger - } + return tokenPath, func() {} } // TestPromptScopeEscalation_PreservesTokenOnFailedReauth is the regression for @@ -141,6 +133,8 @@ func seedTokenEnv(t *testing.T, tokenJSON string) (tokenPath string, restore fun // scheduled (e.g. Gmail) sync. The flow must leave the old token intact when // re-auth does not succeed. func TestPromptScopeEscalation_PreservesTokenOnFailedReauth(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) @@ -183,10 +177,15 @@ func TestDeletionEscalationScopesForAccountPreservesCalendarGrant(t *testing.T) require := require.New(t) assert := assert.New(t) - _, restore := seedTokenEnv(t, gmailCalendarTokenJSON) + tokenPath, restore := seedTokenEnv(t, gmailCalendarTokenJSON) defer restore() + cfg := testConfigValue() + cfg.HomeDir = filepath.Dir(filepath.Dir(tokenPath)) + cfg.Data.DataDir = cfg.HomeDir + cfg.OAuth.ClientSecrets = filepath.Join(cfg.HomeDir, "client_secret.json") + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) - scopes, err := deletionEscalationScopesForAccount(scopeEscalationAccount, true, cfg.OAuth.ClientSecrets) + scopes, err := deletionEscalationScopesForAccount(scopeEscalationAccount, true, cfg.OAuth.ClientSecrets, invocationFromContext(testCtx)) require.NoError(err) assert.ElementsMatch([]string{ @@ -201,10 +200,15 @@ func TestCalendarEscalationScopesForAccountPreservesDriveGrant(t *testing.T) { require := require.New(t) assert := assert.New(t) - _, restore := seedTokenEnv(t, gmailDriveTokenJSON) + tokenPath, restore := seedTokenEnv(t, gmailDriveTokenJSON) defer restore() + cfg := testConfigValue() + cfg.HomeDir = filepath.Dir(filepath.Dir(tokenPath)) + cfg.Data.DataDir = cfg.HomeDir + cfg.OAuth.ClientSecrets = filepath.Join(cfg.HomeDir, "client_secret.json") + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) - scopes, err := calendarEscalationScopesForAccount(scopeEscalationAccount, cfg.OAuth.ClientSecrets) + scopes, err := calendarEscalationScopesForAccount(scopeEscalationAccount, cfg.OAuth.ClientSecrets, invocationFromContext(testCtx)) require.NoError(err) assert.ElementsMatch([]string{ @@ -219,10 +223,15 @@ func TestCalendarEscalationScopesForAccountDoesNotAddGmailToDriveOnlyToken(t *te require := require.New(t) assert := assert.New(t) - _, restore := seedTokenEnv(t, driveOnlyTokenJSON) + tokenPath, restore := seedTokenEnv(t, driveOnlyTokenJSON) defer restore() + cfg := testConfigValue() + cfg.HomeDir = filepath.Dir(filepath.Dir(tokenPath)) + cfg.Data.DataDir = cfg.HomeDir + cfg.OAuth.ClientSecrets = filepath.Join(cfg.HomeDir, "client_secret.json") + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) - scopes, err := calendarEscalationScopesForAccount(scopeEscalationAccount, cfg.OAuth.ClientSecrets) + scopes, err := calendarEscalationScopesForAccount(scopeEscalationAccount, cfg.OAuth.ClientSecrets, invocationFromContext(testCtx)) require.NoError(err) assert.ElementsMatch([]string{ @@ -235,10 +244,15 @@ func TestCalendarEscalationScopesForAccountPreservesLegacyTokenAsGmail(t *testin require := require.New(t) assert := assert.New(t) - _, restore := seedTokenEnv(t, legacyTokenJSON) + tokenPath, restore := seedTokenEnv(t, legacyTokenJSON) defer restore() + cfg := testConfigValue() + cfg.HomeDir = filepath.Dir(filepath.Dir(tokenPath)) + cfg.Data.DataDir = cfg.HomeDir + cfg.OAuth.ClientSecrets = filepath.Join(cfg.HomeDir, "client_secret.json") + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) - scopes, err := calendarEscalationScopesForAccount(scopeEscalationAccount, cfg.OAuth.ClientSecrets) + scopes, err := calendarEscalationScopesForAccount(scopeEscalationAccount, cfg.OAuth.ClientSecrets, invocationFromContext(testCtx)) require.NoError(err) assert.ElementsMatch([]string{ @@ -282,10 +296,15 @@ func TestDeletionEscalationScopesForAccountPreservesDriveGrant(t *testing.T) { require := require.New(t) assert := assert.New(t) - _, restore := seedTokenEnv(t, gmailCalendarDriveTokenJSON) + tokenPath, restore := seedTokenEnv(t, gmailCalendarDriveTokenJSON) defer restore() + cfg := testConfigValue() + cfg.HomeDir = filepath.Dir(filepath.Dir(tokenPath)) + cfg.Data.DataDir = cfg.HomeDir + cfg.OAuth.ClientSecrets = filepath.Join(cfg.HomeDir, "client_secret.json") + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) - scopes, err := deletionEscalationScopesForAccount(scopeEscalationAccount, true, cfg.OAuth.ClientSecrets) + scopes, err := deletionEscalationScopesForAccount(scopeEscalationAccount, true, cfg.OAuth.ClientSecrets, invocationFromContext(testCtx)) require.NoError(err) assert.ElementsMatch([]string{ @@ -301,10 +320,15 @@ func TestDeletionEscalationScopesForAccountPreservesGmailScopesWithoutCalendar(t require := require.New(t) assert := assert.New(t) - _, restore := seedTokenEnv(t, gmailOnlyTokenJSON) + tokenPath, restore := seedTokenEnv(t, gmailOnlyTokenJSON) defer restore() + cfg := testConfigValue() + cfg.HomeDir = filepath.Dir(filepath.Dir(tokenPath)) + cfg.Data.DataDir = cfg.HomeDir + cfg.OAuth.ClientSecrets = filepath.Join(cfg.HomeDir, "client_secret.json") + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) - scopes, err := deletionEscalationScopesForAccount(scopeEscalationAccount, true, cfg.OAuth.ClientSecrets) + scopes, err := deletionEscalationScopesForAccount(scopeEscalationAccount, true, cfg.OAuth.ClientSecrets, invocationFromContext(testCtx)) require.NoError(err) assert.ElementsMatch([]string{ @@ -329,12 +353,17 @@ func TestAddCalendarHeadless_PrintsInstructionsAndPreservesToken(t *testing.T) { require.NoError(err, "read seeded token") addCmd := newAddCalendarLocalCmd() - addCmd.SetContext(context.Background()) + cfg := config.NewDefaultConfig() + cfg.HomeDir = os.Getenv("MSGVAULT_HOME") + cfg.Data.DataDir = cfg.HomeDir + cfg.OAuth.ClientSecrets = filepath.Join(cfg.HomeDir, "client_secret.json") + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + addCmd.SetContext(testCtx) addCmd.SetArgs([]string{"--headless", scopeEscalationAccount}) defer func() { calAddHeadless = false }() getOutput := captureStdout(t) - execErr := addCmd.Execute() + execErr := addCmd.ExecuteContext(testCtx) out := getOutput() require.NoError(execErr, "headless add-calendar must not error or hang") @@ -352,15 +381,20 @@ func TestPlanCLIAddCalendarRequiresScopeEscalationForGmailOnlyToken(t *testing.T assert := assert.New(t) require := require.New(t) - _, restore := seedTokenEnv(t, gmailOnlyTokenJSON) + tokenPath, restore := seedTokenEnv(t, gmailOnlyTokenJSON) defer restore() + cfg := testConfigValue() + cfg.HomeDir = filepath.Dir(filepath.Dir(tokenPath)) + cfg.Data.DataDir = cfg.HomeDir + cfg.OAuth.ClientSecrets = filepath.Join(cfg.HomeDir, "client_secret.json") + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) st, err := store.Open(cfg.DatabaseDSN()) require.NoError(err, "open store") defer func() { _ = st.Close() }() require.NoError(st.InitSchema(), "init schema") - plan, err := planCLIAddCalendar(context.Background(), st, api.CLIAddCalendarPlanRequest{ + plan, err := planCLIAddCalendar(testCtx, st, api.CLIAddCalendarPlanRequest{ Email: scopeEscalationAccount, }) @@ -378,8 +412,13 @@ func TestPlanCLIAddCalendarRequiresScopeEscalationForNonReusableGmailOnlyToken(t assert := assert.New(t) require := require.New(t) - _, restore := seedTokenEnv(t, gmailOnlyOtherClientTokenJSON) + tokenPath, restore := seedTokenEnv(t, gmailOnlyOtherClientTokenJSON) defer restore() + cfg := testConfigValue() + cfg.HomeDir = filepath.Dir(filepath.Dir(tokenPath)) + cfg.Data.DataDir = cfg.HomeDir + cfg.OAuth.ClientSecrets = filepath.Join(cfg.HomeDir, "client_secret.json") + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) cfg.OAuth.Apps = map[string]config.OAuthApp{ "acme": {ClientSecrets: cfg.OAuth.ClientSecrets}, } @@ -389,7 +428,7 @@ func TestPlanCLIAddCalendarRequiresScopeEscalationForNonReusableGmailOnlyToken(t defer func() { _ = st.Close() }() require.NoError(st.InitSchema(), "init schema") - plan, err := planCLIAddCalendar(context.Background(), st, api.CLIAddCalendarPlanRequest{ + plan, err := planCLIAddCalendar(testCtx, st, api.CLIAddCalendarPlanRequest{ Email: scopeEscalationAccount, OAuthApp: "acme", OAuthAppExplicit: true, diff --git a/cmd/msgvault/cmd/search.go b/cmd/msgvault/cmd/search.go index bce70a4f8..0e7308458 100644 --- a/cmd/msgvault/cmd/search.go +++ b/cmd/msgvault/cmd/search.go @@ -3,9 +3,10 @@ package cmd import ( "errors" "fmt" + "io" "os" + "strconv" "strings" - "text/tabwriter" "time" "github.com/spf13/cobra" @@ -145,6 +146,11 @@ Examples: } func runHTTPSearch(cmd *cobra.Command, queryStr string) error { + state := invocationFromCommand(cmd) + if state == nil || state.logger == nil { + return errors.New("invocation state is unavailable") + } + logger := state.logger s, info, err := OpenHTTPStore(cmd.Context()) if err != nil { return fmt.Errorf("open store: %w", err) @@ -201,13 +207,9 @@ func runHTTPSearch(cmd *cobra.Command, queryStr string) error { // Pre-0.18 daemons built the index synchronously inside the request. fmt.Fprintf(os.Stderr, "Built search index (%d messages indexed).\n", resp.IndexedMessages) } - switch resp.IndexState { - case "building": - fmt.Fprintln(os.Stderr, - "Note: the search index is being rebuilt in the background; results may be incomplete until it finishes.") - case "checking": + if resp.IndexState == "building" { fmt.Fprintln(os.Stderr, - "Note: search index completeness is still being verified in the background; results may be incomplete until it finishes.") + "Note: the search index is rebuilding or awaiting a rebuild in the background; results may be incomplete until it finishes.") } if searchCollection != "" { label := resp.ScopeLabel @@ -243,26 +245,49 @@ func runHTTPSearch(cmd *cobra.Command, queryStr string) error { return outputSearchResultsTable(resp.Results) } -// nil error return mirrors outputSearchResultsJSON so callers can return -// either uniformly; tabwriter output never fails. func outputSearchResultsTable(results []query.MessageSummary) error { - w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) - _, _ = fmt.Fprintln(w, "ID\tDATE\tFROM\tSUBJECT\tSIZE") - _, _ = fmt.Fprintln(w, "──\t────\t────\t───────\t────") + return writeSearchResultsTable(os.Stdout, results) +} +func writeSearchResultsTable(out io.Writer, results []query.MessageSummary) error { + return writeSearchResultsTableWidth(out, results, searchTableTerminalWidth(out)) +} + +func writeSearchResultsTableWidth(out io.Writer, results []query.MessageSummary, width int) error { + rows := make([][]searchTableCell, 0, len(results)) for _, msg := range results { - date := msg.SentAt.Format("2006-01-02") - from := truncate(summaryFromDisplay(msg), 30) - subject := truncate(msg.Subject, 50) - size := formatSize(msg.SizeEstimate) - _, _ = fmt.Fprintf(w, "%d\t%s\t%s\t%s\t%s\n", msg.ID, date, from, subject, size) + rows = append(rows, []searchTableCell{ + {text: strconv.FormatInt(msg.ID, 10)}, + {text: msg.SentAt.Format("2006-01-02")}, + {text: normalizeSearchTableText(summaryFromDisplay(msg))}, + {text: summaryTableText(msg.Subject, msg.Snippet)}, + {text: formatSummarySize(msg.SizeEstimate)}, + }) + } + if err := writeSearchTable(out, []string{"ID", "DATE", "FROM", "SUBJECT", "SIZE"}, rows, width); err != nil { + return err + } + if _, err := fmt.Fprintf(out, "\n%s\n", formatShowingResults(len(results))); err != nil { + return fmt.Errorf("write search result count: %w", err) } - - _ = w.Flush() - fmt.Printf("\n%s\n", formatShowingResults(len(results))) return nil } +func summaryTableText(subject, snippet string) string { + text := normalizeSearchTableText(subject) + if text == "" { + text = normalizeSearchTableText(snippet) + } + return text +} + +func formatSummarySize(size int64) string { + if size <= 0 { + return "-" + } + return formatSize(size) +} + func summaryFromDisplay(msg query.MessageSummary) string { for _, value := range []string{msg.FromEmail, msg.FromName, msg.FromPhone} { if strings.TrimSpace(value) != "" { diff --git a/cmd/msgvault/cmd/search_status.go b/cmd/msgvault/cmd/search_status.go index 1622ae71a..2594cc07c 100644 --- a/cmd/msgvault/cmd/search_status.go +++ b/cmd/msgvault/cmd/search_status.go @@ -10,6 +10,7 @@ import ( "github.com/mattn/go-isatty" "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/config" ) // Vars rather than consts so tests can shorten them. The quiet window keeps @@ -22,16 +23,18 @@ var ( // startSearchStatus prints the transient "Searching..." stderr line and keeps // it honest while the request runs: once the quiet window passes, the line -// gains elapsed time and — when the daemon reports one via /health — the -// operation the search is actually waiting on (e.g. "checking the search -// index", which can take a minute on a large archive after a daemon -// restart). The returned stop func erases the line; call it before printing -// results. +// gains elapsed time and any concurrent daemon work reported via /health. +// The returned stop func erases the line; call it before printing results. func startSearchStatus(ctx context.Context, prefix string, info HTTPStoreInfo) func() { line := &searchStatusLine{ - out: os.Stderr, - prefix: prefix, - fetchOp: daemonOperationFetcher(info.URL, httpStoreAPIKey(info)), + out: os.Stderr, + prefix: prefix, + fetchOp: daemonOperationFetcher(info.URL, httpStoreAPIKey(info, func() *config.Config { + if state := invocationFromContext(ctx); state != nil { + return state.cfg + } + return nil + }())), tty: isatty.IsTerminal(os.Stderr.Fd()) || isatty.IsCygwinTerminal(os.Stderr.Fd()), start: time.Now(), @@ -84,13 +87,13 @@ func (l *searchStatusLine) run(ctx context.Context) { func (l *searchStatusLine) render(elapsed time.Duration, op *api.OperationHealth) { if !l.tty { - // A pipe gets no in-place updates, but the one fact worth a log - // line — the daemon is busy with something expensive — is still - // printed once. + // A pipe gets no in-place updates; print one line so a slow search + // with concurrent daemon work is visible in logs. The search is not + // operation-gated, so the label is context, never the cause. if op != nil && op.Label != "" && !l.noticed { l.noticed = true - _, _ = fmt.Fprintf(l.out, "\nDaemon is busy: %s. The search will finish when it does.\n", - op.Label) + _, _ = fmt.Fprintf(l.out, "\nSearch still running after %s. The daemon is also running: %s.\n", + elapsed.Round(time.Second), op.Label) } return } @@ -105,14 +108,13 @@ func (l *searchStatusLine) clear() { _, _ = fmt.Fprintf(l.out, "\r%s\r", strings.Repeat(" ", l.width)) } -// formatSearchStatus renders one status line. The daemon operation label wins -// over a bare elapsed count because it answers the actual question ("why is -// this taking so long"); Busy without a label (unauthenticated /health -// fallback) degrades to the elapsed-only form. +// formatSearchStatus renders one status line. Daemon activity is shown as +// concurrent work, because the search does not wait on it; a busy daemon +// without a label (unauthenticated /health) degrades to elapsed time only. func formatSearchStatus(prefix string, elapsed time.Duration, op *api.OperationHealth) string { rounded := elapsed.Round(time.Second) if op != nil && op.Label != "" { - return fmt.Sprintf("%s daemon is busy: %s (%s)", prefix, op.Label, rounded) + return fmt.Sprintf("%s (%s; daemon also running: %s)", prefix, rounded, op.Label) } return fmt.Sprintf("%s (%s)", prefix, rounded) } @@ -131,12 +133,12 @@ func daemonOperationFetcher(baseURL, apiKey string) func(context.Context) *api.O // httpStoreAPIKey returns the API key for the endpoint OpenHTTPStore // selected, for auxiliary requests (health polling) beside the main client. -func httpStoreAPIKey(info HTTPStoreInfo) string { +func httpStoreAPIKey(info HTTPStoreInfo, cfg *config.Config) string { if cfg == nil { return "" } if info.Kind == HTTPStoreConfiguredRemote { - return cfg.Remote.APIKey + return cfg.Remote.AuthenticationKey() } - return cfg.Server.APIKey + return cfg.Server.AuthenticationKey() } diff --git a/cmd/msgvault/cmd/search_status_test.go b/cmd/msgvault/cmd/search_status_test.go index 741180dd0..8c997e4c1 100644 --- a/cmd/msgvault/cmd/search_status_test.go +++ b/cmd/msgvault/cmd/search_status_test.go @@ -26,10 +26,10 @@ func TestFormatSearchStatus(t *testing.T) { want: "Searching... (12s)", }, { - name: "daemon operation label wins", + name: "daemon activity is reported as concurrent, not as the cause", elapsed: 45 * time.Second, op: &api.OperationHealth{Busy: true, Label: "checking the search index"}, - want: "Searching... daemon is busy: checking the search index (45s)", + want: "Searching... (45s; daemon also running: checking the search index)", }, { name: "busy without label degrades to elapsed only", @@ -84,9 +84,11 @@ func TestSearchStatusLineRenderNonTTYPrintsNoticeOnce(t *testing.T) { op := &api.OperationHealth{Busy: true, Label: "checking the search index"} l.render(7*time.Second, op) l.render(9*time.Second, op) - notice := "Daemon is busy: checking the search index. The search will finish when it does." + notice := "Search still running after 7s. The daemon is also running: checking the search index." assert.Equal(1, strings.Count(buf.String(), notice), "the busy notice must be printed exactly once") + assert.NotContains(buf.String(), "Daemon is busy") + assert.NotContains(buf.String(), "will finish when it does") } func TestStartSearchStatusLoopShowsDaemonActivity(t *testing.T) { @@ -137,6 +139,6 @@ func TestStartSearchStatusLoopShowsDaemonActivity(t *testing.T) { require.FailNow("run loop must stop on cancel") } - assert.Contains(buf.String(), "daemon is busy: checking the search index", + assert.Contains(buf.String(), "daemon also running: checking the search index", "the loop must render the daemon's reported operation") } diff --git a/cmd/msgvault/cmd/search_table.go b/cmd/msgvault/cmd/search_table.go new file mode 100644 index 000000000..d76f78217 --- /dev/null +++ b/cmd/msgvault/cmd/search_table.go @@ -0,0 +1,123 @@ +package cmd + +import ( + "fmt" + "io" + "os" + "strings" + + "github.com/charmbracelet/x/ansi" + "github.com/charmbracelet/x/term" + "go.kenn.io/msgvault/internal/textutil" +) + +// A zero width means the writer is not a terminal and text stays complete. +func searchTableTerminalWidth(out io.Writer) int { + f, ok := out.(*os.File) + if !ok || !term.IsTerminal(f.Fd()) { + return 0 + } + width, _, err := term.GetSize(f.Fd()) + if err != nil || width <= 0 { + return 80 + } + return width +} + +func normalizeSearchTableText(text string) string { + return strings.Join(strings.Fields(textutil.SanitizeTerminal(text)), " ") +} + +// Markers share the subject allocation but remain visible when text is cut. +type searchTableCell struct { + text string + marker string +} + +func (c searchTableCell) display(width int) string { + if width <= 0 || ansi.StringWidth(c.text+c.marker) <= width { + return c.text + c.marker + } + textWidth := width - ansi.StringWidth(c.marker) + tail := "..." + if textWidth < len(tail) { + tail = "" + } + // Iterate entire graphemes, including those beginning with ASCII (keycap + // emoji). ANSI Truncate's ASCII path can disagree with StringWidth. + budget := textWidth - ansi.StringWidth(tail) + var clipped strings.Builder + used := 0 + for text := c.text; text != ""; { + cluster, _ := ansi.FirstGraphemeCluster(text, ansi.GraphemeWidth) + cells := ansi.StringWidth(cluster) + if cells > budget-used { + break + } + clipped.WriteString(cluster) + used += cells + text = text[len(cluster):] + } + clipped.WriteString(tail) + clipped.WriteString(c.marker) + return clipped.String() +} + +// All search shapes share ID, DATE, FROM and SUBJECT as their first columns. +// Fixed fields are never truncated, even when a terminal needs to soft wrap. +func writeSearchTable(out io.Writer, headers []string, rows [][]searchTableCell, width int) error { + widths := make([]int, len(headers)) + for i, header := range headers { + widths[i] = ansi.StringWidth(header) + } + for _, row := range rows { + for i, cell := range row { + widths[i] = max(widths[i], ansi.StringWidth(cell.text+cell.marker)) + } + } + if width > 0 { + fixed := 2 * (len(headers) - 1) + for i, cells := range widths { + if i != 2 && i != 3 { + fixed += cells + } + } + available := width - fixed + widths[2] = max(4, min(widths[2], 30, available-7)) + widths[3] = max(7, min(widths[3], available-widths[2])) + } + + writeRow := func(cells []string) error { + var line strings.Builder + for i, cell := range cells { + line.WriteString(cell) + if i < len(cells)-1 { + line.WriteString(strings.Repeat(" ", widths[i]-ansi.StringWidth(cell)+2)) + } + } + if _, err := fmt.Fprintln(out, line.String()); err != nil { + return fmt.Errorf("write search table row: %w", err) + } + return nil + } + if err := writeRow(headers); err != nil { + return err + } + rules := make([]string, len(headers)) + for i, header := range headers { + rules[i] = strings.Repeat("─", ansi.StringWidth(header)/ansi.StringWidth("─")) + } + if err := writeRow(rules); err != nil { + return err + } + for _, row := range rows { + cells := make([]string, len(row)) + for i, cell := range row { + cells[i] = cell.display(widths[i]) + } + if err := writeRow(cells); err != nil { + return err + } + } + return nil +} diff --git a/cmd/msgvault/cmd/search_table_test.go b/cmd/msgvault/cmd/search_table_test.go new file mode 100644 index 000000000..cfba0ff48 --- /dev/null +++ b/cmd/msgvault/cmd/search_table_test.go @@ -0,0 +1,266 @@ +package cmd + +import ( + "bytes" + "fmt" + "io" + "math" + "os" + "os/exec" + "strconv" + "strings" + "testing" + "time" + "unicode/utf8" + + "github.com/charmbracelet/x/ansi" + "github.com/rivo/uniseg" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/daemonclient" + "go.kenn.io/msgvault/internal/query" + "go.kenn.io/msgvault/internal/textutil" +) + +func TestSearchTableFitsDisplayWidthAndAlignsColumns(t *testing.T) { + rows := []query.MessageSummary{ + {ID: 1, SentAt: time.Date(2026, 9, 27, 0, 0, 0, 0, time.UTC), FromName: "F" + strings.Repeat("sender", 10), Subject: "S" + strings.Repeat("subject ", 40), SizeEstimate: 1024}, + {ID: 2027, SentAt: time.Date(2026, 9, 26, 0, 0, 0, 0, time.UTC), FromName: "F" + strings.Repeat("界e\u0301👩‍💻", 12), Subject: "S" + strings.Repeat("界e\u0301👩‍💻 ", 40), SizeEstimate: 100 * 1024 * 1024}, + } + for _, width := range []int{40, 80, 120, 160, 200, 500} { + t.Run(strconv.Itoa(width), func(t *testing.T) { + assert := assert.New(t) + var out bytes.Buffer + require.NoError(t, writeSearchResultsTableWidth(&out, rows, width)) + lines := strings.Split(out.String(), "\n") + for _, line := range lines[:4] { + assert.LessOrEqual(uniseg.StringWidth(line), width, "%q", line) + assert.True(utf8.ValidString(line)) + } + assert.Equal([]string{"ID", "DATE", "FROM", "SUBJECT", "SIZE"}, strings.Fields(lines[0])) + for i, size := range []string{"1.0K", "100.0M"} { + line := lines[i+2] + assert.Contains(line, size) + assertTableColumn(t, lines[0], "DATE", line, rows[i].SentAt.Format("2006-01-02")) + assertTableColumn(t, lines[0], "FROM", line, "F") + assertTableColumn(t, lines[0], "SUBJECT", line, "S") + assertTableColumn(t, lines[0], "SIZE", line, size) + } + assert.Contains(lines[3], "2027") + if width == 500 { + assert.Contains(lines[2], rows[0].Subject) + assert.Contains(lines[3], rows[1].Subject) + } + }) + } +} + +func assertTableColumn(t *testing.T, header, label, row, value string) { + t.Helper() + headerIndex := strings.Index(header, label) + rowIndex := strings.Index(row, value) + require.NotEqual(t, -1, headerIndex) + require.NotEqual(t, -1, rowIndex) + assert.Equal(t, uniseg.StringWidth(header[:headerIndex]), uniseg.StringWidth(row[:rowIndex]), label) +} + +func TestSearchTableShortSenderLeavesRoomForSubject(t *testing.T) { + subject := strings.Repeat("complete subject ", 5) + var out bytes.Buffer + require.NoError(t, writeSearchResultsTableWidth(&out, []query.MessageSummary{{ID: 1, FromName: "F", Subject: subject}}, 120)) + assert.Contains(t, out.String(), strings.TrimSpace(subject)) +} + +func TestSearchTableEastAsianWidthSetting(t *testing.T) { + // ANSI reads this setting at package initialization. A child of the real + // test binary exercises that startup boundary and the production renderer. + executable, err := os.Executable() + require.NoError(t, err) + cmd := exec.Command(executable, "-test.run=^TestSearchTableShortSenderLeavesRoomForSubject$") + cmd.Env = append(os.Environ(), "RUNEWIDTH_EASTASIAN=1") + output, err := cmd.CombinedOutput() + require.NoError(t, err, "%s", output) +} + +func TestSearchTableTooNarrowPreservesFixedFields(t *testing.T) { + assert := assert.New(t) + var out bytes.Buffer + require.NoError(t, writeSearchResultsTableWidth(&out, []query.MessageSummary{{ID: 1234567890123, FromName: "界👩‍💻", Subject: strings.Repeat("界", 50), SizeEstimate: 100 * 1024 * 1024}}, 20)) + line := strings.Split(out.String(), "\n")[2] + assert.True(utf8.ValidString(line)) + assert.Contains(line, "1234567890123") + assert.Contains(line, "0001-01-01") + assert.Contains(line, "100.0M") + assert.Greater(uniseg.StringWidth(line), 20) +} + +func TestSearchTableHybridScoreWidthsAndBoostBudget(t *testing.T) { + rrf, bm25, vec := 0.5, -1234567890123.125, 987654.25 + rows := []daemonclient.CLIHybridSearchResult{ + {ID: 123456789, SentAt: time.Date(2026, 9, 27, 0, 0, 0, 0, time.UTC), FromEmail: "F" + strings.Repeat("sender", 10), Subject: "S" + strings.Repeat("界e\u0301👩‍💻", 40), SubjectBoosted: true, RRFScore: &rrf, BM25Score: &bm25, VectorScore: &vec}, + {ID: 2, SentAt: time.Date(2026, 9, 26, 0, 0, 0, 0, time.UTC), Message: query.MessageSummary{FromName: "F界", Snippet: "Short"}, SubjectBoosted: true}, + } + for _, explain := range []bool{false, true} { + for _, width := range []int{80, 120, 160, 200} { + t.Run(fmt.Sprintf("%t/%d", explain, width), func(t *testing.T) { + assert := assert.New(t) + var out bytes.Buffer + require.NoError(t, writeHybridResultsTableWidth(&out, rows, explain, width)) + lines := strings.Split(out.String(), "\n") + for _, line := range lines[:4] { + assert.True(utf8.ValidString(line)) + assert.LessOrEqual(uniseg.StringWidth(line), width, "%q", line) + } + for _, line := range lines[2:4] { + assert.Contains(line, " *") + assertTableColumn(t, lines[0], "FROM", line, "F") + assertTableColumn(t, lines[0], "SUBJECT", line, "S") + } + assert.Contains(lines[2], "123456789") + if explain { + for i, score := range []string{"0.5000", "-1234567890123.1250", "987654.2500"} { + assertTableColumn(t, lines[0], []string{"RRF", "BM25", "VEC"}[i], lines[2], score) + } + } + }) + } + } +} + +func TestSearchTableKeycapGraphemesFitSenderAndSubject(t *testing.T) { + for _, tt := range []struct { + name string + row query.MessageSummary + width int + }{ + {"sender", query.MessageSummary{ID: 1, FromName: strings.Repeat("1️⃣", 4)}, 37}, + {"subject", query.MessageSummary{ID: 1, FromName: "F", Subject: "a" + strings.Repeat("1️⃣", 4)}, 35}, + } { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + var out bytes.Buffer + if !assert.NotPanics(func() { + require.NoError(t, writeSearchResultsTableWidth(&out, []query.MessageSummary{tt.row}, tt.width)) + }) { + return + } + lines := strings.Split(out.String(), "\n") + for _, line := range lines[:3] { + assert.LessOrEqual(ansi.StringWidth(line), tt.width, "%q", line) + } + assert.NotContains(lines[2], "1️⃣1️⃣1️⃣...") + }) + } + t.Run("hybrid subject", func(t *testing.T) { + var out bytes.Buffer + require.NoError(t, writeHybridResultsTableWidth(&out, []daemonclient.CLIHybridSearchResult{{ID: 1, FromEmail: "F", Subject: "a" + strings.Repeat("1️⃣", 6), SubjectBoosted: true}}, false, 33)) + line := strings.Split(out.String(), "\n")[2] + assert.LessOrEqual(t, ansi.StringWidth(line), 33) + assert.Contains(t, line, " *") + }) +} + +type searchTableFailAfterWriter struct{ remaining int } + +func (w *searchTableFailAfterWriter) Write(p []byte) (int, error) { + if w.remaining == 0 { + return 0, io.ErrClosedPipe + } + w.remaining-- + return len(p), nil +} + +func TestSearchTablePropagatesEveryWriteError(t *testing.T) { + for _, writes := range []int{0, 1, 2, 3} { + err := writeSearchResultsTableWidth(&searchTableFailAfterWriter{remaining: writes}, []query.MessageSummary{{ID: 1}}, 80) + require.ErrorIs(t, err, io.ErrClosedPipe) + } + for _, writes := range []int{0, 1, 2} { + err := writeHybridResultsTableWidth(&searchTableFailAfterWriter{remaining: writes}, []daemonclient.CLIHybridSearchResult{{ID: 1}}, true, 80) + require.ErrorIs(t, err, io.ErrClosedPipe) + } +} + +func FuzzSearchTableFullText(f *testing.F) { + for _, text := range []string{ + strings.Repeat("long complete text ", 20), + strings.Repeat("界", 80), + strings.Repeat("e\u0301", 80), + strings.Repeat("👩‍💻", 80), + "\x1b[31mhello\n\tworld\x1b[0m\u009b31m", + "\xff\u0301\u200d\u200b\ufe0f", + } { + f.Add(text) + } + f.Fuzz(func(t *testing.T, text string) { + assert := assert.New(t) + require := require.New(t) + // The sanitizer owns control removal. The renderer must preserve its + // complete normalized output, independent of the search shape. + from := strings.Join(strings.Fields(textutil.SanitizeTerminal("F:"+text)), " ") + subject := strings.Join(strings.Fields(textutil.SanitizeTerminal("S:"+text)), " ") + message := query.MessageSummary{ID: 1, FromName: "F:" + text, Snippet: "S:" + text} + var out bytes.Buffer + require.NoError(writeSearchResultsTableWidth(&out, []query.MessageSummary{message}, 0)) + assert.Contains(out.String(), from) + assert.Contains(out.String(), subject) + for _, explain := range []bool{false, true} { + out.Reset() + require.NoError(writeHybridResultsTableWidth(&out, []daemonclient.CLIHybridSearchResult{{ID: 1, Message: message, SubjectBoosted: true}}, explain, 0)) + assert.Contains(out.String(), from) + assert.Contains(out.String(), subject+" *") + } + }) +} + +func FuzzSearchTableBoundedText(f *testing.F) { + for _, text := range []string{ + strings.Repeat("long text ", 20), strings.Repeat("界", 80), + strings.Repeat("e\u0301", 80), strings.Repeat("👩‍💻", 80), + "\xff\x1b[31m\u0301\u200d\u200b\ufe0f", "👍🏽🇺🇳1️⃣界", + } { + f.Add(text, uint64(7), true) + f.Add(text, uint64(80), false) + f.Add(text, uint64(496), false) + } + f.Add(strings.Repeat("1️⃣", 4), uint64(2), false) + f.Add("a"+strings.Repeat("1️⃣", 4), uint64(3), false) + f.Add(strings.Repeat("👩‍💻", 80), uint64(math.MaxUint64), true) + f.Fuzz(func(t *testing.T, text string, extra uint64, boosted bool) { + assert := assert.New(t) + require := require.New(t) + text = strings.Join(strings.Fields(textutil.SanitizeTerminal(text)), " ") + // Saturate only at the machine integer boundary; keep the full generated + // width domain without allocating a buffer proportional to the budget. + width := 4 + int(min(extra, uint64(math.MaxInt-7))) + marker := "" + if boosted { + width += 3 // Subject minimum is seven cells, including the marker. + marker = " *" + } + got := (searchTableCell{text: text, marker: marker}).display(width) + require.True(utf8.ValidString(got)) + assert.LessOrEqual(uniseg.StringWidth(got), width, "%q", got) + assert.LessOrEqual(ansi.StringWidth(got), width, "%q", got) + require.True(strings.HasSuffix(got, marker)) + if ansi.StringWidth(text+marker) <= width { + assert.Equal(text+marker, got) + return + } + require.True(strings.HasSuffix(strings.TrimSuffix(got, marker), "...")) + // A truncated value must be a prefix ending at a whole grapheme, + // followed by the ellipsis and the complete boost marker. + prefix := strings.TrimSuffix(strings.TrimSuffix(got, marker), "...") + require.True(strings.HasPrefix(text, prefix), "%q is not a prefix of %q", prefix, text) + clusters := uniseg.NewGraphemes(text) + boundary := len(prefix) == 0 + for clusters.Next() { + _, end := clusters.Positions() + if end == len(prefix) { + boundary = true + break + } + } + assert.True(boundary, "truncated inside a grapheme: %q -> %q", text, got) + }) +} diff --git a/cmd/msgvault/cmd/search_test.go b/cmd/msgvault/cmd/search_test.go index 32ce14969..7462c5163 100644 --- a/cmd/msgvault/cmd/search_test.go +++ b/cmd/msgvault/cmd/search_test.go @@ -1,6 +1,7 @@ package cmd import ( + "bytes" "database/sql" "encoding/json" "fmt" @@ -8,6 +9,7 @@ import ( "net/http" "net/http/httptest" "os" + "strings" "sync/atomic" "testing" "time" @@ -22,6 +24,81 @@ import ( "go.kenn.io/msgvault/internal/store" ) +func TestSummaryTableText(t *testing.T) { + tests := []struct { + name, subject, snippet, want string + }{ + {"subject wins", "Quarterly review", "When: tomorrow", "Quarterly review"}, + {"chat uses snippet", "", "are we still on for Friday", "are we still on for Friday"}, + {"blank subject uses snippet", " ", "see attached", "see attached"}, + {"escape-only subject uses snippet", "\x1b[31m", "line one\n\tline two", "line one line two"}, + {"control-only subject uses snippet", "\x00\x07", "see attached", "see attached"}, + {"whitespace collapses", "", "line one\n\tline two ", "line one line two"}, + {"terminal controls removed", "", "hi\x1b]0;bad title\a there\x1b[31m!", "hi there!"}, + {"empty", "", "", ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, summaryTableText(tt.subject, tt.snippet)) + }) + } +} + +func TestFormatSummarySize(t *testing.T) { + assert.Equal(t, "-", formatSummarySize(0)) + assert.Equal(t, "-", formatSummarySize(-1)) + assert.Equal(t, "1.0K", formatSummarySize(1043)) +} + +func TestWriteSearchResultsTableShowsChatSnippetAndUnknownSize(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + results := []query.MessageSummary{ + {ID: 101, SentAt: time.Date(2026, 11, 5, 17, 0, 0, 0, time.UTC), FromEmail: "alice@example.com", Subject: "Concert tickets", Snippet: "When: Nov 5", SizeEstimate: 1043}, + {ID: 102, SentAt: time.Date(2026, 8, 11, 15, 46, 5, 0, time.UTC), FromName: "Bob\x1b[31m Example", Snippet: "this is the\nband", HasAttachments: true, AttachmentCount: 1}, + } + var buf bytes.Buffer + require.NoError(writeSearchResultsTable(&buf, results)) + lines := strings.Split(buf.String(), "\n") + require.GreaterOrEqual(len(lines), 4) + assert.Contains(lines[2], "Concert tickets") + assert.Contains(lines[2], "1.0K") + assert.Contains(lines[3], "this is the band") + assert.Contains(lines[3], "Bob Example") + assert.NotContains(lines[3], "0B") + assert.NotContains(buf.String(), "\x1b") + assert.True(strings.HasSuffix(strings.TrimRight(lines[3], " "), " -")) + assert.Contains(buf.String(), "Showing 2 results") +} + +// These entry points write to a real pipe, so terminal-only limits must not apply. +func TestSearchTableFullTextInPipe(t *testing.T) { + assert := assert.New(t) + from := strings.Repeat("sender", 10) + "@example.com" + subject := strings.Repeat("long subject ", 10) + "final words" + snippet := strings.Repeat("chat snippet ", 10) + "last words" + done := captureStdout(t) + err := outputSearchResultsTable([]query.MessageSummary{ + {ID: 1, FromEmail: from, Subject: "\x1b[31m" + subject + "\n"}, + {ID: 2, FromName: from, Snippet: "\t" + snippet + "\x1b[0m"}, + }) + output := done() + require.NoError(t, err) + assert.Contains(output, from) + assert.Contains(output, subject) + assert.Contains(output, snippet) + assert.NotContains(output, "\x1b") +} + +type failingSearchWriter struct{} + +func (failingSearchWriter) Write([]byte) (int, error) { return 0, io.ErrClosedPipe } + +func TestWriteSearchResultsTableReturnsWriterError(t *testing.T) { + err := writeSearchResultsTable(failingSearchWriter{}, []query.MessageSummary{{ID: 1}}) + require.ErrorIs(t, err, io.ErrClosedPipe) +} + // captureStdout redirects os.Stdout to a pipe and returns a function // that restores the original stdout and returns captured output. // The pipe is drained concurrently to avoid deadlock if the command @@ -137,6 +214,9 @@ func TestSummaryFromDisplayFallsBackForPhoneMessages(t *testing.T) { } func TestSearchCmd_AccountFlagForwardsToRemoteHTTP(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) savedCfg := cfg @@ -159,11 +239,14 @@ func TestSearchCmd_AccountFlagForwardsToRemoteHTTP(t *testing.T) { defer srv.Close() cfg = &config.Config{} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Remote.URL = srv.URL cfg.Remote.AllowInsecure = true useLocal = false root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(searchCmd) root.SetArgs([]string{"search", "--account", "alice@example.com", "hello"}) @@ -173,6 +256,9 @@ func TestSearchCmd_AccountFlagForwardsToRemoteHTTP(t *testing.T) { } func TestSearchCmd_MessageTypeFlagForwardsToRemoteMode(t *testing.T) { + cfg := testConfigValue() + useLocal := false + savedCfg := cfg savedUseLocal := useLocal defer func() { @@ -195,11 +281,14 @@ func TestSearchCmd_MessageTypeFlagForwardsToRemoteMode(t *testing.T) { defer srv.Close() cfg = &config.Config{} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Remote.URL = srv.URL cfg.Remote.AllowInsecure = true useLocal = false root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(searchCmd) root.SetArgs([]string{"search", "--message-type", "sms", "lunch"}) @@ -209,6 +298,9 @@ func TestSearchCmd_MessageTypeFlagForwardsToRemoteMode(t *testing.T) { } func TestSearchCmd_DeletionScopeForwardsToRemoteFTS(t *testing.T) { + cfg := testConfigValue() + useLocal := false + savedCfg := cfg savedUseLocal := useLocal defer func() { @@ -232,11 +324,14 @@ func TestSearchCmd_DeletionScopeForwardsToRemoteFTS(t *testing.T) { defer srv.Close() cfg = &config.Config{} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Remote.URL = srv.URL cfg.Remote.AllowInsecure = true useLocal = false root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(searchCmd) root.SetArgs([]string{"search", "--deletion-scope", "deleted", "statement"}) @@ -278,6 +373,9 @@ func TestSearchCmd_DeletionScopeRejectsVectorAndHybrid(t *testing.T) { } func TestSearchCmd_FTSUsesLocalDaemonHTTPAndPreservesJSONOutput(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) dataDir := t.TempDir() @@ -297,10 +395,12 @@ func TestSearchCmd_FTSUsesLocalDaemonHTTPAndPreservesJSONOutput(t *testing.T) { Data: config.DataConfig{DataDir: dataDir}, Remote: config.RemoteConfig{URL: "http://configured-daemonclient.invalid"}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{useLocal: true}) useLocal = true done := captureStdout(t) root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(searchCmd) root.SetArgs([]string{"search", "--json", "lunch"}) @@ -314,6 +414,9 @@ func TestSearchCmd_FTSUsesLocalDaemonHTTPAndPreservesJSONOutput(t *testing.T) { } func TestSearchCmd_FTSCollectionSearchUsesDaemonHTTPAndPreservesBanner(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) dataDir := t.TempDir() @@ -333,11 +436,15 @@ func TestSearchCmd_FTSCollectionSearchUsesDaemonHTTPAndPreservesBanner(t *testin Data: config.DataConfig{DataDir: dataDir}, Remote: config.RemoteConfig{URL: "http://configured-daemonclient.invalid"}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true doneOut := captureStdout(t) doneErr := captureStderr(t) root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(searchCmd) root.SetArgs([]string{"search", "--collection", "Important", "--json"}) @@ -412,26 +519,26 @@ func searchHTTPDaemon(t *testing.T) (*httptest.Server, *atomic.Int32) { return server, searchRequests } -// TestSearchCmd_PrintsBackgroundIndexNote verifies the CLI caveats results -// whenever the daemon reports the FTS index is not yet known complete: a -// rebuild in progress (index_state="building") and an unfinished completeness -// probe (index_state="checking") get distinct notes; a complete index gets -// none. +// TestSearchCmd_PrintsBackgroundIndexNote verifies only known index gaps +// produce a caveat. An unfinished completeness probe alone stays silent. func TestSearchCmd_PrintsBackgroundIndexNote(t *testing.T) { + cfg := testConfigValue() + useLocal := false + tests := []struct { name string indexState string wantNote string }{ { - name: "building warns about the rebuild", + name: "building warns about rebuilding or awaiting rebuild", indexState: "building", - wantNote: "the search index is being rebuilt in the background; results may be incomplete", + wantNote: "the search index is rebuilding or awaiting a rebuild in the background; results may be incomplete", }, { - name: "checking warns the probe has not finished", + name: "checking alone prints no note", indexState: "checking", - wantNote: "search index completeness is still being verified in the background; results may be incomplete", + wantNote: "", }, { name: "complete index prints no note", @@ -478,11 +585,14 @@ func TestSearchCmd_PrintsBackgroundIndexNote(t *testing.T) { HomeDir: dataDir, Data: config.DataConfig{DataDir: dataDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + invocationFromContext(testCtx).options.useLocal = true useLocal = true doneOut := captureStdout(t) doneErr := captureStderr(t) root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(searchCmd) root.SetArgs([]string{"search", "lunch"}) @@ -493,7 +603,7 @@ func TestSearchCmd_PrintsBackgroundIndexNote(t *testing.T) { assert.Contains(out, "Lunch", "results still print") if tt.wantNote == "" { - assert.NotContains(errOut, "Note:", "no index note for a complete index") + assert.NotContains(errOut, "Note:", "no index note without a known gap") } else { assert.Contains(errOut, tt.wantNote, "index state note") } @@ -502,6 +612,9 @@ func TestSearchCmd_PrintsBackgroundIndexNote(t *testing.T) { } func TestSearchCmd_AccountFlagWithoutQuery(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) tmpDir := t.TempDir() @@ -550,12 +663,16 @@ func TestSearchCmd_AccountFlagWithoutQuery(t *testing.T) { Data: config.DataConfig{DataDir: tmpDir}, Remote: config.RemoteConfig{URL: "http://configured-daemonclient.invalid"}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true // Search with --account only (no query terms) — must succeed. done := captureStdout(t) root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(searchCmd) root.SetArgs([]string{ "search", "--account", "alice@example.com", "--json", @@ -570,6 +687,9 @@ func TestSearchCmd_AccountFlagWithoutQuery(t *testing.T) { } func TestSearchCmd_MessageTypeFlagScopesResults(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) tmpDir := t.TempDir() @@ -614,10 +734,14 @@ func TestSearchCmd_MessageTypeFlagScopesResults(t *testing.T) { Data: config.DataConfig{DataDir: tmpDir}, Remote: config.RemoteConfig{URL: "http://configured-daemonclient.invalid"}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true done := captureStdout(t) root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(searchCmd) root.SetArgs([]string{ "search", "--message-type", "calendar_event", "--json", @@ -630,6 +754,8 @@ func TestSearchCmd_MessageTypeFlagScopesResults(t *testing.T) { } func TestSearchCmd_InvalidQueryFailsFastWithoutDB(t *testing.T) { + cfg := testConfigValue() + savedCfg := cfg defer func() { cfg = savedCfg; resetSearchFlags() }() @@ -639,8 +765,11 @@ func TestSearchCmd_InvalidQueryFailsFastWithoutDB(t *testing.T) { HomeDir: "/nonexistent", Data: config.DataConfig{DataDir: "/nonexistent"}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(searchCmd) root.SetArgs([]string{"search", "before:not-a-date"}) @@ -654,6 +783,9 @@ func TestSearchCmd_InvalidQueryFailsFastWithoutDB(t *testing.T) { } func TestSearchCmd_AccountFlagDoesNotLeakAcrossInvocations(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) tmpDir := t.TempDir() dbPath := tmpDir + "/msgvault.db" @@ -693,11 +825,15 @@ func TestSearchCmd_AccountFlagDoesNotLeakAcrossInvocations(t *testing.T) { Data: config.DataConfig{DataDir: tmpDir}, Remote: config.RemoteConfig{URL: "http://configured-daemonclient.invalid"}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true // First invocation: search with --account. done := captureStdout(t) root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(searchCmd) root.SetArgs([]string{ "search", "--account", "alice@example.com", "--json", @@ -710,7 +846,9 @@ func TestSearchCmd_AccountFlagDoesNotLeakAcrossInvocations(t *testing.T) { // Must not carry over the previous account filter. resetSearchFlags() done = captureStdout(t) + testCtx2 := testInvocationContext(t.Context(), cfg, invocationOptions{useLocal: true}) root2 := newTestRootCmd() + root2.SetContext(testCtx2) root2.AddCommand(searchCmd) root2.SetArgs([]string{ "search", "--account", "", "--json", "test msg", @@ -723,12 +861,17 @@ func TestSearchCmd_AccountFlagDoesNotLeakAcrossInvocations(t *testing.T) { } func TestSearchCmd_NoQueryNoAccount(t *testing.T) { + cfg := testConfigValue() + savedCfg := cfg defer func() { cfg = savedCfg; resetSearchFlags() }() cfg = &config.Config{} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(searchCmd) root.SetArgs([]string{"search"}) @@ -741,6 +884,9 @@ func TestSearchCmd_NoQueryNoAccount(t *testing.T) { // collection containing only the first, then runs FTS search with // --collection. Only the first account's message must come back. func TestSearchCmd_CollectionFlagScopesResults(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) tmpDir := t.TempDir() @@ -789,10 +935,14 @@ func TestSearchCmd_CollectionFlagScopesResults(t *testing.T) { Data: config.DataConfig{DataDir: tmpDir}, Remote: config.RemoteConfig{URL: "http://configured-daemonclient.invalid"}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true done := captureStdout(t) root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(searchCmd) root.SetArgs([]string{ "search", "--collection", "alice-only", "--json", @@ -807,6 +957,9 @@ func TestSearchCmd_CollectionFlagScopesResults(t *testing.T) { // TestSearchCmd_CollectionFlagUnknown returns a clear error when the // named collection does not exist. func TestSearchCmd_CollectionFlagUnknown(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) tmpDir := t.TempDir() dbPath := tmpDir + "/msgvault.db" @@ -828,9 +981,13 @@ func TestSearchCmd_CollectionFlagUnknown(t *testing.T) { Data: config.DataConfig{DataDir: tmpDir}, Remote: config.RemoteConfig{URL: "http://configured-daemonclient.invalid"}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(searchCmd) root.SetArgs([]string{ "search", "--collection", "does-not-exist", "anything", @@ -845,6 +1002,8 @@ func TestSearchCmd_CollectionFlagUnknown(t *testing.T) { // FTS allows queryless scoped searches; vector/hybrid don't, because // the embeddings client needs text to vectorize. func TestSearchCmd_VectorOrHybridRequireQueryText(t *testing.T) { + cfg := testConfigValue() + for _, mode := range []string{"vector", "hybrid"} { t.Run(mode, func(t *testing.T) { savedCfg := cfg @@ -871,6 +1030,8 @@ func TestSearchCmd_VectorOrHybridRequireQueryText(t *testing.T) { // like `from:alice` would fail at the engine layer; reject it at the // CLI surface instead. func TestSearchCmd_VectorOrHybridRejectFilterOnlyQuery(t *testing.T) { + cfg := testConfigValue() + for _, mode := range []string{"vector", "hybrid"} { t.Run(mode, func(t *testing.T) { savedCfg := cfg @@ -930,6 +1091,9 @@ func TestOutputSearchResultsJSONShowsDeletedFromSourceOnlyWhenPresent(t *testing // array, never the "No messages found." prose — agents pipe this // straight into jq. func TestSearchCmd_JSONEmptyResultsEmitEmptyArray(t *testing.T) { + cfg := testConfigValue() + useLocal := false + assert := assert.New(t) require := require.New(t) savedCfg := cfg @@ -947,11 +1111,14 @@ func TestSearchCmd_JSONEmptyResultsEmitEmptyArray(t *testing.T) { defer srv.Close() cfg = &config.Config{} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Remote.URL = srv.URL cfg.Remote.AllowInsecure = true useLocal = false root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(searchCmd) root.SetArgs([]string{"search", "--json", "nothing-matches"}) diff --git a/cmd/msgvault/cmd/search_vector.go b/cmd/msgvault/cmd/search_vector.go index 1c783b944..f2a50cd4b 100644 --- a/cmd/msgvault/cmd/search_vector.go +++ b/cmd/msgvault/cmd/search_vector.go @@ -1,10 +1,13 @@ package cmd import ( + "errors" "fmt" + "io" "math" "os" - "text/tabwriter" + "strconv" + "strings" "time" "github.com/spf13/cobra" @@ -15,6 +18,11 @@ import ( // remote server or local daemon. It preserves the historical CLI renderer while // keeping vector backend ownership inside the daemon. func runHybridSearch(cmd *cobra.Command, queryStr, mode string, explain bool) error { + state := invocationFromCommand(cmd) + if state == nil || state.logger == nil { + return errors.New("invocation state is unavailable") + } + logger := state.logger s, _, err := OpenHTTPStore(cmd.Context()) if err != nil { return fmt.Errorf("open store: %w", err) @@ -79,41 +87,65 @@ func outputHybridResultsTable(resp *daemonclient.CLIHybridSearch, explain bool) fmt.Println("No messages found.") fmt.Printf("\nGeneration #%d (%s, fingerprint=%q)\n", resp.Generation.ID, resp.Generation.State, resp.Generation.Fingerprint) + outputHybridTimings(resp, explain) return nil } - w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + if err := writeHybridResultsTable(os.Stdout, resp.Results, explain); err != nil { + return err + } + fmt.Printf("\n%s (generation #%d %s, fingerprint=%q)\n", + formatShowingResults(len(resp.Results)), resp.Generation.ID, resp.Generation.State, resp.Generation.Fingerprint) + outputHybridTimings(resp, explain) + return nil +} + +func writeHybridResultsTable(out io.Writer, results []daemonclient.CLIHybridSearchResult, explain bool) error { + return writeHybridResultsTableWidth(out, results, explain, searchTableTerminalWidth(out)) +} + +func writeHybridResultsTableWidth(out io.Writer, results []daemonclient.CLIHybridSearchResult, explain bool, width int) error { + headers := []string{"ID", "DATE", "FROM", "SUBJECT"} if explain { - _, _ = fmt.Fprintln(w, "ID\tDATE\tFROM\tSUBJECT\tRRF\tBM25\tVEC") - _, _ = fmt.Fprintln(w, "──\t────\t────\t───────\t───\t────\t───") - } else { - _, _ = fmt.Fprintln(w, "ID\tDATE\tFROM\tSUBJECT") - _, _ = fmt.Fprintln(w, "──\t────\t────\t───────") + headers = append(headers, "RRF", "BM25", "VEC") } - for _, r := range resp.Results { - date := r.SentAt.Format("2006-01-02") - from := truncate(r.FromEmail, 30) - subject := truncate(r.Subject, 50) + rows := make([][]searchTableCell, 0, len(results)) + for _, r := range results { + from := r.FromEmail + if strings.TrimSpace(from) == "" { + from = summaryFromDisplay(r.Message) + } + subject := searchTableCell{text: summaryTableText(r.Subject, r.Message.Snippet)} if r.SubjectBoosted { - subject += " *" + subject.marker = " *" + } + row := []searchTableCell{ + {text: strconv.FormatInt(r.ID, 10)}, + {text: r.SentAt.Format("2006-01-02")}, + {text: normalizeSearchTableText(from)}, + subject, } if explain { - _, _ = fmt.Fprintf(w, "%d\t%s\t%s\t%s\t%s\t%s\t%s\n", - r.ID, date, from, subject, - formatOptionalScorePtr(r.RRFScore), - formatOptionalScorePtr(r.BM25Score), - formatOptionalScorePtr(r.VectorScore)) - } else { - _, _ = fmt.Fprintf(w, "%d\t%s\t%s\t%s\n", - r.ID, date, from, subject) + row = append(row, + searchTableCell{text: formatOptionalScorePtr(r.RRFScore)}, + searchTableCell{text: formatOptionalScorePtr(r.BM25Score)}, + searchTableCell{text: formatOptionalScorePtr(r.VectorScore)}, + ) } + rows = append(rows, row) } - if err := w.Flush(); err != nil { - return fmt.Errorf("flush table output: %w", err) + return writeSearchTable(out, headers, rows, width) +} + +func outputHybridTimings(resp *daemonclient.CLIHybridSearch, explain bool) { + if !explain { + return } - fmt.Printf("\n%s (generation #%d %s, fingerprint=%q)\n", - formatShowingResults(len(resp.Results)), resp.Generation.ID, resp.Generation.State, resp.Generation.Fingerprint) - return nil + if resp.Accelerator != "" { + fmt.Printf("Accelerator: %s\n", resp.Accelerator) + } + fmt.Printf("Timings: total=%dms query_embedding=%dms retrieval=%dms hydration=%dms\n", + resp.TookMS, resp.Timings.QueryEmbeddingMS, resp.Timings.RetrievalMS, resp.Timings.HydrationMS) } func outputHybridResultsJSON(resp *daemonclient.CLIHybridSearch, explain bool) error { @@ -140,7 +172,7 @@ func outputHybridResultsJSON(resp *daemonclient.CLIHybridSearch, explain bool) e } rows[i] = row } - return printJSON(map[string]any{ + output := map[string]any{ "generation": map[string]any{ "id": resp.Generation.ID, "model": resp.Generation.Model, @@ -150,8 +182,14 @@ func outputHybridResultsJSON(resp *daemonclient.CLIHybridSearch, explain bool) e }, "pool_saturated": resp.PoolSaturated, "returned_count": resp.ReturnedCount, + "took_ms": resp.TookMS, + "timings": resp.Timings, "results": rows, - }) + } + if resp.Accelerator != "" { + output["accelerator"] = resp.Accelerator + } + return printJSON(output) } func formatOptionalScorePtr(v *float64) string { diff --git a/cmd/msgvault/cmd/search_vector_test.go b/cmd/msgvault/cmd/search_vector_test.go index dde1ea78a..5da4a3966 100644 --- a/cmd/msgvault/cmd/search_vector_test.go +++ b/cmd/msgvault/cmd/search_vector_test.go @@ -3,18 +3,111 @@ package cmd import ( + "bytes" + "context" "encoding/json" "net/http" "net/http/httptest" + "strconv" + "strings" "sync/atomic" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.kenn.io/kit/daemon" "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/daemonclient" + "go.kenn.io/msgvault/internal/query" ) +func TestWriteHybridResultsTableFallsBackToMessageSnippet(t *testing.T) { + sentAt := time.Date(2026, 8, 18, 14, 47, 41, 0, time.UTC) + tests := []struct { + name string + result daemonclient.CLIHybridSearchResult + contains []string + }{ + { + name: "chat hit uses sanitized snippet and display name", + result: daemonclient.CLIHybridSearchResult{ + ID: 201, SentAt: sentAt, + Message: query.MessageSummary{ID: 201, FromName: "Carol\x1b[31m Example", Snippet: "how many\n\x1b]0;bad\a songs are unreleased"}, + }, + contains: []string{"Carol Example", "how many songs are unreleased"}, + }, + { + name: "subject wins and boost marker remains", + result: daemonclient.CLIHybridSearchResult{ + ID: 202, SentAt: sentAt, FromEmail: "dave@example.com", Subject: "Tour dates", SubjectBoosted: true, + Message: query.MessageSummary{Snippet: "ignored"}, + }, + contains: []string{"dave@example.com", "Tour dates *"}, + }, + { + name: "zero Message keeps top-level fields", + result: daemonclient.CLIHybridSearchResult{ + ID: 203, SentAt: sentAt, FromEmail: "erin@example.com", Subject: "Hello", + }, + contains: []string{"erin@example.com", "Hello"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, writeHybridResultsTable(&buf, []daemonclient.CLIHybridSearchResult{tt.result}, false)) + lines := strings.Split(buf.String(), "\n") + require.GreaterOrEqual(t, len(lines), 3) + for _, want := range tt.contains { + assert.Contains(t, lines[2], want) + } + assert.NotContains(t, buf.String(), "\x1b") + }) + } +} + +func TestWriteHybridResultsTableExplainKeepsScores(t *testing.T) { + assert := assert.New(t) + rrf, bm25, vec := 0.5, 1.25, 0.75 + result := daemonclient.CLIHybridSearchResult{ + ID: 204, SentAt: time.Date(2026, 8, 18, 0, 0, 0, 0, time.UTC), + FromEmail: "frank@example.com", Subject: "Album", SubjectBoosted: true, + RRFScore: &rrf, BM25Score: &bm25, VectorScore: &vec, + } + var buf bytes.Buffer + require.NoError(t, writeHybridResultsTable(&buf, []daemonclient.CLIHybridSearchResult{result}, true)) + assert.Contains(buf.String(), "ID") + assert.Contains(buf.String(), "RRF") + assert.Contains(buf.String(), "BM25") + assert.Contains(buf.String(), "VEC") + assert.Contains(buf.String(), "Album *") + assert.Contains(buf.String(), "0.5000") + assert.Contains(buf.String(), "1.2500") + assert.Contains(buf.String(), "0.7500") +} + +func TestSearchHybridFullTextInPipe(t *testing.T) { + from := strings.Repeat("sender", 10) + "@example.com" + text := strings.Repeat("complete text ", 10) + "final words" + for _, explain := range []bool{false, true} { + t.Run(strconv.FormatBool(explain), func(t *testing.T) { + assert := assert.New(t) + done := captureStdout(t) + err := outputHybridResultsTable(&daemonclient.CLIHybridSearch{Results: []daemonclient.CLIHybridSearchResult{ + {ID: 1, FromEmail: from, Subject: "\x1b[31m" + text, SubjectBoosted: true}, + {ID: 2, Message: query.MessageSummary{FromName: from, Snippet: text + "\n"}}, + }}, explain) + output := done() + require.NoError(t, err) + assert.Contains(output, from) + assert.Contains(output, text+" *") + assert.NotContains(output, "\x1b") + assert.Equal(2, strings.Count(output, text)) + }) + } +} + func TestSearchCmd_VectorModeUsesLocalDaemonHTTPAndPreservesJSONOutput(t *testing.T) { require := require.New(t) assert := assert.New(t) @@ -33,11 +126,12 @@ func TestSearchCmd_VectorModeUsesLocalDaemonHTTPAndPreservesJSONOutput(t *testin }) writeStatsHTTPDaemonRuntime(t, dataDir, srv) - restore := configureVectorSearchHTTPTest(t, dataDir, true, "") + testCtx, restore := configureVectorSearchHTTPTest(t, dataDir, true, "") defer restore() done := captureStdout(t) root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(searchCmd) root.SetArgs([]string{ "search", "--mode", "vector", "--json", @@ -52,6 +146,11 @@ func TestSearchCmd_VectorModeUsesLocalDaemonHTTPAndPreservesJSONOutput(t *testin assert.Equal(1, int(requests.Load()), "search endpoint calls") assert.Contains(out, `"returned_count": 1`, "returned_count") + assert.Contains(out, `"took_ms": 12`, "total timing") + assert.Contains(out, `"query_embedding_ms": 2`, "query embedding timing") + assert.Contains(out, `"retrieval_ms": 7`, "retrieval timing") + assert.Contains(out, `"accelerator": "vec1_ivf_opq"`, "accelerator path") + assert.Contains(out, `"hydration_ms": 3`, "hydration timing") assert.Contains(out, `"from_email": "alice@example.com"`, "from_email") assert.Contains(out, `"boosted": true`, "boosted") assert.Contains(out, srv.URL+"/messages/42", "browser URL") @@ -72,15 +171,17 @@ func TestSearchCmd_VectorModeCollectionUsesLocalDaemonHTTPAndPreservesBanner(t * }) writeStatsHTTPDaemonRuntime(t, dataDir, srv) - restore := configureVectorSearchHTTPTest(t, dataDir, true, "") + testCtx, restore := configureVectorSearchHTTPTest(t, dataDir, true, "") defer restore() doneOut := captureStdout(t) doneErr := captureStderr(t) root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(searchCmd) root.SetArgs([]string{ "search", "--mode", "vector", + "--explain", "--collection", "Important", "lunch", }) @@ -93,6 +194,8 @@ func TestSearchCmd_VectorModeCollectionUsesLocalDaemonHTTPAndPreservesBanner(t * assert.Equal(1, int(requests.Load()), "search endpoint calls") assert.Contains(out, "Lunch *", "boosted marker") assert.Contains(out, `Showing 1 result (generation #7 active, fingerprint="fake:4")`, "summary") + assert.Contains(out, "Timings: total=12ms query_embedding=2ms retrieval=7ms hydration=3ms", "timings") + assert.Contains(out, "Accelerator: vec1_ivf_opq", "accelerator path") assert.Contains(errOut, `Searching collection "Important" (2 accounts)`, "collection banner") } @@ -108,10 +211,11 @@ func TestSearchCmd_VectorModeUnknownAccountUsesDaemonError(t *testing.T) { }) writeStatsHTTPDaemonRuntime(t, dataDir, srv) - restore := configureVectorSearchHTTPTest(t, dataDir, true, "") + testCtx, restore := configureVectorSearchHTTPTest(t, dataDir, true, "") defer restore() root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(searchCmd) root.SetArgs([]string{ "search", "--mode", "vector", @@ -134,11 +238,12 @@ func TestSearchCmd_HybridModeUsesConfiguredRemoteHTTP(t *testing.T) { writeVectorSearchResponse(t, w, "alice@example.com", "", 0) }) - restore := configureVectorSearchHTTPTest(t, t.TempDir(), false, srv.URL) + testCtx, restore := configureVectorSearchHTTPTest(t, t.TempDir(), false, srv.URL) defer restore() done := captureStdout(t) root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(searchCmd) root.SetArgs([]string{"search", "--mode", "hybrid", "--explain", "--json", "lunch"}) @@ -149,6 +254,7 @@ func TestSearchCmd_HybridModeUsesConfiguredRemoteHTTP(t *testing.T) { assert.Equal(1, int(requests.Load()), "search endpoint calls") assert.Contains(out, `"bm25_score": 1.25`, "bm25_score") assert.Contains(out, `"vector_score": 0.9`, "vector_score") + assert.Contains(out, `"accelerator": "vec1_ivf_opq"`, "accelerator path") } func vectorSearchHTTPDaemon( @@ -167,8 +273,10 @@ func vectorSearchHTTPDaemon( return srv } -func configureVectorSearchHTTPTest(t *testing.T, dataDir string, local bool, remoteURL string) func() { +func configureVectorSearchHTTPTest(t *testing.T, dataDir string, local bool, remoteURL string) (context.Context, func()) { t.Helper() + cfg := testConfigValue() + useLocal := false savedCfg := cfg savedUseLocal := useLocal cfg = &config.Config{ @@ -179,12 +287,14 @@ func configureVectorSearchHTTPTest(t *testing.T, dataDir string, local bool, rem AllowInsecure: true, }, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + invocationFromContext(testCtx).options.useLocal = local if local { cfg.Remote.URL = "http://configured-daemonclient.invalid" } useLocal = local resetSearchFlags() - return func() { + return testCtx, func() { cfg = savedCfg useLocal = savedUseLocal resetSearchFlags() @@ -201,8 +311,15 @@ func writeVectorSearchResponse( t.Helper() w.Header().Set("Content-Type", "application/json") err := json.NewEncoder(w).Encode(map[string]any{ - "returned": 1, - "pool_saturated": false, + "returned": 1, + "pool_saturated": false, + "accelerator": "vec1_ivf_opq", + "took_ms": 12, + "timings": map[string]any{ + "query_embedding_ms": 2, + "retrieval_ms": 7, + "hydration_ms": 3, + }, "scope_label": scopeLabel, "scope_source_count": scopeSourceCount, "generation": map[string]any{ diff --git a/cmd/msgvault/cmd/serve.go b/cmd/msgvault/cmd/serve.go index b9ec7a8be..e8c814cef 100644 --- a/cmd/msgvault/cmd/serve.go +++ b/cmd/msgvault/cmd/serve.go @@ -12,6 +12,7 @@ import ( "os" "os/signal" "path/filepath" + "slices" "strconv" "strings" "syscall" @@ -24,16 +25,23 @@ import ( "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/deletion" "go.kenn.io/msgvault/internal/discord" + "go.kenn.io/msgvault/internal/gcal" "go.kenn.io/msgvault/internal/gmail" "go.kenn.io/msgvault/internal/granola" imaplib "go.kenn.io/msgvault/internal/imap" + "go.kenn.io/msgvault/internal/jobctx" "go.kenn.io/msgvault/internal/meetingimport" - "go.kenn.io/msgvault/internal/microsoft" + "go.kenn.io/msgvault/internal/muesli" "go.kenn.io/msgvault/internal/notionmeetings" "go.kenn.io/msgvault/internal/oauth" "go.kenn.io/msgvault/internal/operations" + "go.kenn.io/msgvault/internal/peoplesweep" + "go.kenn.io/msgvault/internal/personagenda" "go.kenn.io/msgvault/internal/personenrichment" "go.kenn.io/msgvault/internal/personfacts" + "go.kenn.io/msgvault/internal/personmatch" + "go.kenn.io/msgvault/internal/plaud" + "go.kenn.io/msgvault/internal/provideridentity" "go.kenn.io/msgvault/internal/query" "go.kenn.io/msgvault/internal/scheduler" "go.kenn.io/msgvault/internal/search" @@ -131,25 +139,61 @@ type serveRuntimeScheduler interface { Stop() context.Context } +// serveSchedulers stops every daemon scheduler and reports when all have drained. +type serveSchedulers []serveRuntimeScheduler + +func (s serveSchedulers) Stop() context.Context { + stopped := make([]context.Context, 0, len(s)) + for _, sched := range s { + stopped = append(stopped, sched.Stop()) + } + ctx, cancel := context.WithCancel(context.Background()) + go func() { + for _, done := range stopped { + <-done.Done() + } + cancel() + }() + return ctx +} + +// newServeSchedulers returns the daemon scheduler, whose jobs hold the +// operation gate, and one for Beeper media delivery. A media upload can run +// for minutes, so that job takes the gate only around its Store writes. +// Media passes do not count as activity; shutdown cancels and drains them. +func newServeSchedulers( + syncFunc scheduler.SyncFunc, logger *slog.Logger, idle scheduler.WorkTracker, gate api.LabeledOperationGate, +) (*scheduler.Scheduler, *scheduler.Scheduler) { + sched := scheduler.New(syncFunc).WithLogger(logger). + WithWorkTracker(combineWorkTrackers(idle, labelWorkTracker(gate, "a scheduled sync"))) + media := scheduler.New(nil).WithLogger(logger) + return sched, media +} + type serveRuntimeOperationGate interface { StartDrain() Wait(ctx context.Context) error } func init() { + addServeConfigFlags(serveCmd) rootCmd.AddCommand(serveCmd) rootCmd.AddCommand(daemonCmd) addServeLifecycleCommands(serveCmd) } func runServe(cmd *cobra.Command, args []string) error { - // Validate security posture before doing any work - if err := cfg.Server.ValidateSecure(); err != nil { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil || state.logger == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger + // Resolve the interface before reserving a listener. Credential creation + // waits until this process owns the daemon lock. + if _, err := cfg.ResolveServerBindAddress(); err != nil { return err } - if cfg.Server.APIKey != "" && len(cfg.Server.APIKey) < 16 { - logger.Warn("api_key is very short — use a randomly generated key of at least 32 characters") - } // Missing provider credentials should not prevent the daemon from serving // read-only HTTP requests against an import-only archive. @@ -172,6 +216,7 @@ func runServe(cmd *cobra.Command, args []string) error { if err != nil { return err } + logger.Info("Server listener bound", "address", apiListener.Addr().String(), "bind_source", cfg.BindAddressSource()) listenerReserved := true defer func() { if listenerReserved { @@ -191,6 +236,12 @@ func runServe(cmd *cobra.Command, args []string) error { if err != nil { return fmt.Errorf("claim daemon ownership: %w", err) } + if !cfg.Server.HasCredentialSource() && !cfg.Server.AllowInsecure && cfg.Server.AuthenticationKey() != "" { + logger.Info("Server API credential is persisted", "path", cfg.ServerKeyFilePath()) + } + if cfg.Server.AuthenticationKey() != "" && len(cfg.Server.AuthenticationKey()) < 16 { + logger.Warn("api_key is very short — use a randomly generated key of at least 32 characters") + } heartbeatCtx, stopHeartbeat := context.WithCancel(cmd.Context()) heartbeatDone := make(chan struct{}) go func() { @@ -223,7 +274,7 @@ func runServe(cmd *cobra.Command, args []string) error { resourceCleanupSafe := true defer func() { if !resourceCleanupSafe { - logger.Warn("archive database cleanup skipped", "error", "HTTP shutdown did not complete") + logger.Warn("archive database cleanup skipped", "error", "daemon shutdown did not complete safely") return } if err := closeDaemonStoreAfterInitializers(s, analyticsInit, vectorInit); err != nil { @@ -268,7 +319,7 @@ func runServe(cmd *cobra.Command, args []string) error { // (which may open files and run migrations) respects Ctrl+C. ctx, cancel := context.WithCancel(cmd.Context()) defer cancel() - idleTracker := newDaemonIdleTracker(cfg, cancel) + idleTracker := newDaemonIdleTracker(cfg, cancel, logger) operationGate := api.NewSerialOperationGate() // Closed on shutdown so cached pack readers don't hold attachment pack // files open past the daemon's lifetime (blocks deletion on Windows). @@ -283,13 +334,36 @@ func runServe(cmd *cobra.Command, args []string) error { _ = attachmentMaint.close() } }() + var initialAnalyticsEngine query.Engine + analyticsServerStarted := false + defer func() { + if resourceCleanupSafe && !analyticsServerStarted && initialAnalyticsEngine != nil { + _ = initialAnalyticsEngine.Close() + } + }() + // Due analytics cache rebuilds run here, off the scheduled job that asked + // for them, so a sync never holds the operation gate for a cache build. + cacheRefresher := newBackgroundCacheRefresher(ctx, nil, idleTracker) + daemonCacheRefresher = cacheRefresher + defer func() { daemonCacheRefresher = nil }() + cacheRefresherShutdown := false + defer func() { + if cacheRefresherShutdown { + return + } + if err := shutdownBackgroundCacheRefresher(cacheRefresher); err != nil { + resourceCleanupSafe = false + return + } + cacheRefresherShutdown = true + }() blobStore := attachmentMaint.blob // Vector misconfiguration still fails startup fast; the expensive // backend open/migrate/backfill runs in the background after the API // server is listening (startVectorInit below), so the TUI and other // clients are not blocked by vector maintenance. - if err := precheckVectorFeatures(dbPath); err != nil { + if err := precheckVectorFeatures(dbPath, cfg); err != nil { return fmt.Errorf("vector features: %w", err) } if !cfg.Vector.AnyLaneEnabled() { @@ -319,18 +393,12 @@ func runServe(cmd *cobra.Command, args []string) error { } return engineErr } - initialAnalyticsEngine := engine - analyticsServerStarted := false - defer func() { - if resourceCleanupSafe && !analyticsServerStarted && initialAnalyticsEngine != nil { - _ = initialAnalyticsEngine.Close() - } - }() + initialAnalyticsEngine = engine if !analyticsAsync { logger.Info("daemon startup step complete", "step", "init_analytics_engine") } - getOAuthMgr := oauthManagerCache() + getOAuthMgr := oauthManagerCache(invocationFromCommand(cmd)) // Create sync function for the scheduler. Under scan-and-fill the // Syncer no longer needs an enqueuer — newly-ingested messages get @@ -338,20 +406,23 @@ func runServe(cmd *cobra.Command, args []string) error { // by the background startVectorInit) discovers them on its next run, so // the sync path no longer threads the vector features. syncFunc := func(ctx context.Context, email string) error { + ctx = withInvocation(ctx, state) return runScheduledSource(ctx, attachmentMaint, true, func(ctx context.Context) error { - return runScheduledSync(ctx, email, s, getOAuthMgr) + return runScheduledSync(ctx, email, s, getOAuthMgr, state) }) } // Create and configure scheduler - sched := scheduler.New(syncFunc).WithLogger(logger). - WithWorkTracker(combineWorkTrackers(idleTracker, labelWorkTracker(operationGate, "a scheduled sync"))) + sched, mediaSched := newServeSchedulers(syncFunc, logger, idleTracker, operationGate) + sched.WithAccountPreemptionPolicy(func(identifier string) bool { + return scheduledSyncPreemptible(s, identifier, logger) + }) cardDAVController, err := api.NewCardDAVController(cfg, s, logger) if err != nil { return fmt.Errorf("configure CardDAV: %w", err) } - cardDAVController.SetScheduleReconciler(func(cardDAVConfig config.CardDAVConfig, service api.CardDAVOperations) error { - return reconcileCardDAVSchedulerJob(sched, cardDAVConfig, service, logger) + cardDAVController.SetConnectionScheduleReconciler(func(name string, cardDAVConfig config.CardDAVConfig, service api.CardDAVOperations) error { + return reconcileCardDAVSchedulerJob(sched, cardDAVConfig, service, logger, name) }) if err := cardDAVController.ReconcileSchedule(); err != nil { return err @@ -382,11 +453,11 @@ func runServe(cmd *cobra.Command, args []string) error { if err := sched.AddJob(scheduler.Job{ Name: jobName, Schedule: source.Schedule, - Run: func(ctx context.Context) error { + Run: invocationBoundJobRun(state, func(ctx context.Context) error { return runScheduledSource(ctx, attachmentMaint, true, func(ctx context.Context) error { return runConfiguredSynctechSMSSourceWithStore(ctx, s, source) }) - }, + }), }); err != nil { logger.Error("failed to schedule synctech-sms source", "source", source.Name, "error", err) } else { @@ -416,11 +487,11 @@ func runServe(cmd *cobra.Command, args []string) error { if err := sched.AddJob(scheduler.Job{ Name: jobName, Schedule: source.Schedule, - Run: func(ctx context.Context) error { + Run: invocationBoundJobRun(state, func(ctx context.Context) error { return runScheduledSource(ctx, attachmentMaint, false, func(ctx context.Context) error { return runConfiguredGCalSync(ctx, s, source) }) - }, + }), }); err != nil { logger.Error("failed to schedule gcal source", "source", source.Name, "error", err) } else { @@ -430,11 +501,21 @@ func runServe(cmd *cobra.Command, args []string) error { if err := registerAttachmentMaintenanceJob(sched, attachmentMaint); err != nil { return fmt.Errorf("schedule attachment maintenance: %w", err) } + if err := registerAttachmentPackJob(sched, attachmentMaint); err != nil { + return fmt.Errorf("schedule attachment packing: %w", err) + } + if err := registerSQLiteMaintenanceJob(sched, s); err != nil { + return fmt.Errorf("schedule SQLite maintenance: %w", err) + } if err := configureDocumentReconcileJob( ctx, sched, s, cfg.Attachments.Documents.Enabled, ); err != nil { return fmt.Errorf("configure document reconciliation: %w", err) } + if err := configureBeeperMediaJob(ctx, mediaSched, operationGate, s, blobStore, + filepath.Join(cfg.Data.DataDir, "tmp", "beeper-media"), cfg.Integrations.Docbank, logger); err != nil { + logger.Warn("Beeper media submission unavailable", "error", err) + } if err := registerActivityProjectionJob( sched, s, cfg.Activity, logger); err != nil { return fmt.Errorf("schedule activity projection: %w", err) @@ -462,9 +543,9 @@ func runServe(cmd *cobra.Command, args []string) error { "hint", `set a cron schedule (e.g. "*/30 * * * *") on the [beeper] entry`) } if cfg.Beeper.Enabled && cfg.Beeper.Schedule != "" { - if err := registerScheduledBeeperJob(sched, cfg.Beeper.Schedule, attachmentMaint, func(ctx context.Context) error { + if err := registerScheduledBeeperJob(sched, cfg.Beeper.Schedule, attachmentMaint, invocationBoundJobRun(state, func(ctx context.Context) error { return runConfiguredBeeperSync(ctx, s) - }); err != nil { + })); err != nil { logger.Error("failed to schedule beeper sync", "error", err) } else { logger.Info("scheduled beeper sync", "schedule", cfg.Beeper.Schedule) @@ -477,13 +558,14 @@ func runServe(cmd *cobra.Command, args []string) error { } if cfg.Slack.Enabled && cfg.Slack.Schedule != "" { if err := sched.AddJob(scheduler.Job{ - Name: api.SlackJobName, - Schedule: cfg.Slack.Schedule, - Run: func(ctx context.Context) error { + Name: api.SlackJobName, + Schedule: cfg.Slack.Schedule, + Preemptible: true, + Run: invocationBoundJobRun(state, func(ctx context.Context) error { return runScheduledSource(ctx, attachmentMaint, true, func(ctx context.Context) error { return runConfiguredSlackSync(ctx, s) }) - }, + }), }); err != nil { logger.Error("failed to schedule slack sync", "error", err) } else { @@ -491,7 +573,7 @@ func runServe(cmd *cobra.Command, args []string) error { } } - // Meeting sources (Granola/Circleback) mirror the gcal treatment: warn + // Meeting sources mirror the gcal treatment: warn // when enabled but unscheduled, then register the scheduled ones. for _, src := range cfg.Granola { if src.Enabled && src.Schedule == "" { @@ -513,9 +595,9 @@ func runServe(cmd *cobra.Command, args []string) error { if err := sched.AddJob(scheduler.Job{ Name: jobName, Schedule: source.Schedule, - Run: func(ctx context.Context) error { + Run: invocationBoundJobRun(state, func(ctx context.Context) error { return runConfiguredGranolaSync(ctx, s, source) - }, + }), }); err != nil { logger.Error("failed to schedule granola source", "source", source.Identifier, "error", err) } else { @@ -542,15 +624,41 @@ func runServe(cmd *cobra.Command, args []string) error { if err := sched.AddJob(scheduler.Job{ Name: jobName, Schedule: source.Schedule, - Run: func(ctx context.Context) error { + Run: invocationBoundJobRun(state, func(ctx context.Context) error { return runConfiguredCirclebackSync(ctx, s, source) - }, + }), }); err != nil { logger.Error("failed to schedule circleback source", "source", source.Identifier, "error", err) } else { logger.Info("scheduled circleback source", "source", source.Identifier, "schedule", source.Schedule) } } + for _, src := range cfg.Plaud { + if src.Enabled && src.Schedule == "" { + logger.Warn("plaud source is enabled but has no schedule — the daemon will not sync it; its freshness will eventually go stale", + "source", src.Identifier, + "hint", `set a cron schedule (e.g. "30 */6 * * *") on the [[plaud]] entry`) + } + } + for _, src := range cfg.ScheduledPlaudSources() { + source := src + jobName, ok := api.SchedulerJobNameForSource(plaud.SourceType, source.Identifier) + if !ok { + logger.Error("no scheduler job mapping for plaud source", "source", source.Identifier) + continue + } + if err := sched.AddJob(scheduler.Job{ + Name: jobName, + Schedule: source.Schedule, + Run: invocationBoundJobRun(state, func(ctx context.Context) error { + return runConfiguredPlaudSync(ctx, s, source) + }), + }); err != nil { + logger.Error("failed to schedule plaud source", "source", source.Identifier, "error", err) + } else { + logger.Info("scheduled plaud source", "source", source.Identifier, "schedule", source.Schedule) + } + } for _, src := range cfg.NotionMeetings { if src.Enabled && src.Schedule == "" { logger.Warn("notion meeting source is enabled but has no schedule — the daemon will not sync it; its freshness will eventually go stale", @@ -568,21 +676,48 @@ func runServe(cmd *cobra.Command, args []string) error { if err := sched.AddJob(scheduler.Job{ Name: jobName, Schedule: source.Schedule, - Run: func(ctx context.Context) error { + Run: invocationBoundJobRun(state, func(ctx context.Context) error { return runConfiguredNotionMeetingsSync(ctx, s, source) - }, + }), }); err != nil { logger.Error("failed to schedule notion meeting source", "source", source.Identifier, "error", err) } else { logger.Info("scheduled notion meeting source", "source", source.Identifier, "schedule", source.Schedule) } } + for _, src := range cfg.Muesli { + if src.Enabled && src.Schedule == "" { + logger.Warn("muesli source is enabled but has no schedule — the daemon will not sync it; its freshness will eventually go stale", + "source", src.Identifier, + "hint", `set a cron schedule (e.g. "*/30 * * * *") on the [[muesli]] entry`) + } + } + for _, src := range cfg.ScheduledMuesliSources() { + source := src + jobName, ok := api.SchedulerJobNameForSource(muesli.SourceType, source.Identifier) + if !ok { + logger.Error("no scheduler job mapping for muesli source", "source", source.Identifier) + continue + } + if err := sched.AddJob(scheduler.Job{ + Name: jobName, + Schedule: source.Schedule, + Run: invocationBoundJobRun(state, func(ctx context.Context) error { + return runConfiguredMuesliSync(ctx, s, source) + }), + }); err != nil { + logger.Error("failed to schedule muesli source", "source", source.Identifier, "error", err) + } else { + logger.Info("scheduled muesli source", "source", source.Identifier, "schedule", source.Schedule) + } + } sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) - // Start the scheduler + // Start the schedulers sched.Start() + mediaSched.Start() // Create adapters for the API interfaces refreshCacheAfterWrite := func(_ context.Context, label string) error { @@ -592,21 +727,29 @@ func runServe(cmd *cobra.Command, args []string) error { } meetingImporter := meetingimport.NewImporter(s, meetingimport.Hooks{ AfterSourceSetup: func() error { - return runPostSourceCreateMigrations(s) + return runPostSourceCreateMigrationsForInvocation(s, state) }, RefreshCache: refreshCacheAfterWrite, }).WithLogger(logger) + cacheJobs := newCacheBuildJobs(ctx, idleTracker, nil) + cacheJobs.logger = logger storeAdapter := &storeAPIAdapter{ store: s, + config: cfg, + options: state.options, + logger: logger, draftPolicy: snapshotIMAPDraftPolicy(cfg), + gmailDraftPolicy: snapshotGmailDraftPolicy(cfg), + beeperDraftPolicy: snapshotBeeperDraftPolicy(cfg), draftCacheRefresh: refreshCacheAfterWrite, attachmentMaintenance: attachmentMaint, meetingImporter: meetingImporter, analyticsDir: cfg.AnalyticsDir(), + cacheJobs: cacheJobs, personEnrichmentConfig: cfg.People.Enrichment, lookupEnv: personEnrichmentEnvironmentLookup(cfg), } - schedAdapter := &schedulerAdapter{scheduler: sched} + schedAdapter := &schedulerAdapter{scheduler: sched, media: mediaSched} // Create and start API server var apiServer *api.Server @@ -615,12 +758,16 @@ func runServe(cmd *cobra.Command, args []string) error { Store: storeAdapter, SavedViewStore: s, Engine: engine, - SQLQueryRunner: func(ctx context.Context, sql string) (*query.QueryResult, error) { + SQLQueryRunner: func(requestCtx context.Context, sql string, fresh bool) (*query.QueryResult, *api.CacheBuildAccepted, error) { if apiServer == nil { - return nil, errors.New("daemon API server unavailable") + return nil, nil, errors.New("daemon API server unavailable") } - return runDaemonSQLQuery(ctx, cfg, s, apiServer.QueryEngineForRequest(ctx), sql) + return runDaemonSQLQueryWithJobs(requestCtx, cfg, s, apiServer.QueryEngineForRequest(requestCtx), sql, daemonSQLQueryOptions{fresh: fresh}, cacheJobs) + }, + ArchiveSQLQueryRunner: func(requestCtx context.Context, sql string, fresh bool) (*query.QueryResult, *api.CacheBuildAccepted, error) { + return runDaemonSQLQueryWithJobs(requestCtx, cfg, s, nil, sql, daemonSQLQueryOptions{fresh: fresh, archiveOnly: true}, cacheJobs) }, + CacheBuildStatusReader: cacheJobs.status, ShutdownToken: ownership.shutdownToken, ShutdownFunc: cancel, Scheduler: schedAdapter, @@ -634,6 +781,24 @@ func runServe(cmd *cobra.Command, args []string) error { OperationHistoryReader: storeAdapter, BlobStore: blobStore, } + apiOpts.GmailProfileAddress = func(ctx context.Context, source *store.Source) (string, error) { + client, serviceAccount, err := newDaemonGmailClient( + ctx, source.Identifier, source, getOAuthMgr, state, + ) + if err != nil { + return "", err + } + defer func() { _ = client.Close() }() + profile, err := client.GetProfile(ctx) + if err != nil { + classified := provideridentity.ClassifyGmailProfileError(err, serviceAccount) + return "", fmt.Errorf("read authenticated Gmail profile: %w", classified) + } + if profile == nil { + return "", errors.New("authenticated Gmail profile is missing") + } + return profile.EmailAddress, nil + } applyServerRuntimeConfig(&apiOpts, cfg) if cfg.Vector.AnyLaneEnabled() { apiOpts.VectorStatus = api.VectorStatusInitializing @@ -740,10 +905,24 @@ func runServe(cmd *cobra.Command, args []string) error { shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), serveOperationDrainTimeout) defer shutdownCancel() - shutdownErr := shutdownServeRuntime(shutdownCtx, cmd.OutOrStdout(), apiServer, sched, operationGate) + shutdownErr := shutdownServeRuntime(shutdownCtx, cmd.OutOrStdout(), apiServer, serveSchedulers{sched, mediaSched}, operationGate) + if !cacheJobs.waitContext(shutdownCtx) { + logger.Warn("analytics cache build did not stop within the shutdown drain timeout") + shutdownErr = errors.Join(shutdownErr, errors.New("analytics cache build did not stop during shutdown")) + } if shutdownErr == nil { resourceCleanupSafe = true } + if err := cacheRefresher.Shutdown(shutdownCtx); err != nil { + logger.Warn("analytics cache refresh did not stop within the shutdown drain timeout", "error", err) + if retryErr := shutdownBackgroundCacheRefresher(cacheRefresher); retryErr != nil { + resourceCleanupSafe = false + } else { + cacheRefresherShutdown = true + } + } else { + cacheRefresherShutdown = true + } // Wait for the background vector init regardless of the shutdown // outcome: the deferred s.Close() must not run under a still-running // init goroutine, and vectors.db needs closing whenever init finished. @@ -754,7 +933,7 @@ func runServe(cmd *cobra.Command, args []string) error { if vectorInit != nil { if vectorInit.WaitContext(shutdownCtx) { if resourceCleanupSafe { - vectorInit.CloseFeatures() + vectorInit.CloseFeatures(logger) } } else { logger.Warn("vector init did not stop within the shutdown drain timeout; skipping vectors.db close") @@ -788,33 +967,48 @@ func snapshotIMAPDraftPolicy(cfg *config.Config) []config.IMAPDraftSource { return append([]config.IMAPDraftSource(nil), cfg.IMAP.Drafts...) } -func reconcileCardDAVSchedulerJob(sched *scheduler.Scheduler, cardDAVConfig config.CardDAVConfig, service api.CardDAVOperations, logger *slog.Logger) error { +func snapshotGmailDraftPolicy(cfg *config.Config) []config.GmailDraftSource { + if cfg == nil { + return nil + } + return append([]config.GmailDraftSource(nil), cfg.Gmail.Drafts...) +} + +func snapshotBeeperDraftPolicy(cfg *config.Config) []config.GmailDraftSource { + if cfg == nil { + return nil + } + return append([]config.GmailDraftSource(nil), cfg.Beeper.Drafts...) +} + +func reconcileCardDAVSchedulerJob(sched *scheduler.Scheduler, cardDAVConfig config.CardDAVConfig, service api.CardDAVOperations, logger *slog.Logger, name string) error { + jobName := api.CardDAVJobNameForConnection(name) if !cardDAVConfig.Enabled || cardDAVConfig.Schedule == "" { - sched.RemoveJob(api.CardDAVJobName) + sched.RemoveJob(jobName) if cardDAVConfig.Enabled && cardDAVConfig.Schedule == "" { logger.Warn("carddav is enabled but has no schedule — the daemon will not sync it", - "hint", `set a cron schedule (e.g. "0 */6 * * *") in [carddav]`) + "connection", name, "hint", `set a cron schedule (e.g. "0 */6 * * *") in [carddav]`) } return nil } if service == nil { - sched.RemoveJob(api.CardDAVJobName) + sched.RemoveJob(jobName) hint := "save the CardDAV account with its password to repair the connection" if cardDAVConfig.Provider == "google" { hint = "connect Google in CardDAV account settings, then test and save the account" } logger.Warn("carddav credentials are unavailable or do not match saved discovery; skipping scheduled sync", - "hint", hint) + "connection", name, "hint", hint) return nil } if err := sched.AddJob(scheduler.Job{ - Name: api.CardDAVJobName, Schedule: cardDAVConfig.Schedule, + Name: jobName, Schedule: cardDAVConfig.Schedule, Run: func(ctx context.Context) error { _, err := service.Sync(ctx, carddav.SyncOptions{Trigger: store.CardDAVSyncTriggerScheduled}) return err }, }); err != nil { - return fmt.Errorf("schedule CardDAV sync: %w", err) + return fmt.Errorf("schedule CardDAV connection %s: %w", name, err) } return nil } @@ -873,6 +1067,11 @@ func applyServerRuntimeConfig(options *api.ServerOptions, cfg *config.Config) { } func listenServeAPI(bindAddr string, port int) (net.Listener, error) { + resolved, err := resolveServeBind(bindAddr) + if err != nil { + return nil, err + } + bindAddr = resolved if bindAddr == "" { bindAddr = defaultDaemonBindAddr } @@ -965,51 +1164,155 @@ func shutdownServeRuntime( return nil } -func runDaemonSQLQuery( - ctx context.Context, - c *config.Config, - s *store.Store, - engine query.Engine, - sqlStr string, -) (*query.QueryResult, error) { - if c == nil || s == nil { - return nil, errors.New("daemon query unavailable") +type daemonSQLQueryOptions struct { + fresh bool + archiveOnly bool +} + +// runDaemonSQLQueryWithJobs serves committed data while refresh work runs in +// the daemon's background job registry. The request never waits for a cache +// builder; a fresh request or unusable publication receives a job ID instead. +func runDaemonSQLQueryWithJobs( + ctx context.Context, c *config.Config, s *store.Store, engine query.Engine, + sqlStr string, options daemonSQLQueryOptions, jobs *cacheBuildJobs, +) (*query.QueryResult, *api.CacheBuildAccepted, error) { + if jobs == nil || c == nil || s == nil { + return nil, nil, errors.New("daemon cache refresh unavailable") } - if engine == nil { - return nil, api.ErrSQLQueryEngineUnavailable + if err := query.EnsureReadOnly(sqlStr); err != nil { + return nil, nil, err } if s.IsPostgreSQL() { + if options.archiveOnly { + return nil, nil, api.ErrSQLQueryEngineUnavailable + } if querier, ok := engine.(query.SQLQuerier); ok { - return querier.QuerySQL(ctx, sqlStr) + result, err := querier.QuerySQL(ctx, sqlStr) + return result, nil, err } - return nil, errors.New("SQL query requires DuckDB engine") + return nil, nil, api.ErrSQLQueryEngineUnavailable } - - dbPath := c.DatabaseDSN() - analyticsDir := c.AnalyticsDir() - staleness := cacheNeedsBuild(dbPath, analyticsDir) - if !store.IsPostgresURL(dbPath) && !staleness.NeedsBuild { - if querier, ok := engine.(query.SQLQuerier); ok { - return querier.QuerySQL(ctx, sqlStr) + staleness, err := cacheNeedsBuildForServing(ctx, c.DatabaseDSN(), c.AnalyticsDir()) + if err != nil { + return nil, nil, fmt.Errorf("inspect analytics cache: %w", err) + } + if options.fresh { + job, err := jobs.acceptAfterWrite(buildCacheModeAuto) + if err != nil { + return nil, nil, fmt.Errorf("%w: %w", api.ErrCacheBuildUnavailable, err) } + return nil, &api.CacheBuildAccepted{ + Status: job.Status, JobID: job.JobID, + }, nil } - if staleness.NeedsBuild { - if err := buildCacheSubprocessForRun(ctx, staleness.FullRebuild); err != nil { - return nil, fmt.Errorf("build cache: %w", err) + if !c.Analytics.AutoBuildCache { + if !staleness.HasUsablePublication { + return nil, nil, api.ErrSQLQueryEngineUnavailable + } + } else if !staleness.HasUsablePublication { + job, err := jobs.accept(buildCacheModeAuto) + if err != nil { + return nil, nil, fmt.Errorf("%w: %w", api.ErrCacheBuildUnavailable, err) + } + return nil, &api.CacheBuildAccepted{ + Status: job.Status, JobID: job.JobID, Cache: cacheFreshnessFromStaleness(staleness), + }, nil } - logger.Info("rebuilt analytics cache for SQL query", - "reason", staleness.Reason, - "full_rebuild", staleness.FullRebuild) + if c.Analytics.AutoBuildCache { + if _, deferBuild := scheduledCacheBuildDelay(staleness, c.Analytics.MinRebuildInterval, time.Now()); !deferBuild { + if _, err := jobs.accept(buildCacheModeScheduledAuto); err != nil { + return nil, nil, fmt.Errorf("%w: %w", api.ErrCacheBuildUnavailable, err) + } + } + } + } else if c.Analytics.AutoBuildCache && staleness.HasUsablePublication { + if err := jobs.verifyWhenDue(staleness.PublishedAt, c.Analytics.MinRebuildInterval, time.Now()); err != nil { + return nil, nil, fmt.Errorf("%w: %w", api.ErrCacheBuildUnavailable, err) + } + } + // Raw SQL over a SQLite archive must use the committed Parquet views, + // even if the aggregate engine is a SQL-capable fallback. + querier, ok := engine.(*query.DuckDBEngine) + if options.archiveOnly { + duckOptions, err := daemonDuckDBOptions(c) + if err != nil { + return nil, nil, err + } + duckEngine, err := query.NewArchiveDuckDBEngine(ctx, c.AnalyticsDir(), duckOptions) + if err != nil { + if errors.Is(err, query.ErrCacheUnavailable) { + return acceptUnavailableCacheQuery(c, jobs, options.fresh) + } + return nil, nil, fmt.Errorf("open archive SQL engine: %w", err) + } + defer func() { _ = duckEngine.Close() }() + querier = duckEngine + } else if !ok { + duckEngine, err := openDaemonDuckDBEngine(c, s) + if err != nil { + if errors.Is(err, query.ErrCacheUnavailable) { + return acceptUnavailableCacheQuery(c, jobs, options.fresh) + } + return nil, nil, fmt.Errorf("open DuckDB query engine: %w", err) + } + defer func() { _ = duckEngine.Close() }() + querier = duckEngine } + result, err := queryCommittedSQL(ctx, querier, sqlStr, staleness) + if err != nil { + if errors.Is(err, query.ErrCacheUnavailable) { + return acceptUnavailableCacheQuery(c, jobs, options.fresh) + } + return nil, nil, err + } + if result.Cache != nil { + result.Cache.Building = jobs.active() + } + return result, nil, nil +} - duckEngine, err := openDaemonDuckDBEngine(c, s) +func acceptUnavailableCacheQuery( + c *config.Config, jobs *cacheBuildJobs, fresh bool, +) (*query.QueryResult, *api.CacheBuildAccepted, error) { + if !fresh && !c.Analytics.AutoBuildCache { + return nil, nil, api.ErrSQLQueryEngineUnavailable + } + job, err := jobs.accept(buildCacheModeAuto) if err != nil { - return nil, fmt.Errorf("open DuckDB query engine: %w", err) + return nil, nil, fmt.Errorf("%w: %w", api.ErrCacheBuildUnavailable, err) } - defer func() { _ = duckEngine.Close() }() + return nil, &api.CacheBuildAccepted{Status: job.Status, JobID: job.JobID}, nil +} - return duckEngine.QuerySQL(ctx, sqlStr) +func cacheFreshnessFromStaleness(staleness cacheStaleness) *query.CacheFreshness { + if !staleness.HasUsablePublication { + return nil + } + return &query.CacheFreshness{ + Generation: staleness.Generation, PublishedAt: staleness.PublishedAt, + StaleReason: staleness.Reason, PendingAdditions: staleness.PendingAdditions, Building: true, + } +} + +func queryCommittedSQL( + ctx context.Context, querier query.SQLQuerier, sqlStr string, staleness cacheStaleness, +) (*query.QueryResult, error) { + result, err := querier.QuerySQL(ctx, sqlStr) + if err != nil || result == nil || result.Cache == nil { + return result, err + } + if result.Cache.Generation != staleness.Generation || + !result.Cache.PublishedAt.Equal(staleness.PublishedAt) { + // A publication may have landed between the staleness probe and the + // engine's read lock. Keep the engine's actual generation metadata. + return result, nil + } + if staleness.NeedsBuild { + result.Cache.StaleReason = staleness.Reason + result.Cache.PendingAdditions = staleness.PendingAdditions + } + return result, nil } // openDaemonAnalyticsEngine picks the daemon's analytics engine once at @@ -1026,6 +1329,7 @@ func openDaemonAnalyticsEngine( return nil, "", startupCacheBuildOutcomeNone, errors.New("daemon analytics engine unavailable") } + logger := loggerFromContext(ctx) if s.IsPostgreSQL() { outcome := startupCacheBuildOutcomeNone if intent != startupCacheBuildIntentNone { @@ -1050,10 +1354,32 @@ func openDaemonAnalyticsEngine( dbPath := c.DatabaseDSN() analyticsDir := c.AnalyticsDir() - staleness := cacheNeedsBuild(dbPath, analyticsDir) + staleness := cacheNeedsBuildContext(ctx, dbPath, analyticsDir) + if err := ctx.Err(); err != nil { + return nil, "", startupCacheBuildOutcomeNone, err + } outcome := startupCacheBuildOutcomeNone - shouldBuild := intent != startupCacheBuildIntentNone || - (staleness.NeedsBuild && c.Analytics.AutoBuildCache) + automaticBuild := intent == startupCacheBuildIntentNone && + staleness.NeedsBuild && c.Analytics.AutoBuildCache + if automaticBuild { + // A restart must not force the full rebuild the post-sync path is + // throttling: serve the recent publication and refresh once the + // interval has elapsed. + if remaining, throttle := scheduledCacheBuildDelay( + staleness, c.Analytics.MinRebuildInterval, scheduledCacheBuildNow(), + ); throttle { + logger.Info("serving existing analytics cache; rebuild deferred by min_rebuild_interval", + "published_at", staleness.PublishedAt, + "remaining", remaining.String(), + "reason", staleness.Reason) + if refresher := daemonCacheRefresher; refresher != nil { + refresher.RequestAfter(remaining, "startup") + } + automaticBuild = false + } + } + shouldBuild := intent != startupCacheBuildIntentNone || automaticBuild + var automaticBuildErr error if shouldBuild { // Build the cache before serving rather than starting on live-SQL // fallback: incremental rebuilds take seconds, and startup progress @@ -1077,6 +1403,15 @@ func openDaemonAnalyticsEngine( } else { buildErr = buildCacheSubprocessForRun(ctx, staleness.FullRebuild) } + if err := ctx.Err(); err != nil { + if intent != startupCacheBuildIntentNone { + outcome = startupCacheBuildOutcomeFailed + if engineMode == config.AnalyticsEngineDuckDB { + outcome = startupCacheBuildOutcomeFatal + } + } + return nil, "", outcome, err + } if buildErr != nil { if intent != startupCacheBuildIntentNone { outcome = startupCacheBuildOutcomeFailed @@ -1086,25 +1421,43 @@ func openDaemonAnalyticsEngine( "reason", reason, "full_rebuild", fullBuild, "error", buildErr) - if engineMode == config.AnalyticsEngineDuckDB { - if intent != startupCacheBuildIntentNone { + if intent != startupCacheBuildIntentNone { + if engineMode == config.AnalyticsEngineDuckDB { outcome = startupCacheBuildOutcomeFatal + return nil, "", outcome, fmt.Errorf("build analytics cache: %w", buildErr) } - return nil, "", outcome, fmt.Errorf("build analytics cache: %w", buildErr) - } - if intent != startupCacheBuildIntentNone { return query.NewEngine(s.DB(), false), api.AnalyticsModeSQLFallback, outcome, nil } + // A usable publication can still be served below, including a + // partial snapshot awaiting a full repair. + automaticBuildErr = buildErr } else { logger.Info("daemon startup step complete", "step", "build_analytics_cache", "reason", reason, "full_rebuild", fullBuild) } - staleness = cacheNeedsBuild(dbPath, analyticsDir) + staleness = cacheNeedsBuildContext(ctx, dbPath, analyticsDir) + if err := ctx.Err(); err != nil { + if intent != startupCacheBuildIntentNone { + outcome = startupCacheBuildOutcomeFailed + if engineMode == config.AnalyticsEngineDuckDB { + outcome = startupCacheBuildOutcomeFatal + } + } + return nil, "", outcome, err + } } - if !staleness.NeedsBuild { + // A stale but usable publication is still served while automatic refresh + // is on (the post-sync path throttles rebuilds the same way at runtime), + // and always for engine="duckdb", which never falls back to live SQL. + servesStale := staleness.HasUsablePublication && + (c.Analytics.AutoBuildCache || engineMode == config.AnalyticsEngineDuckDB) + if !staleness.NeedsBuild || servesStale { + if err := ctx.Err(); err != nil { + return nil, "", outcome, err + } duckEngine, err := openDaemonDuckDBEngineForRun(c, s) if err != nil { if intent != startupCacheBuildIntentNone { @@ -1133,6 +1486,9 @@ func openDaemonAnalyticsEngine( return duckEngine, api.AnalyticsModeDuckDB, outcome, nil } + if err := ctx.Err(); err != nil { + return nil, "", outcome, err + } if intent != startupCacheBuildIntentNone { outcome = startupCacheBuildOutcomeFailed } @@ -1140,6 +1496,9 @@ func openDaemonAnalyticsEngine( if intent != startupCacheBuildIntentNone { outcome = startupCacheBuildOutcomeFatal } + if automaticBuildErr != nil { + return nil, "", outcome, fmt.Errorf("build analytics cache: %w", automaticBuildErr) + } reason := staleness.Reason if reason == "" { reason = "analytics cache is missing or incomplete" @@ -1164,38 +1523,34 @@ func openDaemonDuckDBEngine(c *config.Config, s *store.Store) (*query.DuckDBEngi if c == nil || s == nil { return nil, errors.New("daemon DuckDB engine unavailable") } - spillParent, err := query.PrepareDaemonSpillDir(c.HomeDir) + options, err := daemonDuckDBOptions(c) if err != nil { return nil, err } - // Each engine spills into its own subdirectory: the daemon opens both a - // long-lived engine and short-lived per-query engines (runDaemonSQLQuery), - // and OwnTempDirectory deletes the directory on Close — sharing one - // directory would let a temporary engine remove the live engine's spill - // files. The pid-owned parent is reaped by PrepareDaemonSpillDir once - // this process exits. + return query.NewDuckDBEngine(c.AnalyticsDir(), c.DatabaseDSN(), s.DB(), options) +} + +func daemonDuckDBOptions(c *config.Config) (query.DuckDBOptions, error) { + spillParent, err := query.PrepareDaemonSpillDir(c.HomeDir) + if err != nil { + return query.DuckDBOptions{}, err + } + // Each engine owns its spill directory, so closing a temporary SQL + // engine cannot remove the long-lived analytics engine's spill files. tempDirectory, err := os.MkdirTemp(spillParent, "engine-") if err != nil { - return nil, fmt.Errorf("create engine spill directory: %w", err) - } - // DisableSQLiteScanner keeps DuckDB's bundled SQLite library from - // ATTACHing the live database for the daemon's lifetime, which can - // interfere with the daemon's own go-sqlite3 WAL/lock state. Detail - // queries route through the shared go-sqlite3 connection instead; - // aggregates still read Parquet. - return query.NewDuckDBEngine( - c.AnalyticsDir(), - c.DatabaseDSN(), - s.DB(), - query.DuckDBOptions{ - DisableSQLiteScanner: true, - TempDirectory: tempDirectory, - OwnTempDirectory: true, - MemoryLimit: c.Analytics.QueryMemoryLimit, - Threads: c.Analytics.QueryThreads, - MaxTempDirectorySize: c.Analytics.QueryTempLimit, - }, - ) + return query.DuckDBOptions{}, fmt.Errorf("create engine spill directory: %w", err) + } + // Keep DuckDB's bundled SQLite library away from the live database; + // detail queries use the daemon's shared go-sqlite3 connection. + return query.DuckDBOptions{ + DisableSQLiteScanner: true, + TempDirectory: tempDirectory, + OwnTempDirectory: true, + MemoryLimit: c.Analytics.QueryMemoryLimit, + Threads: c.Analytics.QueryThreads, + MaxTempDirectorySize: c.Analytics.QueryTempLimit, + }, nil } func hasServeOAuthConfig(c *config.Config) bool { @@ -1205,10 +1560,11 @@ func hasServeOAuthConfig(c *config.Config) bool { return c.OAuth.HasAnyConfig() || c.Microsoft.ClientID != "" } -func newDaemonIdleTracker(c *config.Config, stop context.CancelFunc) *api.IdleTracker { +func newDaemonIdleTracker(c *config.Config, stop context.CancelFunc, logger *slog.Logger) *api.IdleTracker { if c == nil || os.Getenv(serveBackgroundChildEnv) != "1" { return nil } + logger = repairLogger(logger) timeout := c.Server.DaemonIdleTimeout if raw := os.Getenv(daemonIdleTimeoutEnv); raw != "" { parsed, err := time.ParseDuration(raw) @@ -1234,9 +1590,16 @@ func newDaemonIdleTracker(c *config.Config, stop context.CancelFunc) *api.IdleTr // Since api.APIMessage, api.StoreStats, etc. are type aliases for store types, // the adapter methods are simple pass-throughs with no conversion needed. type storeAPIAdapter struct { - store *store.Store - draftPolicy []config.IMAPDraftSource - draftClientFactory func(context.Context, *store.Source) (*imaplib.Client, error) + store *store.Store + config *config.Config + options invocationOptions + logger *slog.Logger + draftPolicy []config.IMAPDraftSource + draftClientFactory func(context.Context, *store.Source) (*imaplib.Client, error) + gmailDraftPolicy []config.GmailDraftSource + beeperDraftPolicy []config.GmailDraftSource + gmailDraftClientFactory func(context.Context, *store.Source) (gmail.DraftAPI, error) + calendarClientFactory func(context.Context, config.GCalSource, bool) (gcal.ControlAPI, error) // draftCacheRefresh rebuilds the analytics cache after a draft is durable, // the same best-effort hook the meeting importer uses. draftCacheRefresh func(context.Context, string) error @@ -1245,10 +1608,58 @@ type storeAPIAdapter struct { // analyticsDir is the daemon's Parquet analytics cache directory, used // to read the revision committed by the derived-refresh child. analyticsDir string + cacheJobs *cacheBuildJobs personEnrichmentConfig personenrichment.Config lookupEnv personenrichment.CredentialLookup } +func (a *storeAPIAdapter) invocationContext(ctx context.Context) context.Context { + if a == nil { + return ctx + } + state := newInvocation() + state.cfg = a.config + state.options = a.options + if a.logger != nil { + state.logger = a.logger + } + return withInvocation(ctx, state) +} + +func (a *storeAPIAdapter) GrantPersonMatchConsentContext(ctx context.Context, disclosure personmatch.Disclosure, actor string, mutationGate func(context.Context) (func(), error)) (*store.PersonMatchConsent, bool, error) { + return a.store.GrantPersonMatchConsentContext(ctx, disclosure, actor, mutationGate) +} + +func (a *storeAPIAdapter) RevokePersonMatchConsentContext(ctx context.Context, fingerprint, actor string, mutationGate func(context.Context) (func(), error)) (bool, error) { + return a.store.RevokePersonMatchConsentContext(ctx, fingerprint, actor, mutationGate) +} + +func (a *storeAPIAdapter) HasPersonMatchConsentContext(ctx context.Context, fingerprint string) (bool, error) { + return a.store.HasPersonMatchConsentContext(ctx, fingerprint) +} + +func (a *storeAPIAdapter) PersonMatchConsentEgressContext( + ctx context.Context, fingerprint string, dispatch func() error, +) (bool, error) { + return a.store.PersonMatchConsentEgressContext(ctx, fingerprint, dispatch) +} + +func (a *storeAPIAdapter) EnsurePersonMatchScoringCandidatesContext(ctx context.Context, limit int) (int, error) { + return a.store.EnsurePersonMatchScoringCandidatesContext(ctx, limit) +} + +func (a *storeAPIAdapter) ClaimNextIdentityMatchJudgmentContext(ctx context.Context, owner string, leaseDuration time.Duration, scoringVersion ...string) (*store.IdentityMatchJudgmentLease, error) { + return a.store.ClaimNextIdentityMatchJudgmentContext(ctx, owner, leaseDuration, scoringVersion...) +} + +func (a *storeAPIAdapter) RecordIdentityMatchJudgmentContext(ctx context.Context, lease store.IdentityMatchJudgmentLease, input store.IdentityMatchJudgmentInput) (*store.IdentityMatchJudgment, error) { + return a.store.RecordIdentityMatchJudgmentContext(ctx, lease, input) +} + +func (a *storeAPIAdapter) ListIdentityMatchJudgmentsContext(ctx context.Context, candidateID int64, limit int, beforeID ...int64) ([]store.IdentityMatchJudgment, error) { + return a.store.ListIdentityMatchJudgmentsContext(ctx, candidateID, limit, beforeID...) +} + var _ api.MessageStore = (*storeAPIAdapter)(nil) var _ api.CtxMessageStore = (*storeAPIAdapter)(nil) var _ api.MessageIdentityStore = (*storeAPIAdapter)(nil) @@ -1297,6 +1708,12 @@ var _ api.DocumentSearchStore = (*storeAPIAdapter)(nil) var _ api.DocumentStatusStore = (*storeAPIAdapter)(nil) var _ api.DocumentVectorStatusStore = (*storeAPIAdapter)(nil) var _ api.ActivityStore = (*storeAPIAdapter)(nil) +var _ api.ParticipantIdentityContextStore = (*storeAPIAdapter)(nil) + +// personagenda.IdentityStore backs the live person agenda routes; without the +// forwarding method below the backend starts nil and every agenda endpoint +// answers 503 task_integration_unavailable. +var _ personagenda.IdentityStore = (*storeAPIAdapter)(nil) func (a *storeAPIAdapter) ContactStateContext( ctx context.Context, personID int64, now time.Time, @@ -1316,6 +1733,12 @@ func (a *storeAPIAdapter) PersonDayContext( return a.store.PersonDayContext(ctx, request) } +// ListPersonUIDsContext forwards to the store so the daemon's adapter, not a +// bare *store.Store, satisfies the agenda identity lookup. +func (a *storeAPIAdapter) ListPersonUIDsContext(ctx context.Context, personID int64) ([]string, error) { + return a.store.ListPersonUIDsContext(ctx, personID) +} + func (a *storeAPIAdapter) DayContext( ctx context.Context, request store.DayRequest, ) (*store.DayPage, error) { @@ -1402,6 +1825,7 @@ func (a *storeAPIAdapter) SearchDocuments( ctx context.Context, request store.DocumentSearchRequest, ) (store.DocumentSearchResponse, error) { + ctx = a.invocationContext(ctx) if err := reconcileDocumentOccurrencesForSearch(ctx, a.store); err != nil { return store.DocumentSearchResponse{}, err } @@ -1409,6 +1833,7 @@ func (a *storeAPIAdapter) SearchDocuments( } func (a *storeAPIAdapter) ReconcileDocumentOccurrences(ctx context.Context) error { + ctx = a.invocationContext(ctx) return reconcileDocumentOccurrencesForSearch(ctx, a.store) } @@ -1559,6 +1984,12 @@ func (a *storeAPIAdapter) SearchMessagesQueryContext(ctx context.Context, q *sea return a.store.SearchMessagesQueryContext(ctx, q, offset, limit) } +func (a *storeAPIAdapter) SearchMessageIDsQueryContext( + ctx context.Context, q *search.Query, limit int, +) ([]int64, int64, error) { + return a.store.SearchMessageIDsQueryContext(ctx, q, limit) +} + func (a *storeAPIAdapter) NeedsFTSBackfill() bool { return a.store.NeedsFTSBackfill() } @@ -1598,6 +2029,7 @@ func (a *storeAPIAdapter) BuildCLICache( fullRebuild bool, emit func(api.CLICacheBuildEvent) error, ) error { + ctx = a.invocationContext(ctx) return buildCacheSubprocessStream(ctx, fullRebuild, false, emit) } @@ -1606,6 +2038,7 @@ func (a *storeAPIAdapter) RunCLISync( req api.CLISyncRequest, emit func(api.CLISyncEvent) error, ) error { + ctx = a.invocationContext(ctx) return a.runCLISyncOperationWithRunner(ctx, req, emit, runDaemonCLISubprocessStream) } @@ -1619,15 +2052,17 @@ func (a *storeAPIAdapter) runCLISyncOperationWithRunner( if err == nil && ctx.Err() != nil { err = ctx.Err() } - if req.OperationID == "" { - return err - } - status := "done" - if err != nil { - status = "failed" + if req.OperationID != "" { + status := "done" + if err != nil { + status = "failed" + } + if finishErr := a.store.FinishSyncOperation(req.OperationID, status); finishErr != nil { + return errors.Join(err, fmt.Errorf("finish sync operation: %w", finishErr)) + } } - if finishErr := a.store.FinishSyncOperation(req.OperationID, status); finishErr != nil { - return errors.Join(err, fmt.Errorf("finish sync operation: %w", finishErr)) + if queueErr := a.queueCacheRefreshAfterManualSync(req.BuildCache, req.NoBuildCache); queueErr != nil { + err = errors.Join(err, queueErr) } return err } @@ -1674,6 +2109,12 @@ func emitFolderArgs(args []string, flag string, values []string) []string { func cliSyncSubprocessArgs(req api.CLISyncRequest) []string { if req.Full { args := []string{"sync-full"} + if req.BuildCache { + args = append(args, "--build-cache") + } + if req.NoBuildCache { + args = append(args, "--no-build-cache") + } if req.SourceIDSet { args = append(args, "--source-id", strconv.FormatInt(req.SourceID, 10)) } @@ -1703,6 +2144,12 @@ func cliSyncSubprocessArgs(req api.CLISyncRequest) []string { return args } args := []string{syncIncrementalCmd.Name()} + if req.BuildCache { + args = append(args, "--build-cache") + } + if req.NoBuildCache { + args = append(args, "--no-build-cache") + } if req.SourceIDSet { args = append(args, "--source-id", strconv.FormatInt(req.SourceID, 10)) } @@ -1719,6 +2166,7 @@ func (a *storeAPIAdapter) RunCLIVerify( req api.CLIVerifyRequest, emit func(api.CLIVerifyEvent) error, ) error { + ctx = a.invocationContext(ctx) return runDaemonCLISubprocessStream(ctx, cliVerifySubprocessArgs(req), func(stream, data string) error { if emit == nil { return nil @@ -1746,6 +2194,7 @@ func (a *storeAPIAdapter) RunCLIRepairEncoding( ctx context.Context, emit func(api.CLIRepairEncodingEvent) error, ) error { + ctx = a.invocationContext(ctx) return runDaemonCLISubprocessStream(ctx, []string{"repair-encoding"}, func(stream, data string) error { if emit == nil { return nil @@ -1759,6 +2208,7 @@ func (a *storeAPIAdapter) RunCLIRepairMessage( req api.CLIRepairMessageRequest, emit func(api.CLIRepairMessageEvent) error, ) error { + ctx = a.invocationContext(ctx) return a.runCLIRepairMessageWithRunner(ctx, req, emit, runDaemonCLISubprocessStream) } @@ -1813,6 +2263,7 @@ func (a *storeAPIAdapter) RunCLICommand( req api.CLIRunRequest, emit func(api.CLIRunEvent) error, ) error { + ctx = a.invocationContext(ctx) return a.runCLICommandWithRunner(ctx, req, emit, runDaemonCLISubprocessStreamWithEnv) } @@ -1829,16 +2280,34 @@ func (a *storeAPIAdapter) runCLICommandWithRunner( req api.CLIRunRequest, emit func(api.CLIRunEvent) error, run cliCommandSubprocessRunner, -) error { +) (runErr error) { + if manualSyncCLICommand(req.Args) { + defer func() { + force, skip := manualSyncCacheFlagValues(req.Args) + runErr = errors.Join(runErr, a.queueCacheRefreshAfterManualSync(force, skip)) + }() + } emitSubprocess := func(stream, data string) error { if emit == nil { return nil } return emit(api.CLIRunEvent{Type: stream, Data: data}) } - if api.IsCLIRunDraftReply(req.Args) { + if api.IsCLIRunDraftCreate(req.Args) { + if api.IsCLIRunDraftForward(req.Args) { + return a.runCLIForwardDraft(ctx, req, emit) + } + if api.IsCLIRunDraftCompose(req.Args) { + return a.runCLIComposeDraft(ctx, req, emit) + } return a.runCLIReplyDraft(ctx, req, emit) } + if api.IsCLIRunDraftSendAs(req.Args) { + return a.runCLIDraftSendAs(ctx, req, emit) + } + if api.IsCLIRunDraftLifecycle(req.Args) { + return a.runCLIDraftLifecycle(ctx, req, emit) + } runSubprocess := func(ctx context.Context) error { args := req.Args if req.GrantDecided { @@ -1945,6 +2414,7 @@ func (a *storeAPIAdapter) PlanCLIAddCalendar( ctx context.Context, req api.CLIAddCalendarPlanRequest, ) (api.CLIAddCalendarPlanResponse, error) { + ctx = a.invocationContext(ctx) return planCLIAddCalendar(ctx, a.store, req) } @@ -1952,6 +2422,7 @@ func (a *storeAPIAdapter) PlanCLIEmbeddings( ctx context.Context, req api.CLIEmbeddingsPlanRequest, ) (api.CLIEmbeddingsPlanResponse, error) { + ctx = a.invocationContext(ctx) return planCLIEmbeddings(ctx, req) } @@ -1959,11 +2430,15 @@ func (a *storeAPIAdapter) PlanCLIDeleteStaged( ctx context.Context, req api.CLIDeleteStagedPlanRequest, ) (api.CLIDeleteStagedPlanResponse, error) { + ctx = a.invocationContext(ctx) return planCLIDeleteStaged(ctx, a.store, req) } func (a *storeAPIAdapter) deletionManager() (*deletion.Manager, error) { - mgr, err := deletion.NewManager(filepath.Join(cfg.Data.DataDir, "deletions")) + if a.config == nil { + return nil, errors.New("configuration is unavailable") + } + mgr, err := deletion.NewManager(filepath.Join(a.config.Data.DataDir, "deletions")) if err != nil { return nil, fmt.Errorf("create deletion manager: %w", err) } @@ -2017,6 +2492,7 @@ func (a *storeAPIAdapter) PlanCLIDeduplicate( ctx context.Context, req api.CLIDeduplicatePlanRequest, ) (api.CLIDeduplicatePlanResponse, error) { + ctx = a.invocationContext(ctx) return planCLIDeduplicate(ctx, a.store, req) } @@ -2077,6 +2553,12 @@ func (a *storeAPIAdapter) BackupDatabaseContext(ctx context.Context, dst string) return a.store.BackupDatabaseContext(ctx, dst) } +// CountMessagesBySourceContext counts every source's messages in one pass +// for account listings. +func (a *storeAPIAdapter) CountMessagesBySourceContext(ctx context.Context) (map[int64]store.SourceMessageCounts, error) { + return a.store.CountMessagesBySourceContext(ctx) +} + func (a *storeAPIAdapter) CountMessagesForSource(sourceID int64) (int64, error) { return a.store.CountMessagesForSource(sourceID) } @@ -2329,35 +2811,69 @@ func (a *storeAPIAdapter) ListIdentityMatchCandidatesContext( return a.store.ListIdentityMatchCandidatesContext(ctx, states, limit, offset) } -func (a *storeAPIAdapter) GetIdentityMatchCandidateContext( +func (a *storeAPIAdapter) ListIdentityMatchReviewsContext( + ctx context.Context, states []store.IdentityMatchState, limit, offset int, +) ([]store.IdentityMatchCandidate, error) { + return a.store.ListIdentityMatchReviewsContext(ctx, states, limit, offset) +} + +func (a *storeAPIAdapter) GetIdentityMatchReviewContext( ctx context.Context, candidateID int64, ) (*store.IdentityMatchCandidate, error) { - return a.store.GetIdentityMatchCandidateContext(ctx, candidateID) + return a.store.GetIdentityMatchReviewContext(ctx, candidateID) } -func (a *storeAPIAdapter) AcceptIdentityMatchCandidateContext( - ctx context.Context, candidateID int64, decidedBy string, notes *string, +func (a *storeAPIAdapter) DecideIdentityMatchReviewedContext( + ctx context.Context, candidateID int64, token string, + decision store.IdentityMatchState, notes *string, ) (*store.IdentityMatchCandidate, int64, error) { - return a.store.AcceptIdentityMatchCandidateContext(ctx, candidateID, decidedBy, notes) + return a.store.DecideIdentityMatchReviewedContext(ctx, candidateID, token, decision, notes) } -func (a *storeAPIAdapter) DecideIdentityMatchCandidateContext( - ctx context.Context, candidateID int64, state store.IdentityMatchState, - decidedBy string, notes *string, +func (a *storeAPIAdapter) GetIdentityMatchCandidateContext( + ctx context.Context, candidateID int64, ) (*store.IdentityMatchCandidate, error) { - return a.store.DecideIdentityMatchCandidateContext(ctx, candidateID, state, decidedBy, notes) + return a.store.GetIdentityMatchCandidateContext(ctx, candidateID) } -func (a *storeAPIAdapter) CreatePersonFromParticipantContext( - ctx context.Context, participantID int64, +func (a *storeAPIAdapter) CreatePersonFromParticipantWithDisplayNameContext( + ctx context.Context, participantID int64, displayName *string, ) (*store.Person, bool, error) { - return a.store.CreatePersonFromParticipantContext(ctx, participantID) + return a.store.CreatePersonFromParticipantWithDisplayNameContext(ctx, participantID, displayName) } func (a *storeAPIAdapter) GetPersonContext(ctx context.Context, id int64) (*store.Person, error) { return a.store.GetPersonContext(ctx, id) } +func (a *storeAPIAdapter) EnsurePersonInferenceProfile(ctx context.Context, profile peoplesweep.ProviderProfile) (bool, error) { + return a.store.EnsurePersonInferenceProfile(ctx, profile) +} + +func (a *storeAPIAdapter) RecordPersonInferenceCheck(ctx context.Context, check store.PersonInferenceCheck) error { + return a.store.RecordPersonInferenceCheck(ctx, check) +} + +func (a *storeAPIAdapter) HasSuccessfulPersonInferenceCheck(ctx context.Context, fingerprint string) (bool, error) { + return a.store.HasSuccessfulPersonInferenceCheck(ctx, fingerprint) +} + +func (a *storeAPIAdapter) InvalidatePersonInferenceCheck(ctx context.Context, fingerprint string) (bool, error) { + return a.store.InvalidatePersonInferenceCheck(ctx, fingerprint) +} + +func (a *storeAPIAdapter) GrantPersonInferenceConsent(ctx context.Context, fingerprint, actor string) (*store.PersonInferenceConsent, bool, error) { + return a.store.GrantPersonInferenceConsent(ctx, fingerprint, actor) +} + +func (a *storeAPIAdapter) HasActivePersonInferenceConsent(ctx context.Context, fingerprint string) (bool, error) { + return a.store.HasActivePersonInferenceConsent(ctx, fingerprint) +} + +func (a *storeAPIAdapter) RevokePersonInferenceConsent(ctx context.Context, fingerprint, actor string) (bool, error) { + return a.store.RevokePersonInferenceConsent(ctx, fingerprint, actor) +} + func (a *storeAPIAdapter) GetPersonTrackingContext( ctx context.Context, id int64, ) (*store.PersonTracking, error) { @@ -2990,6 +3506,12 @@ func (a *storeAPIAdapter) ClusterEdges(id int64) ([]store.LinkEdge, error) { return a.store.ClusterEdges(id) } +func (a *storeAPIAdapter) GetParticipantIdentityContext( + ctx context.Context, participantIDs []int64, +) (*store.ParticipantIdentityContext, error) { + return a.store.GetParticipantIdentityContext(ctx, participantIDs) +} + // RefreshIdentityDatasets rebuilds identity-derived Parquet in a short-lived, // resource-bounded child. The child owns the cache lock and its DuckDB // allocator exits with the process; the long-lived daemon does neither. @@ -3004,12 +3526,12 @@ func (a *storeAPIAdapter) RefreshIdentityDatasets(ctx context.Context) (int64, e return state.IdentityRevision, nil } -func (a *storeAPIAdapter) GetActiveSync(sourceID int64) (*store.SyncRun, error) { - return a.store.GetActiveSync(sourceID) +func (a *storeAPIAdapter) GetActiveSyncReadOnly(ctx context.Context, sourceID int64) (*store.SyncRun, error) { + return a.store.GetActiveSyncReadOnly(ctx, sourceID) } -func (a *storeAPIAdapter) GetLatestSync(sourceID int64) (*store.SyncRun, error) { - return a.store.GetLatestSync(sourceID) +func (a *storeAPIAdapter) GetLatestSyncContext(ctx context.Context, sourceID int64) (*store.SyncRun, error) { + return a.store.GetLatestSyncContext(ctx, sourceID) } func (a *storeAPIAdapter) GetSyncOperation(operationID string) (*store.SyncOperation, error) { @@ -3020,16 +3542,16 @@ func (a *storeAPIAdapter) CreateSyncOperation(sourceID int64, operationID string return a.store.CreateSyncOperation(sourceID, operationID) } -func (a *storeAPIAdapter) GetLastSuccessfulSync(sourceID int64) (*store.SyncRun, error) { - return a.store.GetLastSuccessfulSync(sourceID) +func (a *storeAPIAdapter) GetLastSuccessfulSyncContext(ctx context.Context, sourceID int64) (*store.SyncRun, error) { + return a.store.GetLastSuccessfulSyncContext(ctx, sourceID) } -func (a *storeAPIAdapter) CountSyncRunItems(syncRunID int64, status string) (int64, error) { - return a.store.CountSyncRunItems(syncRunID, status) +func (a *storeAPIAdapter) CountSyncRunItemsContext(ctx context.Context, syncRunID int64, status string) (int64, error) { + return a.store.CountSyncRunItemsContext(ctx, syncRunID, status) } -func (a *storeAPIAdapter) ListSyncRunItems(syncRunID int64, status string, limit int) ([]store.SyncRunItem, error) { - return a.store.ListSyncRunItems(syncRunID, status, limit) +func (a *storeAPIAdapter) ListSyncRunItemsContext(ctx context.Context, syncRunID int64, status string, limit int) ([]store.SyncRunItem, error) { + return a.store.ListSyncRunItemsContext(ctx, syncRunID, status, limit) } const personEnrichmentJob = "person-enrichment" @@ -3191,6 +3713,7 @@ func registerPersonEnrichmentJob( } factories := make(map[string]personenrichment.ProviderFactory) providerConfigs := make(map[string]personenrichment.ProviderConfig) + providerFingerprints := make(map[string]string) activeFingerprints := make([]string, 0, len(enrichmentConfig.Providers)) for _, configured := range enrichmentConfig.Providers { provider := configured @@ -3206,6 +3729,7 @@ func registerPersonEnrichmentJob( } activeFingerprints = append(activeFingerprints, profile.Fingerprint) providerConfigs[provider.Name] = provider + providerFingerprints[provider.Name] = profile.Fingerprint switch provider.Kind { case personenrichment.ProviderExa: factories[provider.Name] = func(config personenrichment.ProviderConfig, credential string) (personenrichment.Provider, error) { @@ -3232,8 +3756,9 @@ func registerPersonEnrichmentJob( worker, err := personenrichment.NewWorker(st, st, *gate, factories, personenrichment.WorkerOptions{ Owner: "daemon-person-enrichment", LeaseDuration: enrichmentConfig.LeaseDuration, RenewEvery: enrichmentConfig.LeaseDuration / 4, Clock: time.Now, - Jitter: func(delay time.Duration) time.Duration { return delay }, - ProviderConfigs: providerConfigs, + Jitter: func(delay time.Duration) time.Duration { return delay }, + ProviderConfigs: providerConfigs, + ProviderFingerprints: providerFingerprints, }) if err != nil { return fmt.Errorf("configure person enrichment worker: %w", err) @@ -3252,9 +3777,18 @@ func registerPersonEnrichmentJob( // schedulerAdapter adapts scheduler.Scheduler to api.SyncScheduler. // Since api.AccountStatus is a type alias for scheduler.AccountStatus, -// the adapter methods are simple pass-throughs. +// the adapter methods are simple pass-throughs. Jobs on the media scheduler +// join the same job status and trigger by name. type schedulerAdapter struct { scheduler *scheduler.Scheduler + media *scheduler.Scheduler +} + +func (a *schedulerAdapter) jobScheduler(name string) *scheduler.Scheduler { + if a.media != nil && a.media.IsJobScheduled(name) { + return a.media + } + return a.scheduler } func (a *schedulerAdapter) IsScheduled(email string) bool { @@ -3278,19 +3812,24 @@ func (a *schedulerAdapter) Status() []api.AccountStatus { } func (a *schedulerAdapter) JobStatus() []api.JobStatus { - return a.scheduler.JobStatus() + jobs := a.scheduler.JobStatus() + if a.media != nil { + jobs = append(jobs, a.media.JobStatus()...) + slices.SortFunc(jobs, func(x, y api.JobStatus) int { return strings.Compare(x.Name, y.Name) }) + } + return jobs } func (a *schedulerAdapter) IsJobScheduled(name string) bool { - return a.scheduler.IsJobScheduled(name) + return a.jobScheduler(name).IsJobScheduled(name) } func (a *schedulerAdapter) TriggerJob(name string) error { - return a.scheduler.TriggerJob(name) + return a.jobScheduler(name).TriggerJob(name) } func (a *schedulerAdapter) StartJob(name string) error { - return a.scheduler.StartJob(name) + return a.jobScheduler(name).StartJob(name) } // runScheduledSync performs a sync for a scheduled account. It resolves @@ -3311,7 +3850,15 @@ func (a *schedulerAdapter) StartJob(name string) error { // this is the email address, for IMAP it's the full // `imaps://user@host:port` URL recorded by `add-imap`, for Teams it is // the UPN/email recorded by `add-o365`. -func runScheduledSync(ctx context.Context, identifier string, s *store.Store, getOAuthMgr func(string) (*oauth.Manager, error)) error { +func runScheduledSync(ctx context.Context, identifier string, s *store.Store, getOAuthMgr func(string) (*oauth.Manager, error), state *invocation) error { + if state == nil { + state = invocationFromContext(ctx) + } + if state == nil || state.cfg == nil || state.logger == nil { + return errors.New("configuration is unavailable") + } + logger := state.logger + ctx = withInvocation(ctx, state) logger.Info("starting scheduled sync", "identifier", identifier) srcs, srcErr := findScheduledSyncSources(s, identifier) @@ -3324,7 +3871,10 @@ func runScheduledSync(ctx context.Context, identifier string, s *store.Store, ge // row exists). if len(srcs) == 0 { startTime := time.Now() - summary, syncErr := runScheduledGmailSync(ctx, identifier, nil, s, getOAuthMgr) + summary, syncErr := runScheduledGmailSync(ctx, identifier, nil, s, getOAuthMgr, state) + if scheduledSyncYielded(ctx) { + return scheduledSyncYieldResult(ctx, syncErr) + } if syncErr == nil { logger.Info("sync completed", "identifier", identifier, @@ -3350,20 +3900,28 @@ func runScheduledSync(ctx context.Context, identifier string, s *store.Store, ge ) switch sourceType { case sourceTypeGmail: - summary, err = runScheduledGmailSync(ctx, identifier, src, s, getOAuthMgr) + summary, err = runScheduledGmailSync(ctx, identifier, src, s, getOAuthMgr, state) case sourceTypeIMAP: - summary, err = runScheduledIMAPSync(ctx, src, s) + summary, err = runScheduledIMAPSync(ctx, src, s, state) case sourceTypeTeams: - err = runScheduledTeamsSync(ctx, src, s) + err = runScheduledTeamsSync(ctx, src, s, state) + case sourceTypeMSMail: + err = runScheduledMSMailSync(ctx, src, s, state) case sourceTypeDiscord: var discordSummary *discord.ImportSummary discordSummary, err = importDiscordSourceForScheduledRun( - ctx, s, src, defaultDiscordCommandDeps(), false, time.Time{}, nil, + ctx, s, src, defaultDiscordCommandDeps().bind(ctx), false, time.Time{}, nil, ) - logScheduledDiscordIssues(identifier, discordSummary) + logScheduledDiscordIssues(identifier, discordSummary, logger) default: err = fmt.Errorf("source %q has type %q which is not supported by the daemon scheduler", identifier, sourceType) } + if scheduledSyncYielded(ctx) { + if err != nil { + errs = append(errs, fmt.Errorf("%s (%s): %w", identifier, sourceType, err)) + } + return scheduledSyncYieldResult(ctx, errs...) + } if err != nil { errs = append(errs, fmt.Errorf("%s (%s): %w", identifier, sourceType, err)) continue @@ -3393,7 +3951,19 @@ func runScheduledSync(ctx context.Context, identifier string, s *store.Store, ge return errors.Join(errs...) } -func logScheduledDiscordIssues(identifier string, summary *discord.ImportSummary) { +func scheduledSyncYielded(ctx context.Context) bool { + return jobctx.YieldedToWaiter(ctx) || jobctx.PreemptionRequested(ctx) +} + +func scheduledSyncYieldResult(ctx context.Context, errs ...error) error { + if jobctx.YieldedToWaiter(ctx) { + errs = append(errs, scheduler.ErrYieldedToWaiter) + } + return errors.Join(errs...) +} + +func logScheduledDiscordIssues(identifier string, summary *discord.ImportSummary, logger *slog.Logger) { + logger = repairLogger(logger) if summary == nil { return } @@ -3421,7 +3991,7 @@ func logScheduledDiscordIssues(identifier string, summary *discord.ImportSummary // findScheduledSyncSources resolves ALL syncable source rows for a // scheduler identifier. Returns at most one row per syncable type -// (gmail, imap, teams, discord), in that stable order. Non-syncable types +// (gmail, imap, msmail, teams, discord), in that stable order. Non-syncable types // (mbox, apple-mail, etc.) are skipped. // // Returns an empty slice (not nil) when no syncable source matches — @@ -3438,7 +4008,7 @@ func findScheduledSyncSources(s *store.Store, identifier string) ([]*store.Sourc seen := make(map[string]*store.Source, 4) for _, src := range rows { switch src.SourceType { - case sourceTypeGmail, sourceTypeIMAP, sourceTypeTeams: + case sourceTypeGmail, sourceTypeIMAP, sourceTypeMSMail, sourceTypeTeams: if _, dup := seen[src.SourceType]; !dup { seen[src.SourceType] = src } @@ -3451,9 +4021,9 @@ func findScheduledSyncSources(s *store.Store, identifier string) ([]*store.Sourc } } - // Return in stable order: gmail, imap, teams, discord. + // Return in stable order: gmail, imap, msmail, teams, discord. var result []*store.Source - for _, t := range []string{sourceTypeGmail, sourceTypeIMAP, sourceTypeTeams, sourceTypeDiscord} { + for _, t := range []string{sourceTypeGmail, sourceTypeIMAP, sourceTypeMSMail, sourceTypeTeams, sourceTypeDiscord} { if src, ok := seen[t]; ok { result = append(result, src) } @@ -3461,12 +4031,41 @@ func findScheduledSyncSources(s *store.Store, identifier string) ([]*store.Sourc return result, nil } -// runScheduledGmailSync runs an incremental Gmail sync for the daemon. -// Token-source lookup uses oauthMgr.TokenSource directly (not -// getTokenSourceWithReauth) because serve runs as a daemon and cannot -// open a browser for OAuth — the error path tells the user how to -// re-authorize from a terminal. -func runScheduledGmailSync(ctx context.Context, email string, src *store.Source, s *store.Store, getOAuthMgr func(string) (*oauth.Manager, error)) (*gmail.SyncSummary, error) { +// scheduledSyncPreemptible disables scheduler-job preemption when an account +// dispatch includes a full IMAP pass, whose offset-based paging cannot resume +// after the syncer stops. An unavailable source lookup also takes the safe +// path and lets the current pass finish. +func scheduledSyncPreemptible(s *store.Store, identifier string, logger *slog.Logger) bool { + sources, err := findScheduledSyncSources(s, identifier) + if err != nil { + logger.Warn("could not determine scheduled sync preemption safety; allowing current pass to finish", + "identifier", identifier, + "error", err, + ) + return false + } + for _, source := range sources { + if source.SourceType == sourceTypeIMAP { + return false + } + } + return true +} + +// newDaemonGmailClient reuses source-bound credentials without interactive reauth. +// serviceAccount reports which credentials the returned client uses. +func newDaemonGmailClient( + ctx context.Context, email string, src *store.Source, + getOAuthMgr func(string) (*oauth.Manager, error), state *invocation, +) (client gmail.API, serviceAccount bool, err error) { + if state == nil { + state = invocationFromContext(ctx) + } + if state == nil || state.cfg == nil || state.logger == nil { + return nil, false, errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger appName := "" if src != nil { appName = sourceOAuthApp(src) @@ -3475,19 +4074,29 @@ func runScheduledGmailSync(ctx context.Context, email string, src *store.Source, var tokenSource oauth2.TokenSource var tsErr error - if saKeyPath := cfg.OAuth.ServiceAccountKeyFor(appName); saKeyPath != "" { + saKeyPath := cfg.OAuth.ServiceAccountKeyFor(appName) + if saKeyPath != "" { saMgr, saErr := oauth.NewServiceAccountManager(saKeyPath, oauth.Scopes) if saErr != nil { - return nil, fmt.Errorf("service account for %s: %w", email, saErr) + return nil, false, provideridentity.NewGmailCredentialError( + provideridentity.GmailServiceAccountConfiguration, + fmt.Errorf("service account for %s: %w", email, saErr), + ) } tokenSource, tsErr = saMgr.TokenSource(ctx, email) if tsErr != nil { - return nil, fmt.Errorf("service account token for %s: %w", email, tsErr) + return nil, false, provideridentity.NewGmailCredentialError( + provideridentity.GmailServiceAccountConfiguration, + fmt.Errorf("service account token for %s: %w", email, tsErr), + ) } } else { oauthMgr, oaErr := getOAuthMgr(appName) if oaErr != nil { - return nil, fmt.Errorf("resolve OAuth credentials for %s: %w", email, oaErr) + return nil, false, provideridentity.NewGmailCredentialError( + provideridentity.GmailOAuthConfiguration, + fmt.Errorf("resolve OAuth credentials for %s: %w", email, oaErr), + ) } tokenSource, tsErr = oauthMgr.TokenSource(ctx, email) if tsErr != nil { @@ -3496,26 +4105,57 @@ func runScheduledGmailSync(ctx context.Context, email string, src *store.Source, // auth errors. Suggesting reauth on every network blip sends // the user down the wrong path. if syncerr.IsTransientNetwork(tsErr) { - return nil, fmt.Errorf("get token source: %w (transient network error; will retry on next schedule)", tsErr) + return nil, false, fmt.Errorf( + "get token source: %w (transient network error; will retry on next schedule)", tsErr, + ) } if oauthMgr.HasToken(email) { - return nil, fmt.Errorf("get token source: %w (token may be expired; %s)", tsErr, gmailReauthHint(email, accountIsNarrowed(oauthMgr, email))) + return nil, false, provideridentity.ClassifyGmailProfileError(fmt.Errorf( + "get token source: %w (token may be expired; %s)", + tsErr, gmailReauthHint(email, accountIsNarrowed(oauthMgr, email)), + ), false) } - return nil, fmt.Errorf("get token source: %w (run 'msgvault add-account %s' first)", tsErr, email) + missing := fmt.Errorf("get token source: %w (run 'msgvault add-account %s' first)", tsErr, email) + if errors.Is(tsErr, os.ErrNotExist) { + return nil, false, provideridentity.NewGmailCredentialError( + provideridentity.GmailTokenMissing, missing, + ) + } + return nil, false, missing } } rateLimiter := gmail.NewRateLimiter(float64(cfg.Sync.RateLimitQPS)) - client := gmail.NewClient(tokenSource, + return gmail.NewClient(tokenSource, gmail.WithLogger(logger), gmail.WithRateLimiter(rateLimiter), - ) + ), saKeyPath != "", nil +} + +// runScheduledGmailSync runs an incremental Gmail sync for the daemon. +// Token-source lookup uses oauthMgr.TokenSource directly (not +// getTokenSourceWithReauth) because serve runs as a daemon and cannot +// open a browser for OAuth — the error path tells the user how to +// re-authorize from a terminal. +func runScheduledGmailSync(ctx context.Context, email string, src *store.Source, s *store.Store, getOAuthMgr func(string) (*oauth.Manager, error), state *invocation) (*gmail.SyncSummary, error) { + if state == nil { + state = invocationFromContext(ctx) + } + if state == nil || state.cfg == nil || state.logger == nil { + return nil, errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger + client, _, err := newDaemonGmailClient(ctx, email, src, getOAuthMgr, state) + if err != nil { + return nil, err + } defer func() { _ = client.Close() }() opts := sync.DefaultOptions() opts.AttachmentsDir = cfg.AttachmentsDir() - syncer := newMessageSyncer(client, s, opts).WithLogger(logger) + syncer := newMessageSyncer(client, s, opts, state).WithLogger(logger) source, err := s.GetOrCreateSource(sourceTypeGmail, email) if err != nil { @@ -3525,8 +4165,8 @@ func runScheduledGmailSync(ctx context.Context, email string, src *store.Source, // — see comment in account_identity.go. serve is a daemon, so the // confirmation message has no terminal; discard it. Helper logs any // failure path through its own logger.Warn. - confirmDefaultIdentity(io.Discard, s, source.ID, email, email, "account-identifier") - if err := runPostSourceCreateMigrations(s); err != nil { + confirmDefaultIdentity(io.Discard, s, source.ID, email, email, "account-identifier", logger) + if err := runPostSourceCreateMigrationsForInvocation(s, state); err != nil { return nil, fmt.Errorf("post-source-create migrations: %w", err) } @@ -3549,8 +4189,16 @@ func runScheduledGmailSync(ctx context.Context, email string, src *store.Source, // and relying on the store to dedupe by message-id. NoResume is forced // on because IMAP page tokens are numeric offsets that don't survive // across processes (see syncfull.go). -func runScheduledIMAPSync(ctx context.Context, src *store.Source, s *store.Store) (*gmail.SyncSummary, error) { - imapOpts := imapFolderStateOptions(s, src, false) +func runScheduledIMAPSync(ctx context.Context, src *store.Source, s *store.Store, state *invocation) (*gmail.SyncSummary, error) { + if state == nil { + state = invocationFromContext(ctx) + } + if state == nil || state.cfg == nil || state.logger == nil { + return nil, errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger + imapOpts := imapFolderStateOptions(s, src, false, cfg, logger) apiClient, err := buildAPIClient(ctx, src, nil, nil, imapOpts...) if err != nil { return nil, fmt.Errorf("build IMAP client: %w", err) @@ -3562,7 +4210,7 @@ func runScheduledIMAPSync(ctx context.Context, src *store.Source, s *store.Store opts.AttachmentsDir = cfg.AttachmentsDir() opts.NoResume = true - syncer := newMessageSyncer(apiClient, s, opts).WithLogger(logger) + syncer := newMessageSyncer(apiClient, s, opts, state).WithLogger(logger) // runPostSourceCreateMigrations is keyed off Gmail-only legacy // state, so it's a no-op for fresh IMAP installs; we still call it @@ -3577,8 +4225,8 @@ func runScheduledIMAPSync(ctx context.Context, src *store.Source, s *store.Store // row with NULL display_name skips the write rather than re-injecting // the URL. displayName := src.DisplayName.String - confirmDefaultIdentity(io.Discard, s, src.ID, displayName, displayName, "account-identifier") - if err := runPostSourceCreateMigrations(s); err != nil { + confirmDefaultIdentity(io.Discard, s, src.ID, displayName, displayName, "account-identifier", logger) + if err := runPostSourceCreateMigrationsForInvocation(s, state); err != nil { return nil, fmt.Errorf("post-source-create migrations: %w", err) } @@ -3599,7 +4247,15 @@ func runScheduledIMAPSync(ctx context.Context, src *store.Source, s *store.Store } // runScheduledTeamsSync runs a Teams sync for the daemon. -func runScheduledTeamsSync(ctx context.Context, src *store.Source, s *store.Store) error { +func runScheduledTeamsSync(ctx context.Context, src *store.Source, s *store.Store, state *invocation) error { + if state == nil { + state = invocationFromContext(ctx) + } + if state == nil || state.cfg == nil || state.logger == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger email := src.Identifier // Seed the default identity and converge legacy migrations before @@ -3609,27 +4265,21 @@ func runScheduledTeamsSync(ctx context.Context, src *store.Source, s *store.Stor // "me" identity. Auto-default-identity must run BEFORE the legacy // migration retry (see account_identity.go); serve is a daemon, so // the confirmation message has no terminal and is discarded. - confirmDefaultIdentity(io.Discard, s, src.ID, email, email, "account-identifier") - if err := runPostSourceCreateMigrations(s); err != nil { + confirmDefaultIdentity(io.Discard, s, src.ID, email, email, "account-identifier", logger) + if err := runPostSourceCreateMigrationsForInvocation(s, state); err != nil { return fmt.Errorf("post-source-create migrations: %w", err) } - mgr := microsoft.NewGraphManager(cfg.Microsoft.ClientID, cfg.Microsoft.EffectiveTenantID(), cfg.Microsoft.EffectiveRedirectURI(), cfg.TokensDir(), logger) - tokenFn, err := mgr.TokenSource(ctx, email) + client, err := newTeamsClient(ctx, cfg, logger, email) if err != nil { return err } - qps := float64(cfg.Sync.RateLimitQPS) - if qps <= 0 { - qps = 5 - } - client := teams.NewClient("https://graph.microsoft.com/v1.0", teams.TokenFunc(tokenFn), qps) - opts := scheduledTeamsImportOptions(email) + opts := scheduledTeamsImportOptions(email, cfg) _, err = teams.NewImporter(s, client).Import(ctx, opts) return err } -func scheduledTeamsImportOptions(email string) teams.ImportOptions { +func scheduledTeamsImportOptions(email string, cfg *config.Config) teams.ImportOptions { return teams.ImportOptions{ Email: email, AttachmentsDir: cfg.AttachmentsDir(), diff --git a/cmd/msgvault/cmd/serve_analytics.go b/cmd/msgvault/cmd/serve_analytics.go index a7c9b0923..c029020f6 100644 --- a/cmd/msgvault/cmd/serve_analytics.go +++ b/cmd/msgvault/cmd/serve_analytics.go @@ -147,6 +147,7 @@ func prepareDaemonAnalyticsEngine( return nil, "", startupCacheBuildOutcomeNone, false, errors.New("daemon analytics engine unavailable") } + logger := loggerFromContext(ctx) engineMode := c.Analytics.Engine if engineMode == "" { @@ -202,6 +203,7 @@ func startDaemonAnalyticsInitializer( h := newDaemonAnalyticsInitHandle() apiServer.SetAnalyticsInitializationActive(true) go func() { + logger := loggerFromContext(ctx) defer close(h.done) defer apiServer.SetAnalyticsInitializationActive(false) if tracker != nil { diff --git a/cmd/msgvault/cmd/serve_analytics_test.go b/cmd/msgvault/cmd/serve_analytics_test.go index d35932fb8..572a9e3f2 100644 --- a/cmd/msgvault/cmd/serve_analytics_test.go +++ b/cmd/msgvault/cmd/serve_analytics_test.go @@ -42,6 +42,8 @@ func TestPrepareDaemonAnalyticsEngineAutoStartsWithSQLFallback(t *testing.T) { } func TestRunServeAllowsDeletionIDsWhileAnalyticsBuildBlocked(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) oldCfg := cfg @@ -64,7 +66,7 @@ func TestRunServeAllowsDeletionIDsWhileAnalyticsBuildBlocked(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) cmd := &cobra.Command{Use: serveCmd.Use} - cmd.SetContext(ctx) + cmd.SetContext(testInvocationContext(ctx, cfg, invocationOptions{})) errCh := make(chan error, 1) go func() { errCh <- runServe(cmd, nil) }() diff --git a/cmd/msgvault/cmd/serve_api_wiring_test.go b/cmd/msgvault/cmd/serve_api_wiring_test.go index ccac0206c..f914a6bcf 100644 --- a/cmd/msgvault/cmd/serve_api_wiring_test.go +++ b/cmd/msgvault/cmd/serve_api_wiring_test.go @@ -113,6 +113,61 @@ func TestStoreAPIAdapterExposesFileMetadataCatalog(t *testing.T) { assertions.Empty(files) } +func TestStoreAPIAdapterServesPersonMatchScoringRun(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + fixture := storetest.New(t) + st := fixture.Store + cfg := config.NewDefaultConfig() + cfg.People.IdentityScoring.Enabled = true + cfg.People.IdentityScoring.CredentialEnv = "MSGVAULT_JEV_DAEMON_WIRING_FIXTURE_KEY" + cfg.People.IdentityScoring.RetentionDeclaration = "fixture retention declaration" + t.Setenv(cfg.People.IdentityScoring.CredentialEnv, "fixture-key") + + srv := api.NewServerWithOptions(api.ServerOptions{ + Config: cfg, + Store: &storeAPIAdapter{store: st}, + Logger: slog.New(slog.DiscardHandler), + }) + disclosure, err := cfg.People.IdentityScoring.Disclosure() + require.NoError(err) + fingerprint, err := disclosure.Fingerprint() + require.NoError(err) + consentRequest := httptest.NewRequest(http.MethodPost, "/api/v1/identity/scoring/consent", + strings.NewReader(fmt.Sprintf(`{"disclosure_fingerprint":%q}`, fingerprint))) + consentRequest.Header.Set("Content-Type", "application/json") + consentResponse := httptest.NewRecorder() + srv.Router().ServeHTTP(consentResponse, consentRequest) + require.Equal(http.StatusOK, consentResponse.Code, consentResponse.Body.String()) + + runRequest := httptest.NewRequest(http.MethodPost, "/api/v1/identity/scoring/run", + strings.NewReader(`{"limit":1}`)) + runRequest.Header.Set("Content-Type", "application/json") + runResponse := httptest.NewRecorder() + srv.Router().ServeHTTP(runResponse, runRequest) + require.Equal(http.StatusOK, runResponse.Code, runResponse.Body.String()) + var result api.PersonMatchScoringResponse + require.NoError(json.Unmarshal(runResponse.Body.Bytes(), &result)) + assert.Nil(result.Error) + assert.Zero(result.Processed) + assert.Empty(result.Results) + + providerDispatched := false + egressStore, ok := any(&storeAPIAdapter{store: st}).(interface { + PersonMatchConsentEgressContext(ctx context.Context, fingerprint string, dispatch func() error) (bool, error) + }) + require.True(ok) + consented, err := egressStore.PersonMatchConsentEgressContext( + t.Context(), fingerprint, func() error { + providerDispatched = true + return nil + }, + ) + require.NoError(err) + assert.True(consented) + assert.True(providerDispatched) +} + func TestStoreAPIAdapterExposesCuratedPeopleCompletion(t *testing.T) { require := require.New(t) assert := assert.New(t) diff --git a/cmd/msgvault/cmd/serve_archive_sql_test.go b/cmd/msgvault/cmd/serve_archive_sql_test.go new file mode 100644 index 000000000..4904b40c8 --- /dev/null +++ b/cmd/msgvault/cmd/serve_archive_sql_test.go @@ -0,0 +1,53 @@ +package cmd + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDaemonArchiveSQLUsesRestrictedEngine(t *testing.T) { + assertions := assert.New(t) + requirements := require.New(t) + c, s := openTestDaemonAnalyticsStore(t) + c.Analytics.AutoBuildCache = false + _, err := s.DB().Exec(` + INSERT INTO sources (id, source_type, identifier) VALUES (1, 'gmail', 'user@example.com'); + INSERT INTO conversations (id, source_id, source_conversation_id, conversation_type) + VALUES (1, 1, 'thread-1', 'email_thread'); + INSERT INTO messages (id, source_id, source_message_id, conversation_id, message_type, sent_at) + VALUES (1, 1, 'message-1', 1, 'email', '2024-01-01 00:00:00'); + `) + requirements.NoError(err) + _, err = buildCache(c.DatabaseDSN(), c.AnalyticsDir(), true) + requirements.NoError(err) + ownerEngine, err := openDaemonDuckDBEngine(c, s) + requirements.NoError(err) + t.Cleanup(func() { requirements.NoError(ownerEngine.Close()) }) + jobs := newCacheBuildJobs(t.Context(), nil, func(context.Context, buildCacheMode) error { return nil }) + + result, accepted, err := runDaemonSQLQueryWithJobs(t.Context(), c, s, ownerEngine, + "SELECT COUNT(*) FROM messages", daemonSQLQueryOptions{archiveOnly: true}, jobs) + requirements.NoError(err) + assertions.Nil(accepted) + requirements.Len(result.Rows, 1) + assertions.EqualValues(1, result.Rows[0][0]) + + outside := filepath.Join(t.TempDir(), "outside.txt") + requirements.NoError(os.WriteFile(outside, []byte("synthetic outside content"), 0o600)) + sql := "SELECT content FROM read_text('" + strings.ReplaceAll(filepath.ToSlash(outside), "'", "''") + "')" + _, _, err = runDaemonSQLQueryWithJobs(t.Context(), c, s, ownerEngine, + sql, daemonSQLQueryOptions{archiveOnly: true}, jobs) + requirements.Error(err) + assertions.Contains(err.Error(), "Permission Error") + + result, _, err = runDaemonSQLQueryWithJobs(t.Context(), c, s, ownerEngine, + sql, daemonSQLQueryOptions{}, jobs) + requirements.NoError(err) + assertions.Equal("synthetic outside content", result.Rows[0][0]) +} diff --git a/cmd/msgvault/cmd/serve_beeper_draft.go b/cmd/msgvault/cmd/serve_beeper_draft.go new file mode 100644 index 000000000..9a5de6511 --- /dev/null +++ b/cmd/msgvault/cmd/serve_beeper_draft.go @@ -0,0 +1,352 @@ +package cmd + +import ( + "context" + jsonv2 "encoding/json/v2" + "errors" + "fmt" + "strings" + + "go.kenn.io/msgvault/internal/agentgrant" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/beeper" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/textutil" +) + +type beeperDraftOutput struct { + Status string `json:"status"` + DraftID string `json:"draft_id"` + Revision int64 `json:"revision"` + Lifecycle string `json:"lifecycle"` + SourceID int64 `json:"source_id"` + ChatID string `json:"chat_id"` + Content *string `json:"content"` + PendingOperation string `json:"pending_operation,omitempty"` + CandidateContent string `json:"candidate_content,omitempty"` +} + +func emitBeeperDraft(emit func(api.CLIRunEvent) error, stream string, intent draftLifecycleIntent, status string, draft store.BeeperDraft) error { + if emit == nil { + return nil + } + output := beeperDraftOutput{ + Status: status, DraftID: draft.DraftID, Revision: draft.Revision, Lifecycle: draftLifecycleActive, + SourceID: draft.SourceID, ChatID: draft.ChatID, Content: draft.Text, + } + if draft.DiscardedAt != nil { + output.Lifecycle = "discarded" + } + if draft.Pending != nil { + output.PendingOperation, output.CandidateContent = draft.Pending.Operation, draft.Pending.Text + } + if intent.MetadataOnly { + output.Content, output.CandidateContent = nil, "" + } + if intent.JSON { + data, err := jsonv2.Marshal(output) + if err != nil { + return err + } + return emit(api.CLIRunEvent{Type: stream, Data: string(data) + "\n"}) + } + var b strings.Builder + fmt.Fprintf(&b, "draft %s revision %d %s\n", textutil.SanitizeTerminal(output.DraftID), output.Revision, output.Lifecycle) + fmt.Fprintf(&b, "status: %s\n", textutil.SanitizeTerminal(output.Status)) + fmt.Fprintf(&b, "beeper chat: %s\n", textutil.SanitizeTerminal(output.ChatID)) + if output.Content != nil { + fmt.Fprintf(&b, "content:\n%s\n", strings.TrimRight(textutil.SanitizeTerminalMultiline(*output.Content), "\n")) + } + if output.PendingOperation != "" { + fmt.Fprintf(&b, "pending operation: %s\n", output.PendingOperation) + } + if output.CandidateContent != "" { + fmt.Fprintf(&b, "candidate content:\n%s\n", strings.TrimRight(textutil.SanitizeTerminalMultiline(output.CandidateContent), "\n")) + } + return emit(api.CLIRunEvent{Type: stream, Data: b.String()}) +} + +// beeperDraftFailure shows the draft's recorded state before returning code. +func (a *storeAPIAdapter) beeperDraftFailure(ctx context.Context, emit func(api.CLIRunEvent) error, intent draftLifecycleIntent, draftID, code string, cause error) error { + evidenceCtx, cancel := localDraftEvidenceContext(ctx) + defer cancel() + if draft, err := a.store.GetBeeperDraftContext(evidenceCtx, draftID); err == nil { + _ = emitBeeperDraft(emit, cliStreamStderr, intent, code, draft) + } + return draftReplyError(code, cause) +} + +// lockBeeperSource serializes draft writes on a source, so one command's +// read, claim, write and finish never interleave with another's. +func (a *storeAPIAdapter) lockBeeperSource(ctx context.Context, sourceID int64) (*store.SyncExecution, error) { + execution, err := a.store.AcquireSyncExecutionContext(ctx, sourceID) + if errors.Is(err, store.ErrSyncAlreadyActive) { + return nil, draftReplyError("sync_active", err) + } + if err != nil { + return nil, draftReplyError("sync_lock_failed", err) + } + return execution, nil +} + +func (a *storeAPIAdapter) beeperDraftClient() (*beeper.Client, error) { + token, err := beeper.LoadToken(a.config.TokensDir()) + if err != nil { + return nil, draftReplyError("provider_unavailable", err) + } + return beeperClient(a.config, token), nil +} + +// readBeeperDraft reads the chat's composer from Beeper and checks that the +// chat belongs to the source's account. +func readBeeperDraft(ctx context.Context, client *beeper.Client, source *store.Source, chatID string) (beeper.DraftState, error) { + chat, err := client.GetChat(ctx, chatID) + if errors.Is(err, beeper.ErrNotFound) { + return beeper.DraftState{}, draftReplyError("chat_not_found", err) + } + if err != nil { + return beeper.DraftState{}, draftReplyError("provider_unavailable", err) + } + if chat.ID != chatID || chat.AccountID != source.Identifier { + return beeper.DraftState{}, draftReplyError("provider_identity_mismatch", errors.New("beeper returned another chat")) + } + state, err := chat.DraftState() + if err != nil { + return beeper.DraftState{}, draftReplyError("not_supported", err) + } + return state, nil +} + +// beeperDraftMatches reports whether Beeper still shows the committed draft. +func beeperDraftMatches(state beeper.DraftState, text *string) bool { + if state.Other { + return false + } + if text == nil { + return state.Empty + } + return !state.Empty && state.Text == *text +} + +func validBeeperDraftBody(body string) error { + if strings.TrimSpace(body) == "" || strings.ContainsRune(body, 0) { + return draftReplyError("invalid_args", errors.New("--body must be nonblank text")) + } + return nil +} + +// runBeeperDraftCreate writes a new draft into an empty Beeper chat composer. +// --to names the chat. +func (a *storeAPIAdapter) runBeeperDraftCreate(ctx context.Context, grant *agentgrant.Grant, intent draftComposeIntent, source *store.Source, emit func(api.CLIRunEvent) error) error { + if err := authorizeDelegatedDraftSource(grant, source); err != nil { + return err + } + if len(intent.To) != 1 || len(intent.Cc)+len(intent.Bcc) != 0 || intent.Subject != "" || intent.From != "" { + return draftReplyError("invalid_args", errors.New("a Beeper draft takes one --to chat ID, --body, and no other fields")) + } + if err := validBeeperDraftBody(intent.Body); err != nil { + return err + } + if err := authorizeDraftPolicy("beeper", a.beeperDraftPolicy, source.ID, source.SourceType); err != nil { + return err + } + client, err := a.beeperDraftClient() + if err != nil { + return err + } + execution, err := a.lockBeeperSource(ctx, source.ID) + if err != nil { + return err + } + defer func() { _ = execution.Release() }() + chatID := strings.TrimSpace(intent.To[0]) + if existing, err := a.store.LiveBeeperDraftContext(ctx, source.ID, chatID); err == nil { + if grant != nil { + // Reading draft text needs draft.edit; creators get the ID. + existing.Text, existing.Pending = nil, nil + } + _ = emitBeeperDraft(emit, cliStreamStderr, draftLifecycleIntent{JSON: intent.JSON}, "draft_exists", existing) + return draftReplyError("draft_exists", store.ErrBeeperDraftExists) + } else if !errors.Is(err, store.ErrBeeperDraftNotFound) { + return draftReplyError("draft_read_failed", err) + } + state, err := readBeeperDraft(ctx, client, source, chatID) + if err != nil { + return err + } + if !state.Empty { + return draftReplyError("draft_conflict", errors.New("beeper chat already has a draft")) + } + draft, err := a.store.CreateBeeperDraftContext(ctx, source.ID, chatID, intent.Body) + if errors.Is(err, store.ErrBeeperDraftExists) { + return draftReplyError("draft_exists", err) + } + if err != nil { + return draftReplyError("local_persistence_failed", err) + } + return a.writeBeeperDraft(ctx, client, source, draft, state, draftLifecycleIntent{JSON: intent.JSON}, "created", emit) +} + +// runBeeperDraftLifecycle serves draft-get, draft-edit and draft-delete for +// a Beeper draft. Each write first checks that Beeper still shows the +// committed draft, so text someone else typed in the chat is never replaced. +func (a *storeAPIAdapter) runBeeperDraftLifecycle(ctx context.Context, intent draftLifecycleIntent, grant *agentgrant.Grant, draft store.BeeperDraft, emit func(api.CLIRunEvent) error) error { + if intent.Operation == api.CLIRunDraftRecoverCommand { + if grant != nil { + return draftReplyNotPermitted(errors.New("draft-recover supports IMAP drafts only")) + } + return draftReplyError("not_supported", errors.New("draft-recover supports IMAP drafts only")) + } + if grant != nil { + source, err := a.store.GetSourceByIDContext(ctx, draft.SourceID) + if err != nil { + return draftReplyNotPermitted(err) + } + permissions := api.CLIRunDraftLifecyclePermissions(intent.Operation) + // Native drafts have no archived sender; create alone cannot read them. + if intent.Operation == api.CLIRunDraftGetCommand { + permissions = []agentgrant.Permission{agentgrant.PermissionDraftEdit, agentgrant.PermissionDraftDelete} + } + if err := chatDraftAuthorizer(grant, permissions...)(source.SourceType, source.Identifier); err != nil { + return err + } + intent.MetadataOnly = !grant.Allows(agentgrant.PermissionDraftEdit, agentgrant.SourceRef{ + Type: source.SourceType, Identifier: source.Identifier, + }) + } + if intent.Operation == api.CLIRunDraftGetCommand { + return emitBeeperDraft(emit, cliStreamStdout, intent, "ok", draft) + } + if draft.Revision != intent.Revision { + return draftReplyError("revision_mismatch", fmt.Errorf("expected revision %d, found %d", intent.Revision, draft.Revision)) + } + if draft.DiscardedAt != nil { + if intent.Operation == api.CLIRunDraftDeleteCommand { + return emitBeeperDraft(emit, cliStreamStdout, intent, "already_discarded", draft) + } + return draftReplyError("draft_discarded", errors.New("discarded drafts cannot be edited")) + } + if intent.Operation == api.CLIRunDraftEditCommand { + if err := validBeeperDraftBody(intent.Body); err != nil { + return err + } + } + source, err := a.store.GetSourceByIDContext(ctx, draft.SourceID) + if err != nil { + return draftReplyError("invalid_source", err) + } + if err := authorizeDraftPolicy("beeper", a.beeperDraftPolicy, source.ID, source.SourceType); err != nil { + return err + } + client, err := a.beeperDraftClient() + if err != nil { + return err + } + execution, err := a.lockBeeperSource(ctx, source.ID) + if err != nil { + return err + } + defer func() { _ = execution.Release() }() + if draft, err = a.store.GetBeeperDraftContext(ctx, draft.DraftID); err != nil { + return draftReplyError("draft_read_failed", err) + } + if draft.Revision != intent.Revision || draft.DiscardedAt != nil { + return draftReplyError("revision_mismatch", errors.New("draft changed while acquiring source ownership")) + } + state, err := readBeeperDraft(ctx, client, source, draft.ChatID) + if err != nil { + return err + } + if draft.Pending != nil { + // Settle an interrupted write from what Beeper shows now. + switch { + case draft.Pending.Operation == store.BeeperDraftOperationDelete && state.Empty: + finished, err := a.store.FinishBeeperDraftContext(ctx, draft.DraftID, draft.Revision, "") + if err != nil { + return draftReplyError("local_persistence_failed", err) + } + if intent.Operation != api.CLIRunDraftDeleteCommand { + _ = emitBeeperDraft(emit, cliStreamStderr, intent, "recovered", finished) + return draftReplyError("draft_discarded", errors.New("an earlier delete finished; the draft is discarded")) + } + return emitBeeperDraft(emit, cliStreamStdout, intent, "deleted", finished) + case state.Empty, beeperDraftMatches(state, draft.Text): + if draft, err = a.store.AbortBeeperDraftContext(ctx, draft.DraftID, draft.Revision); err != nil { + return draftReplyError("local_persistence_failed", err) + } + if draft.DiscardedAt != nil && intent.Operation == api.CLIRunDraftDeleteCommand { + return emitBeeperDraft(emit, cliStreamStdout, intent, "deleted", draft) + } + if draft.DiscardedAt != nil { + _ = emitBeeperDraft(emit, cliStreamStderr, intent, "recovered", draft) + return draftReplyError("draft_discarded", errors.New("the draft's first write never reached Beeper; the draft is discarded")) + } + default: + return a.beeperDraftFailure(ctx, emit, intent, draft.DraftID, "pending_operation", + errors.New("beeper shows a draft that may be an unconfirmed write; clear it in Beeper to continue")) + } + } + // An empty composer holds nothing to protect, so a write may proceed. + if !state.Empty && !beeperDraftMatches(state, draft.Text) { + return draftReplyError("draft_conflict", errors.New("the Beeper draft changed outside msgvault")) + } + operation, status := store.BeeperDraftOperationEdit, "edited" + if intent.Operation == api.CLIRunDraftDeleteCommand { + operation, status = store.BeeperDraftOperationDelete, "deleted" + } + claimed, err := a.store.ClaimBeeperDraftContext(ctx, draft.DraftID, draft.Revision, operation, intent.Body) + if err != nil { + if errors.Is(err, store.ErrBeeperDraftRevision) { + return draftReplyError("revision_mismatch", err) + } + return draftReplyError("claim_failed", err) + } + return a.writeBeeperDraft(ctx, client, source, claimed, state, intent, status, emit) +} + +// writeBeeperDraft sends a claimed write and commits what Beeper reports. A +// write Beeper refused before anything changed drops the claim; any other +// failure keeps it for the next command to settle. +func (a *storeAPIAdapter) writeBeeperDraft(ctx context.Context, client *beeper.Client, source *store.Source, claimed store.BeeperDraft, state beeper.DraftState, intent draftLifecycleIntent, status string, emit func(api.CLIRunEvent) error) error { + changed := false + fail := func(err error) error { + evidenceCtx, cancel := localDraftEvidenceContext(ctx) + defer cancel() + var writeErr *beeper.DraftWriteError + if errors.As(err, &writeErr) && writeErr.Status >= 400 && writeErr.Status < 500 && !changed { + if _, abortErr := a.store.AbortBeeperDraftContext(evidenceCtx, claimed.DraftID, claimed.Revision); abortErr == nil { + return a.beeperDraftFailure(ctx, emit, intent, claimed.DraftID, "provider_rejected", err) + } + } + return a.beeperDraftFailure(ctx, emit, intent, claimed.DraftID, "remote_unknown", err) + } + if !state.Empty { + chat, err := client.SetDraft(ctx, claimed.ChatID, nil) + if err != nil { + return fail(err) + } + changed = true + if state, err = chat.DraftState(); err != nil || !state.Empty { + return fail(errors.New("beeper did not confirm the cleared draft")) + } + } + text := "" + if claimed.Pending.Operation == store.BeeperDraftOperationEdit { + chat, err := client.SetDraft(ctx, claimed.ChatID, &claimed.Pending.Text) + if err != nil { + return fail(err) + } + changed = true + if state, err = chat.DraftState(); err != nil || state.Empty || state.Other || chat.ID != claimed.ChatID || chat.AccountID != source.Identifier { + return fail(errors.New("beeper did not confirm the draft text")) + } + text = state.Text + } + evidenceCtx, cancel := localDraftEvidenceContext(ctx) + defer cancel() + finished, err := a.store.FinishBeeperDraftContext(evidenceCtx, claimed.DraftID, claimed.Revision, text) + if err != nil { + return a.beeperDraftFailure(ctx, emit, intent, claimed.DraftID, "accepted_local_failed", err) + } + return emitBeeperDraft(emit, cliStreamStdout, intent, status, finished) +} diff --git a/cmd/msgvault/cmd/serve_beeper_draft_test.go b/cmd/msgvault/cmd/serve_beeper_draft_test.go new file mode 100644 index 000000000..dda439cbd --- /dev/null +++ b/cmd/msgvault/cmd/serve_beeper_draft_test.go @@ -0,0 +1,338 @@ +package cmd + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/agentgrant" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/beeper" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil" +) + +const beeperDraftTestChat = "!chat:beeper.local" + +// fakeBeeperDrafts models Beeper Desktop's composer: setting text over a +// draft is refused with 409, and accepted text comes back formatted. +type fakeBeeperDrafts struct { + mu sync.Mutex + account string + draft string // raw JSON value of the chat's draft field + requests int + patches int + failText bool // answer 500 to the next text write without applying it +} + +func (f *fakeBeeperDrafts) ServeHTTP(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + f.requests++ + if r.URL.Path != "/v1/chats/"+beeperDraftTestChat || r.Header.Get("Authorization") != "Bearer test-token" { + w.WriteHeader(http.StatusNotFound) + return + } + if r.Method == http.MethodPatch { + f.patches++ + var body struct { + Draft *struct { + Text string `json:"text"` + } `json:"draft"` + } + data, _ := io.ReadAll(r.Body) + if json.Unmarshal(data, &body) != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + switch { + case body.Draft == nil: + f.draft = "null" + case f.failText: + f.failText = false + w.WriteHeader(http.StatusInternalServerError) + return + case f.draft != "null": + w.WriteHeader(http.StatusConflict) + return + default: + formatted, err := json.Marshal(map[string]string{"text": "

" + body.Draft.Text + "

"}) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + f.draft = string(formatted) + } + } + _, _ = io.WriteString(w, `{"id":"`+beeperDraftTestChat+`","accountID":"`+f.account+`","draft":`+f.draft+`}`) +} + +func (f *fakeBeeperDrafts) set(draft string, failText bool) { + f.mu.Lock() + defer f.mu.Unlock() + f.draft, f.failText = draft, failText +} + +func (f *fakeBeeperDrafts) current() string { + f.mu.Lock() + defer f.mu.Unlock() + return f.draft +} + +func (f *fakeBeeperDrafts) counts() (int, int) { + f.mu.Lock() + defer f.mu.Unlock() + return f.requests, f.patches +} + +type beeperDraftFixture struct { + store *store.Store + source *store.Source + beeper *fakeBeeperDrafts + adapter *storeAPIAdapter +} + +func newBeeperDraftFixture(t *testing.T) beeperDraftFixture { + t.Helper() + st := testutil.NewTestStore(t) + source, err := st.GetOrCreateSource("beeper", "whatsapp") + require.NoError(t, err) + fake := &fakeBeeperDrafts{account: source.Identifier, draft: "null"} + server := httptest.NewServer(fake) + t.Cleanup(server.Close) + cfg := &config.Config{Data: config.DataConfig{DataDir: t.TempDir()}} + cfg.Beeper.URL = server.URL + require.NoError(t, beeper.SaveToken(cfg.TokensDir(), "test-token")) + adapter := &storeAPIAdapter{ + store: st, config: cfg, logger: testLoggerValue(), + beeperDraftPolicy: []config.GmailDraftSource{{SourceID: source.ID, Enabled: true}}, + } + return beeperDraftFixture{store: st, source: source, beeper: fake, adapter: adapter} +} + +func (f beeperDraftFixture) run(t *testing.T, grant *agentgrant.Grant, args ...string) (beeperDraftOutput, error) { + t.Helper() + args = append(args, "--json") + var events []api.CLIRunEvent + emit := func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + } + req := api.CLIRunRequest{Args: args, Grant: grant} + var err error + if args[0] == api.CLIRunDraftComposeCommand { + err = f.adapter.runCLIComposeDraft(t.Context(), req, emit) + } else { + err = f.adapter.runCLIDraftLifecycle(t.Context(), req, emit) + } + var output beeperDraftOutput + if len(events) > 0 { + require.NoError(t, json.Unmarshal([]byte(events[len(events)-1].Data), &output)) + } + return output, err +} + +func (f beeperDraftFixture) create(t *testing.T, grant *agentgrant.Grant, body string) (beeperDraftOutput, error) { + t.Helper() + return f.run(t, grant, api.CLIRunDraftComposeCommand, "--source-id", strconv.FormatInt(f.source.ID, 10), + "--to", beeperDraftTestChat, "--body", body) +} + +func (f beeperDraftFixture) edit(t *testing.T, draft beeperDraftOutput, body string) (beeperDraftOutput, error) { + t.Helper() + return f.run(t, nil, api.CLIRunDraftEditCommand, draft.DraftID, "--revision", strconv.FormatInt(draft.Revision, 10), "--body", body) +} + +func assertBeeperDraftCode(t *testing.T, err error, code string) { + t.Helper() + var coded *api.CLIRunCodedError + require.ErrorAs(t, err, &coded) + assert.Equal(t, code, coded.Code) +} + +func TestBeeperDraftWritesCheckTheComposerFirst(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := newBeeperDraftFixture(t) + + created, err := f.create(t, nil, "hello") + require.NoError(err) + assert.Equal("created", created.Status) + assert.Equal(int64(1), created.Revision) + require.NotNil(created.Content) + assert.Equal("

hello

", *created.Content) + + _, err = f.create(t, nil, "again") + assertBeeperDraftCode(t, err, "draft_exists") + + edited, err := f.edit(t, created, "second") + require.NoError(err) + assert.Equal(int64(2), edited.Revision) + assert.Equal("

second

", *edited.Content) + + // Text typed in Beeper is never replaced or cleared. + f.beeper.set(`{"text":"typed in Beeper"}`, false) + _, patchesBefore := f.beeper.counts() + _, err = f.edit(t, edited, "third") + assertBeeperDraftCode(t, err, "draft_conflict") + _, err = f.run(t, nil, api.CLIRunDraftDeleteCommand, edited.DraftID, "--revision", "2") + assertBeeperDraftCode(t, err, "draft_conflict") + _, patchesAfter := f.beeper.counts() + assert.Equal(patchesBefore, patchesAfter) + + // An interrupted edit stays pending until Beeper shows how it ended. + f.beeper.set(`{"text":"

second

"}`, true) + pending, err := f.edit(t, edited, "third") + assertBeeperDraftCode(t, err, "remote_unknown") + assert.Equal(store.BeeperDraftOperationEdit, pending.PendingOperation) + assert.Equal("third", pending.CandidateContent) + retried, err := f.edit(t, edited, "third") + require.NoError(err) + assert.Equal(int64(3), retried.Revision) + assert.Equal("

third

", *retried.Content) + assert.Empty(retried.PendingOperation) + + deleted, err := f.run(t, nil, api.CLIRunDraftDeleteCommand, retried.DraftID, "--revision", "3") + require.NoError(err) + assert.Equal("discarded", deleted.Lifecycle) + assert.Equal("null", f.beeper.current()) + + recreated, err := f.create(t, nil, "fresh") + require.NoError(err) + assert.NotEqual(created.DraftID, recreated.DraftID) + requestsBefore, _ := f.beeper.counts() + got, err := f.run(t, nil, api.CLIRunDraftGetCommand, recreated.DraftID) + require.NoError(err) + assert.Equal("

fresh

", *got.Content) + requestsAfter, _ := f.beeper.counts() + assert.Equal(requestsBefore, requestsAfter) +} + +func TestBeeperDraftDelegatedLifecycle(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + f := newBeeperDraftFixture(t) + grant := &agentgrant.Grant{ + ID: "beeper-grant", + Permissions: []agentgrant.Permission{agentgrant.PermissionDraftCreate, agentgrant.PermissionDraftEdit, agentgrant.PermissionDraftDelete}, + Sources: []agentgrant.SourceRef{{Type: "beeper", Identifier: f.source.Identifier}}, + } + created, err := f.create(t, grant, "hello") + require.NoError(err) + edited, err := f.run(t, grant, api.CLIRunDraftEditCommand, created.DraftID, "--revision", "1", "--body", "edited") + require.NoError(err) + assert.JSONEq(`{"text":"

edited

"}`, f.beeper.current()) + loaded, err := f.run(t, grant, api.CLIRunDraftGetCommand, created.DraftID) + require.NoError(err) + require.NotNil(loaded.Content) + assert.Equal("

edited

", *loaded.Content) + _, err = f.run(t, grant, api.CLIRunDraftDeleteCommand, created.DraftID, "--revision", strconv.FormatInt(edited.Revision, 10)) + require.NoError(err) + assert.Equal("null", f.beeper.current()) +} + +func TestBeeperDraftDeleteOnlyResponses(t *testing.T) { + for _, format := range []string{"json", "text"} { + t.Run(format, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + f := newBeeperDraftFixture(t) + created, err := f.create(t, nil, "owner draft") + require.NoError(err) + grant := &agentgrant.Grant{ + ID: "delete-only", + Permissions: []agentgrant.Permission{agentgrant.PermissionDraftDelete}, + Sources: []agentgrant.SourceRef{{Type: "beeper", Identifier: f.source.Identifier}}, + } + run := func(status string, args ...string) error { + t.Helper() + if format == "json" { + args = append(args, "--json") + } + var output strings.Builder + err := f.adapter.runCLIDraftLifecycle(t.Context(), api.CLIRunRequest{Args: args, Grant: grant}, func(event api.CLIRunEvent) error { + output.WriteString(event.Data) + return nil + }) + assert.Contains(output.String(), created.DraftID) + assert.NotContains(output.String(), "owner draft") + if format == "json" { + var got beeperDraftOutput + require.NoError(json.Unmarshal([]byte(output.String()), &got)) + assert.Equal(status, got.Status) + assert.Positive(got.Revision) + assert.Nil(got.Content) + assert.Empty(got.CandidateContent) + } else { + assert.Contains(output.String(), "status: "+status) + assert.NotContains(output.String(), "content:") + } + return err + } + require.NoError(run("ok", api.CLIRunDraftGetCommand, created.DraftID)) + + // An unconfirmed edit has both committed and candidate text to hide. + _, err = f.store.ClaimBeeperDraftContext(t.Context(), created.DraftID, created.Revision, store.BeeperDraftOperationEdit, "pending owner draft") + require.NoError(err) + require.NoError(run("ok", api.CLIRunDraftGetCommand, created.DraftID)) + f.beeper.set(`{"text":"

pending owner draft

"}`, false) + err = run("pending_operation", api.CLIRunDraftDeleteCommand, created.DraftID, "--revision", "1") + assertBeeperDraftCode(t, err, "pending_operation") + + // Redaction must leave the stored text available to an editor. + editor := *grant + editor.Permissions = []agentgrant.Permission{agentgrant.PermissionDraftEdit} + visible, err := f.run(t, &editor, api.CLIRunDraftGetCommand, created.DraftID) + require.NoError(err) + require.NotNil(visible.Content) + assert.Equal("

owner draft

", *visible.Content) + assert.Equal("pending owner draft", visible.CandidateContent) + assert.Equal(store.BeeperDraftOperationEdit, visible.PendingOperation) + + f.beeper.set("null", false) + require.NoError(run("deleted", api.CLIRunDraftDeleteCommand, created.DraftID, "--revision", "1")) + require.NoError(run("already_discarded", api.CLIRunDraftDeleteCommand, created.DraftID, "--revision", "2")) + }) + } +} + +func TestBeeperDraftDelegatedDenialPrecedesProvider(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := newBeeperDraftFixture(t) + created, err := f.create(t, nil, "hello") + require.NoError(err) + requestsBefore, _ := f.beeper.counts() + + other := &agentgrant.Grant{ + ID: "beeper-grant", + Permissions: []agentgrant.Permission{agentgrant.PermissionDraftCreate, agentgrant.PermissionDraftEdit}, + Sources: []agentgrant.SourceRef{{Type: "beeper", Identifier: "telegram"}}, + } + _, err = f.create(t, other, "agent") + assertBeeperDraftCode(t, err, "not_permitted") + _, err = f.run(t, other, api.CLIRunDraftEditCommand, created.DraftID, "--revision", "1", "--body", "agent") + assertBeeperDraftCode(t, err, "not_permitted") + requestsAfter, _ := f.beeper.counts() + assert.Equal(requestsBefore, requestsAfter) + + creator := &agentgrant.Grant{ + ID: "beeper-grant", + Permissions: []agentgrant.Permission{agentgrant.PermissionDraftCreate}, + Sources: []agentgrant.SourceRef{{Type: "beeper", Identifier: f.source.Identifier}}, + } + _, err = f.run(t, creator, api.CLIRunDraftGetCommand, created.DraftID) + assertBeeperDraftCode(t, err, "not_permitted") + existing, err := f.create(t, creator, "agent") + assertBeeperDraftCode(t, err, "draft_exists") + assert.Equal(created.DraftID, existing.DraftID) + assert.Nil(existing.Content) +} diff --git a/cmd/msgvault/cmd/serve_carddav_test.go b/cmd/msgvault/cmd/serve_carddav_test.go index 29cd61dd4..deb19b2ae 100644 --- a/cmd/msgvault/cmd/serve_carddav_test.go +++ b/cmd/msgvault/cmd/serve_carddav_test.go @@ -28,22 +28,23 @@ type scheduledCardDAVFixture struct { func TestRecoverCardDAVSyncRunsAtStartupTerminalizesOrphansAndLogsOnlyCount(t *testing.T) { require := require.New(t) - assert := assert.New(t) + assertions := assert.New(t) st := testutil.NewTestStore(t) _, err := st.StartCardDAVSyncRunContext(t.Context(), store.CardDAVSyncRunStart{ - Trigger: store.CardDAVSyncTriggerScheduled, + AccountID: store.DefaultCardDAVAccountID, + Trigger: store.CardDAVSyncTriggerScheduled, }) require.NoError(err) var logs bytes.Buffer logger := slog.New(slog.NewTextHandler(&logs, nil)) require.NoError(recoverCardDAVSyncRunsAtStartup(t.Context(), st, logger)) - runs, err := st.ListCardDAVSyncRunsContext(t.Context(), 10, nil) + runs, err := st.ListCardDAVSyncRunsContext(t.Context(), 10, nil, store.AllCardDAVAccounts) require.NoError(err) require.Len(runs, 1) - assert.Equal(store.CardDAVSyncRunFailed, runs[0].State) - assert.Equal("daemon_restarted", runs[0].ErrorCode) - assert.Contains(logs.String(), "count=1") - assert.NotContains(strings.ToLower(logs.String()), "error_message") + assertions.Equal(store.CardDAVSyncRunFailed, runs[0].State) + assertions.Equal("daemon_restarted", runs[0].ErrorCode) + assertions.Contains(logs.String(), "count=1") + assertions.NotContains(strings.ToLower(logs.String()), "error_message") } func TestRecoverCardDAVSyncRunsAtStartupReturnsFailure(t *testing.T) { @@ -100,22 +101,22 @@ func TestRegisterCardDAVSchedulerJobRequiresEnabledSchedule(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert := assert.New(t) + assertions := assert.New(t) require := require.New(t) sched := scheduler.New(nil) t.Cleanup(func() { sched.Stop() }) service := &scheduledCardDAVFixture{} logger := slog.New(slog.DiscardHandler) - require.NoError(reconcileCardDAVSchedulerJob(sched, tt.config, service, logger)) + require.NoError(reconcileCardDAVSchedulerJob(sched, tt.config, service, logger, config.DefaultCardDAVConnection)) status := sched.JobStatus() if len(tt.wantStatus) == 0 { - assert.Empty(status) + assertions.Empty(status) return } require.Len(status, 1) - assert.Equal(tt.wantStatus[0].Name, status[0].Name) - assert.Equal(tt.wantStatus[0].Schedule, status[0].Schedule) + assertions.Equal(tt.wantStatus[0].Name, status[0].Name) + assertions.Equal(tt.wantStatus[0].Schedule, status[0].Schedule) }) } } @@ -126,7 +127,7 @@ func TestRegisterCardDAVSchedulerJobSkipsUnavailableService(t *testing.T) { require.NoError(t, reconcileCardDAVSchedulerJob(sched, config.CardDAVConfig{Enabled: true, Schedule: "0 */6 * * *"}, nil, - slog.New(slog.DiscardHandler))) + slog.New(slog.DiscardHandler), config.DefaultCardDAVConnection)) assert.False(t, sched.IsJobScheduled(api.CardDAVJobName)) } @@ -137,12 +138,14 @@ func TestGoogleCardDAVSchedulerWaitsForAuthorization(t *testing.T) { secrets := filepath.Join(dir, "client.json") required.NoError(os.WriteFile(secrets, []byte(`{"web":{"client_id":"synthetic-client","client_secret":"synthetic-secret","redirect_uris":["https://archive.example/"]}}`), 0600)) cfg := config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.HomeDir, cfg.Data.DataDir = dir, dir cfg.OAuth.ClientSecrets = secrets cfg.CardDAV = config.CardDAVConfig{Provider: "google", BaseURL: carddav.GoogleDiscoveryURL, Username: "person@example.com", Enabled: true, Schedule: "0 1 * * *"} required.NoError(cfg.Save()) st := testutil.NewTestStore(t) - _, _, err := st.ReplaceCardDAVDiscoveryContext(t.Context(), store.CardDAVDiscoveryInput{ + _, _, err := st.ReplaceCardDAVDiscoveryContext(testCtx, store.CardDAVDiscoveryInput{ BaseURL: cfg.CardDAV.BaseURL, Username: cfg.CardDAV.Username, PrincipalURL: "https://www.googleapis.com/principal/", HomeURL: "https://www.googleapis.com/contacts/", }) @@ -157,16 +160,16 @@ func TestGoogleCardDAVSchedulerWaitsForAuthorization(t *testing.T) { sched := scheduler.New(nil) t.Cleanup(func() { sched.Stop() }) controller.SetScheduleReconciler(func(settings config.CardDAVConfig, service api.CardDAVOperations) error { - return reconcileCardDAVSchedulerJob(sched, settings, service, logger) + return reconcileCardDAVSchedulerJob(sched, settings, service, logger, config.DefaultCardDAVConnection) }) required.NoError(controller.ReconcileSchedule()) - status, err := controller.Status(t.Context()) + status, err := controller.Status(testCtx, "") required.NoError(err) assertions.Equal("google_authorization_required", status.RepairReason) assertions.False(sched.IsJobScheduled(api.CardDAVJobName), "startup must skip an unauthorized Google account") request := api.CardDAVAccountRequest{Provider: "google", Username: cfg.CardDAV.Username, Enabled: new(true), Schedule: "0 2 * * *"} - _, err = controller.Save(t.Context(), request) + _, err = controller.Save(testCtx, request) required.NoError(err) assertions.False(sched.IsJobScheduled(api.CardDAVJobName), "schedule-only saves must also skip missing authorization") @@ -175,7 +178,7 @@ func TestGoogleCardDAVSchedulerWaitsForAuthorization(t *testing.T) { token := fmt.Sprintf(`{"access_token":"synthetic-access","refresh_token":"synthetic-refresh","client_id":"synthetic-client","scopes":[%q]}`, oauth.ScopeCardDAV) required.NoError(os.WriteFile(tokenPath, []byte(token), 0600)) request.Schedule = "0 3 * * *" - _, err = controller.Save(t.Context(), request) + _, err = controller.Save(testCtx, request) required.NoError(err) assertions.True(sched.IsJobScheduled(api.CardDAVJobName)) jobs := sched.JobStatus() @@ -184,13 +187,13 @@ func TestGoogleCardDAVSchedulerWaitsForAuthorization(t *testing.T) { required.NoError(os.Remove(tokenPath)) request.Schedule = "0 4 * * *" - _, err = controller.Save(t.Context(), request) + _, err = controller.Save(testCtx, request) required.NoError(err) assertions.False(sched.IsJobScheduled(api.CardDAVJobName), "saving after credentials are removed must unschedule the account") } func TestReconcileCardDAVSchedulerJobUpdatesRunsAndRemovesStableJob(t *testing.T) { - assert := assert.New(t) + assertions := assert.New(t) require := require.New(t) tracker := &fakeDaemonWorkTracker{allow: true} @@ -199,22 +202,58 @@ func TestReconcileCardDAVSchedulerJobUpdatesRunsAndRemovesStableJob(t *testing.T service := &scheduledCardDAVFixture{} logger := slog.New(slog.DiscardHandler) - require.NoError(reconcileCardDAVSchedulerJob(sched, config.CardDAVConfig{Enabled: true, Schedule: "0 1 * * *"}, service, logger)) + require.NoError(reconcileCardDAVSchedulerJob(sched, config.CardDAVConfig{Enabled: true, Schedule: "0 1 * * *"}, service, logger, config.DefaultCardDAVConnection)) require.NoError(sched.TriggerJob(api.CardDAVJobName)) - assert.Equal(1, service.syncs) + assertions.Equal(1, service.syncs) require.Len(service.options, 1) - assert.Equal(store.CardDAVSyncTriggerScheduled, service.options[0].Trigger) + assertions.Equal(store.CardDAVSyncTriggerScheduled, service.options[0].Trigger) begin, done := tracker.counts() - assert.Equal(1, begin) - assert.Equal(1, done) + assertions.Equal(1, begin) + assertions.Equal(1, done) - require.NoError(reconcileCardDAVSchedulerJob(sched, config.CardDAVConfig{Enabled: true, Schedule: "0 2 * * *"}, service, logger)) + require.NoError(reconcileCardDAVSchedulerJob(sched, config.CardDAVConfig{Enabled: true, Schedule: "0 2 * * *"}, service, logger, config.DefaultCardDAVConnection)) status := sched.JobStatus() require.Len(status, 1) - assert.Equal("0 2 * * *", status[0].Schedule) + assertions.Equal("0 2 * * *", status[0].Schedule) require.NoError(sched.TriggerJob(api.CardDAVJobName)) - assert.Equal(2, service.syncs) + assertions.Equal(2, service.syncs) + + require.NoError(reconcileCardDAVSchedulerJob(sched, config.CardDAVConfig{Enabled: false, Schedule: "0 2 * * *"}, service, logger, config.DefaultCardDAVConnection)) + assertions.False(sched.IsJobScheduled(api.CardDAVJobName)) +} + +func TestCardDAVNamedSchedulerJobsRemainIndependent(t *testing.T) { + assertions := assert.New(t) + require := require.New(t) - require.NoError(reconcileCardDAVSchedulerJob(sched, config.CardDAVConfig{Enabled: false, Schedule: "0 2 * * *"}, service, logger)) - assert.False(sched.IsJobScheduled(api.CardDAVJobName)) + sched := scheduler.New(nil) + t.Cleanup(func() { sched.Stop() }) + logger := slog.New(slog.DiscardHandler) + primary, work := &scheduledCardDAVFixture{}, &scheduledCardDAVFixture{} + settings := config.CardDAVConfig{Enabled: true, Schedule: "0 1 * * *"} + require.NoError(reconcileCardDAVSchedulerJob(sched, settings, primary, logger, config.DefaultCardDAVConnection)) + require.NoError(reconcileCardDAVSchedulerJob(sched, settings, work, logger, "work")) + require.Len(sched.JobStatus(), 2) + require.NoError(sched.TriggerJob("carddav:work")) + assertions.Equal(0, primary.syncs) + assertions.Equal(1, work.syncs) + require.Len(work.options, 1) + assertions.Equal(store.CardDAVSyncTriggerScheduled, work.options[0].Trigger) + settings.Schedule = "0 2 * * *" + require.NoError(reconcileCardDAVSchedulerJob(sched, settings, work, logger, "work")) + require.Len(sched.JobStatus(), 2) + require.NoError(sched.TriggerJob(api.CardDAVJobName)) + assertions.Equal(1, primary.syncs) + for _, job := range sched.JobStatus() { + if job.Name == api.CardDAVJobName { + assertions.Equal("0 1 * * *", job.Schedule) + } else { + assertions.Equal("0 2 * * *", job.Schedule) + } + } + for _, settings := range []config.CardDAVConfig{{Enabled: false, Schedule: "0 2 * * *"}, {Enabled: true, Schedule: "0 2 * * *"}} { + require.NoError(reconcileCardDAVSchedulerJob(sched, settings, nil, logger, "work")) + assertions.False(sched.IsJobScheduled("carddav:work")) + assertions.True(sched.IsJobScheduled(api.CardDAVJobName)) + } } diff --git a/cmd/msgvault/cmd/serve_chat_draft.go b/cmd/msgvault/cmd/serve_chat_draft.go new file mode 100644 index 000000000..7fd353429 --- /dev/null +++ b/cmd/msgvault/cmd/serve_chat_draft.go @@ -0,0 +1,195 @@ +package cmd + +import ( + "context" + jsonv2 "encoding/json/v2" + "errors" + "fmt" + "slices" + "strings" + "unicode/utf8" + + "go.kenn.io/msgvault/internal/agentgrant" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/textutil" +) + +// chatDraftOutput reports a local chat draft. Location is always msgvault: +// the text never reaches the provider's composer. +type chatDraftOutput struct { + Status string `json:"status"` + Location string `json:"location"` + DraftID string `json:"draft_id"` + Revision int64 `json:"revision"` + Body string `json:"body"` + SourceID int64 `json:"source_id"` + SourceType string `json:"source_type"` + Source string `json:"source"` + ConversationID int64 `json:"conversation_id"` + SourceConversationID string `json:"source_conversation_id"` + ConversationType string `json:"conversation_type"` + ReplyToSourceMessageID string `json:"reply_to_source_message_id,omitempty"` +} + +// chatDraftAuthorizer admits the owner, or a grant holding one of permissions +// on the conversation's source. +func chatDraftAuthorizer(grant *agentgrant.Grant, permissions ...agentgrant.Permission) store.ChatDraftAuthorizer { + return func(sourceType, identifier string) error { + ref := agentgrant.SourceRef{Type: sourceType, Identifier: identifier} + if grant == nil || slices.ContainsFunc(permissions, func(p agentgrant.Permission) bool { return grant.Allows(p, ref) }) { + return nil + } + return draftReplyNotPermitted(fmt.Errorf("source %s:%s is not in grant %s", sourceType, identifier, grant.ID)) + } +} + +// redactChatDraftBody keeps delete access metadata-only, like the email draft path. +func redactChatDraftBody(grant *agentgrant.Grant, draft store.ChatDraft) store.ChatDraft { + if grant != nil && !grant.Allows(agentgrant.PermissionDraftEdit, agentgrant.SourceRef{ + Type: draft.SourceType, Identifier: draft.SourceIdentifier, + }) { + draft.Body = "" + } + return draft +} + +func chatDraftStoreError(err error, grant *agentgrant.Grant) error { + var coded *api.CLIRunCodedError + switch { + case errors.As(err, &coded): + return err + case grant != nil && (errors.Is(err, store.ErrChatDraftNotFound) || errors.Is(err, store.ErrChatDraftInvalidDestination)): + return draftReplyNotPermitted(err) + case errors.Is(err, store.ErrChatDraftNotFound): + return draftReplyError("draft_not_found", err) + case errors.Is(err, store.ErrChatDraftInvalidDestination): + return draftReplyError("invalid_destination", err) + case errors.Is(err, store.ErrChatDraftUnsupportedSource): + return draftReplyError("unsupported_source", err) + case errors.Is(err, store.ErrChatDraftRevisionConflict): + return draftReplyError("revision_mismatch", err) + default: + return draftReplyError("local_store_failed", err) + } +} + +// invalidChatDraftBody rejects text PostgreSQL cannot store, so both backends agree. +func invalidChatDraftBody(body string) error { + if strings.ContainsRune(body, 0) || !utf8.ValidString(body) { + return draftReplyError("invalid_args", errors.New("--body must be valid UTF-8 without NUL")) + } + return nil +} + +func (a *storeAPIAdapter) runCLIChatDraftCreate( + ctx context.Context, intent draftComposeIntent, grant *agentgrant.Grant, emit func(api.CLIRunEvent) error, +) error { + if err := invalidChatDraftBody(intent.Body); err != nil { + return err + } + authorize := chatDraftAuthorizer(grant, agentgrant.PermissionDraftCreate) + authorized := false + draft, err := a.store.CreateChatDraftContext(ctx, intent.ConversationID, intent.ReplyTo, intent.Body, + func(sourceType, identifier string) error { + err := authorize(sourceType, identifier) + authorized = err == nil + return err + }) + if err != nil { + if authorized { + grant = nil // the grant covers this source, so later errors reveal nothing to hide + } + return chatDraftStoreError(err, grant) + } + return emitChatDrafts(emit, intent.JSON, false, "created", draft) +} + +func (a *storeAPIAdapter) runCLIChatDraftLifecycle( + ctx context.Context, intent draftLifecycleIntent, grant *agentgrant.Grant, emit func(api.CLIRunEvent) error, +) error { + if intent.Operation == api.CLIRunDraftRecoverCommand { + return draftReplyError("not_supported", errors.New("local chat drafts have nothing to recover")) + } + // Chat drafts carry no sender, so draft.create alone never reads them. + permissions := slices.DeleteFunc(api.CLIRunDraftLifecyclePermissions(intent.Operation), func(p agentgrant.Permission) bool { return p == agentgrant.PermissionDraftCreate }) + authorize := chatDraftAuthorizer(grant, permissions...) + if intent.ConversationID != 0 { + drafts, err := a.store.ListChatDraftsContext(ctx, intent.ConversationID, authorize) + if err != nil { + return chatDraftStoreError(err, grant) + } + for i := range drafts { + drafts[i] = redactChatDraftBody(grant, drafts[i]) + } + return emitChatDrafts(emit, intent.JSON, true, "ok", drafts...) + } + draft, err := a.store.GetChatDraftContext(ctx, intent.DraftID) + if err == nil { + err = authorize(draft.SourceType, draft.SourceIdentifier) + } + if err != nil { + return chatDraftStoreError(err, grant) + } + status := "ok" + switch intent.Operation { + case api.CLIRunDraftEditCommand: + status = "edited" + if err := invalidChatDraftBody(intent.Body); err != nil { + return err + } + draft, err = a.store.UpdateChatDraftContext(ctx, draft.DraftID, intent.Revision, intent.Body) + case api.CLIRunDraftDeleteCommand: + status = "deleted" + err = a.store.DeleteChatDraftContext(ctx, draft.DraftID, intent.Revision) + } + if err != nil { + return chatDraftStoreError(err, grant) + } + if intent.Operation != api.CLIRunDraftEditCommand { + draft = redactChatDraftBody(grant, draft) + } + return emitChatDrafts(emit, intent.JSON, false, status, draft) +} + +// emitChatDrafts writes one draft, or a list as a JSON array, to stdout. +func emitChatDrafts(emit func(api.CLIRunEvent) error, asJSON, list bool, status string, drafts ...store.ChatDraft) error { + outputs := make([]chatDraftOutput, 0, len(drafts)) + var text strings.Builder + for _, d := range drafts { + output := chatDraftOutput{ + Status: status, Location: "msgvault", DraftID: d.DraftID, Revision: d.Revision, Body: d.Body, + SourceID: d.SourceID, SourceType: d.SourceType, Source: d.SourceIdentifier, + ConversationID: d.ConversationID, SourceConversationID: d.SourceConversationID, + ConversationType: d.ConversationType, ReplyToSourceMessageID: d.ReplyToSourceMessageID, + } + outputs = append(outputs, output) + fmt.Fprintf(&text, "location=msgvault draft=%s revision=%d status=%s source=%s source_type=%s conversation=%d conversation_type=%s source_conversation=%s", + textutil.SanitizeTerminal(d.DraftID), d.Revision, status, textutil.SanitizeTerminal(d.SourceIdentifier), + textutil.SanitizeTerminal(d.SourceType), d.ConversationID, textutil.SanitizeTerminal(d.ConversationType), + textutil.SanitizeTerminal(d.SourceConversationID)) + if d.ReplyToSourceMessageID != "" { + text.WriteString(" reply_to=" + textutil.SanitizeTerminal(d.ReplyToSourceMessageID)) + } + text.WriteString("\nbody:\n" + textutil.SanitizeTerminalMultiline(d.Body) + "\n") + } + if len(drafts) == 0 { + text.WriteString("drafts=0\n") + } + data := text.String() + if asJSON { + var value any = outputs + if !list { + value = outputs[0] + } + encoded, err := jsonv2.Marshal(value) + if err != nil { + return draftReplyError("output_failed", err) + } + data = string(encoded) + "\n" + } + if err := emit(api.CLIRunEvent{Type: cliStreamStdout, Data: data}); err != nil { + return draftReplyError("output_failed", err) + } + return nil +} diff --git a/cmd/msgvault/cmd/serve_chat_draft_test.go b/cmd/msgvault/cmd/serve_chat_draft_test.go new file mode 100644 index 000000000..69bbb1c4a --- /dev/null +++ b/cmd/msgvault/cmd/serve_chat_draft_test.go @@ -0,0 +1,215 @@ +package cmd + +import ( + "bytes" + "encoding/json" + "fmt" + "log/slog" + "net/http/httptest" + "strconv" + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/agentgrant" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/testutil" +) + +func TestChatDraftThroughDraftCommands(t *testing.T) { + assert, require := assert.New(t), require.New(t) + st := testutil.NewTestStore(t) + source, err := st.GetOrCreateSource("slack", "slack-account") + require.NoError(err) + conversationID, err := st.EnsureConversationWithType(source.ID, "C1", "channel", "General") + require.NoError(err) + conversation := strconv.FormatInt(conversationID, 10) + server := httptest.NewServer(api.NewServerWithOptions(api.ServerOptions{ + Config: &config.Config{HomeDir: t.TempDir()}, Store: &storeAPIAdapter{store: st}, Logger: slog.New(slog.DiscardHandler), + }).Router()) + t.Cleanup(server.Close) + testCtx := configureRemoteDaemonForTest(t, server.URL) + run := func(args ...string) (string, error) { + root := &cobra.Command{Use: "msgvault"} + root.SetContext(testCtx) + root.AddCommand(newDraftComposeCommand(), newDraftGetCommand(), newDraftEditCommand(), newDraftDeleteCommand(), newDraftRecoverCommand()) + silenceUsageInRunE(root) + var stdout, stderr bytes.Buffer + root.SetOut(&stdout) + root.SetErr(&stderr) + root.SetArgs(args) + if err := root.ExecuteContext(testCtx); err != nil { + return stdout.String() + stderr.String(), fmt.Errorf("%v: %w", args, err) + } + return stdout.String() + stderr.String(), nil + } + + createdJSON, err := run("draft-compose", "--conversation", conversation, "--body", "hello", "--json") + require.NoError(err) + var created chatDraftOutput + require.NoError(json.Unmarshal([]byte(createdJSON), &created)) + assert.Equal("msgvault", created.Location) + assert.Equal("C1", created.SourceConversationID) + listJSON, err := run("draft-get", "--conversation", conversation, "--json") + require.NoError(err) + var listed []chatDraftOutput + require.NoError(json.Unmarshal([]byte(listJSON), &listed)) + created.Status = "ok" + assert.Equal([]chatDraftOutput{created}, listed) + text, err := run("draft-get", created.DraftID) + require.NoError(err) + assert.Contains(text, "location=msgvault draft="+created.DraftID+" revision=1") + + _, err = run("draft-edit", created.DraftID, "--revision", "1", "--body", "edited") + require.NoError(err) + stale, err := run("draft-delete", created.DraftID, "--revision", "1") + require.Error(err) + assert.Contains(stale, "revision_mismatch") + recovered, err := run("draft-recover", created.DraftID, "--revision", "2") + require.Error(err) + assert.Contains(recovered, "not_supported") + _, err = run("draft-delete", created.DraftID, "--revision", "2") + require.NoError(err) + listJSON, err = run("draft-get", "--conversation", conversation, "--json") + require.NoError(err) + assert.JSONEq("[]", listJSON) +} + +func TestChatDraftDelegatedGrantIsSourceScoped(t *testing.T) { + assert, require := assert.New(t), require.New(t) + st := testutil.NewTestStore(t) + granted, err := st.GetOrCreateSource("slack", "granted") + require.NoError(err) + other, err := st.GetOrCreateSource("discord", "other") + require.NoError(err) + grantedConversation, err := st.EnsureConversationWithType(granted.ID, "C1", "channel", "granted") + require.NoError(err) + otherConversation, err := st.EnsureConversationWithType(other.ID, "D1", "channel", "other") + require.NoError(err) + adapter := &storeAPIAdapter{store: st} + grant := &agentgrant.Grant{ + ID: "chat-grant", Permissions: []agentgrant.Permission{agentgrant.PermissionDraftCreate}, + Sources: []agentgrant.SourceRef{{Type: granted.SourceType, Identifier: granted.Identifier}}, + } + run := func(args ...string) (string, error) { + var out strings.Builder + emit := func(event api.CLIRunEvent) error { out.WriteString(event.Data); return nil } + req := api.CLIRunRequest{Args: args, Grant: grant} + runner := adapter.runCLIDraftLifecycle + if args[0] == api.CLIRunDraftComposeCommand { + runner = adapter.runCLIComposeDraft + } + err := runner(t.Context(), req, emit) + return out.String(), err + } + + createdJSON, err := run(api.CLIRunDraftComposeCommand, "--conversation", strconv.FormatInt(grantedConversation, 10), "--body", "ok", "--json") + require.NoError(err) + var created chatDraftOutput + require.NoError(json.Unmarshal([]byte(createdJSON), &created)) + + owned, err := st.CreateChatDraftContext(t.Context(), otherConversation, 0, "private", func(string, string) error { return nil }) + require.NoError(err) + for _, args := range [][]string{ + {api.CLIRunDraftComposeCommand, "--conversation", strconv.FormatInt(otherConversation, 10), "--body", "x"}, + {api.CLIRunDraftGetCommand, "--conversation", strconv.FormatInt(otherConversation, 10)}, + {api.CLIRunDraftGetCommand, owned.DraftID}, + {api.CLIRunDraftGetCommand, "chat-draft-missing"}, + {api.CLIRunDraftGetCommand, created.DraftID}, + {api.CLIRunDraftGetCommand, "--conversation", strconv.FormatInt(grantedConversation, 10)}, + {api.CLIRunDraftEditCommand, created.DraftID, "--revision", "1", "--body", "needs draft.edit"}, + } { + out, err := run(args...) + require.Error(err, args) + assert.Equal("not_permitted", err.Error(), args) + assert.Empty(out, args) + } + unchanged, err := st.ListChatDraftsContext(t.Context(), otherConversation, func(string, string) error { return nil }) + require.NoError(err) + assert.Len(unchanged, 1) +} + +func TestChatDraftDeleteGrantRedactsBody(t *testing.T) { + assertions, requirements := assert.New(t), require.New(t) + st := testutil.NewTestStore(t) + source, err := st.GetOrCreateSource("slack", "slack-account") + requirements.NoError(err) + conversationID, err := st.EnsureConversationWithType(source.ID, "C1", "channel", "General") + requirements.NoError(err) + body := "private chat reply" + draft, err := st.CreateChatDraftContext(t.Context(), conversationID, 0, body, func(string, string) error { return nil }) + requirements.NoError(err) + grantFor := func(permission agentgrant.Permission) *agentgrant.Grant { + return &agentgrant.Grant{ + ID: "chat-grant", Permissions: []agentgrant.Permission{permission}, + Sources: []agentgrant.SourceRef{{Type: source.SourceType, Identifier: source.Identifier}}, + } + } + adapter := &storeAPIAdapter{store: st} + run := func(grant *agentgrant.Grant, args ...string) (string, error) { + var output strings.Builder + err := adapter.runCLIDraftLifecycle(t.Context(), api.CLIRunRequest{Args: args, Grant: grant}, func(event api.CLIRunEvent) error { + output.WriteString(event.Data) + return nil + }) + return output.String(), err + } + + withEdit, err := run(grantFor(agentgrant.PermissionDraftEdit), api.CLIRunDraftGetCommand, draft.DraftID, "--json") + requirements.NoError(err) + assertions.Contains(withEdit, body) + + for _, tc := range []struct { + name string + args []string + json bool + list bool + }{ + {name: "get text", args: []string{api.CLIRunDraftGetCommand, draft.DraftID}}, + {name: "get JSON", args: []string{api.CLIRunDraftGetCommand, draft.DraftID}, json: true}, + {name: "list text", args: []string{api.CLIRunDraftGetCommand, "--conversation", strconv.FormatInt(conversationID, 10)}, list: true}, + {name: "list JSON", args: []string{api.CLIRunDraftGetCommand, "--conversation", strconv.FormatInt(conversationID, 10)}, json: true, list: true}, + } { + t.Run(tc.name, func(t *testing.T) { + assert, require := assert.New(t), require.New(t) + args := append([]string(nil), tc.args...) + if tc.json { + args = append(args, "--json") + } + got, err := run(grantFor(agentgrant.PermissionDraftDelete), args...) + require.NoError(err) + assert.NotContains(got, body) + assert.Contains(got, draft.DraftID) + if tc.json && tc.list { + var outputs []chatDraftOutput + require.NoError(json.Unmarshal([]byte(got), &outputs)) + require.Len(outputs, 1) + assert.Empty(outputs[0].Body) + } else if tc.json { + var output chatDraftOutput + require.NoError(json.Unmarshal([]byte(got), &output)) + assert.Empty(output.Body) + } + }) + } + + deleted, err := run(grantFor(agentgrant.PermissionDraftDelete), api.CLIRunDraftDeleteCommand, draft.DraftID, "--revision", "1", "--json") + requirements.NoError(err) + assertions.NotContains(deleted, body) + var deletedJSON chatDraftOutput + requirements.NoError(json.Unmarshal([]byte(deleted), &deletedJSON)) + assertions.Equal("deleted", deletedJSON.Status) + assertions.Empty(deletedJSON.Body) + _, err = st.GetChatDraftContext(t.Context(), draft.DraftID) + requirements.Error(err) + + textDraft, err := st.CreateChatDraftContext(t.Context(), conversationID, 0, body, func(string, string) error { return nil }) + requirements.NoError(err) + deletedText, err := run(grantFor(agentgrant.PermissionDraftDelete), api.CLIRunDraftDeleteCommand, textDraft.DraftID, "--revision", "1") + requirements.NoError(err) + assertions.Contains(deletedText, textDraft.DraftID) + assertions.NotContains(deletedText, body) +} diff --git a/cmd/msgvault/cmd/serve_compose_draft.go b/cmd/msgvault/cmd/serve_compose_draft.go new file mode 100644 index 000000000..21748328c --- /dev/null +++ b/cmd/msgvault/cmd/serve_compose_draft.go @@ -0,0 +1,189 @@ +package cmd + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + "time" + + "go.kenn.io/msgvault/internal/api" + imaplib "go.kenn.io/msgvault/internal/imap" + "go.kenn.io/msgvault/internal/sourceops" +) + +type draftComposeIntent struct { + From string + Account string + SourceID int64 + SourceIDSet bool + To []string + Cc []string + Bcc []string + Subject string + Body string + JSON bool + // ConversationID selects a local chat draft instead of an IMAP draft. + ConversationID int64 + ReplyTo int64 +} + +func invalidDraftComposeArgs(format string, args ...any) (draftComposeIntent, error) { + return draftComposeIntent{}, draftReplyError("invalid_args", fmt.Errorf(format, args...)) +} + +func parseDraftComposeArgs(args []string) (draftComposeIntent, error) { + if !api.IsCLIRunDraftCompose(args) { + return invalidDraftComposeArgs("expected %s as the first argument", api.CLIRunDraftComposeCommand) + } + var intent draftComposeIntent + var fromSet, accountSet, sourceIDSet, subjectSet, bodySet, jsonSet bool + rest := args[1:] + for len(rest) > 0 { + arg := rest[0] + rest = rest[1:] + nameValue, ok := strings.CutPrefix(arg, "--") + if !ok { + return invalidDraftComposeArgs("draft-compose accepts flags only") + } + name, value, hasValue := strings.Cut(nameValue, "=") + switch name { + case draftFromFlag, "account", "source-id", "subject", "body", "to", "cc", "bcc", "conversation", "reply-to": + if !hasValue { + if len(rest) == 0 { + return invalidDraftComposeArgs("--%s requires a value", name) + } + value, rest = rest[0], rest[1:] + } + switch name { + case draftFromFlag: + if fromSet { + return invalidDraftComposeArgs("--from given more than once") + } + if strings.TrimSpace(value) == "" { + return invalidDraftComposeArgs("--from must not be empty") + } + intent.From, fromSet = value, true + case "account": + if accountSet || strings.TrimSpace(value) == "" { + return invalidDraftComposeArgs("--account must be given once with a value") + } + intent.Account, accountSet = strings.TrimSpace(value), true + case "source-id": + if sourceIDSet { + return invalidDraftComposeArgs("--source-id given more than once") + } + id, parseErr := strconv.ParseInt(strings.TrimSpace(value), 10, 64) + if parseErr != nil || id <= 0 { + return invalidDraftComposeArgs("source ID must be a positive integer") + } + intent.SourceID, intent.SourceIDSet, sourceIDSet = id, true, true + case "conversation", "reply-to": + id, parseErr := strconv.ParseInt(strings.TrimSpace(value), 10, 64) + if parseErr != nil || id <= 0 { + return invalidDraftComposeArgs("--%s must be a positive integer", name) + } + target := &intent.ReplyTo + if name == "conversation" { + target = &intent.ConversationID + } + if *target != 0 { + return invalidDraftComposeArgs("--%s given more than once", name) + } + *target = id + case "subject": + if subjectSet { + return invalidDraftComposeArgs("--subject given more than once") + } + intent.Subject, subjectSet = value, true + case "body": + if bodySet { + return invalidDraftComposeArgs("--body given more than once") + } + intent.Body, bodySet = value, true + case "to": + if strings.TrimSpace(value) == "" { + return invalidDraftComposeArgs("--to must not be empty") + } + intent.To = append(intent.To, value) + case "cc": + if strings.TrimSpace(value) == "" { + return invalidDraftComposeArgs("--cc must not be empty") + } + intent.Cc = append(intent.Cc, value) + case "bcc": + if strings.TrimSpace(value) == "" { + return invalidDraftComposeArgs("--bcc must not be empty") + } + intent.Bcc = append(intent.Bcc, value) + } + case "json": + if jsonSet || (hasValue && value != "true") { + return invalidDraftComposeArgs("--json accepts one flag without a value") + } + intent.JSON, jsonSet = true, true + case "log-level", "verbose", "log-sql", "log-sql-slow-ms": + if !hasValue && name != "verbose" && name != "log-sql" && len(rest) > 0 { + rest = rest[1:] + } + default: + return invalidDraftComposeArgs("unknown flag --%s", name) + } + } + if intent.ConversationID != 0 { + if fromSet || accountSet || sourceIDSet || subjectSet || len(intent.To)+len(intent.Cc)+len(intent.Bcc) > 0 { + return invalidDraftComposeArgs("--conversation accepts only --body, --reply-to, and --json") + } + return intent, nil + } + if intent.ReplyTo != 0 { + return invalidDraftComposeArgs("--reply-to requires --conversation") + } + if !accountSet && !sourceIDSet { + return invalidDraftComposeArgs("--account or --source-id is required") + } + if accountSet && sourceIDSet { + return invalidDraftComposeArgs("--account and --source-id are mutually exclusive") + } + if len(intent.To)+len(intent.Cc)+len(intent.Bcc) == 0 { + return invalidDraftComposeArgs("at least one of --to, --cc, or --bcc is required") + } + return intent, nil +} + +func (a *storeAPIAdapter) runCLIComposeDraft( + ctx context.Context, + req api.CLIRunRequest, + emit func(api.CLIRunEvent) error, +) error { + if len(req.Env) != 0 || req.Cwd != "" { + return draftReplyError("invalid_args", errors.New("draft-compose accepts no environment or working directory")) + } + intent, err := parseDraftComposeArgs(req.Args) + if err != nil { + return err + } + if intent.ConversationID != 0 { + return a.runCLIChatDraftCreate(ctx, intent, req.Grant, emit) + } + if source, err := sourceops.ResolveExactOne(a.store, sourceops.Selector{ + Account: intent.Account, SourceID: intent.SourceID, SourceIDSet: intent.SourceIDSet, + }); err == nil && source.SourceType == "beeper" { + return a.runBeeperDraftCreate(ctx, req.Grant, intent, source, emit) + } + target, from, _, err := a.resolveDraftTarget( + ctx, nil, intent.Account, intent.SourceID, intent.SourceIDSet, intent.From, req.Grant, + ) + if err != nil { + return err + } + draft, err := imaplib.BuildCompose(imaplib.ComposeOptions{ + From: from, To: intent.To, Cc: intent.Cc, Bcc: intent.Bcc, + Subject: intent.Subject, Body: intent.Body, + }, time.Now(), "") + if err != nil { + return draftReplyError("invalid_compose_metadata", err) + } + return a.createDraft(ctx, target, draft, intent.JSON, emit) +} diff --git a/cmd/msgvault/cmd/serve_compose_draft_test.go b/cmd/msgvault/cmd/serve_compose_draft_test.go new file mode 100644 index 000000000..69bf19105 --- /dev/null +++ b/cmd/msgvault/cmd/serve_compose_draft_test.go @@ -0,0 +1,158 @@ +package cmd + +import ( + "bufio" + "bytes" + "encoding/json" + "log/slog" + "net/http" + "net/http/httptest" + "strconv" + "testing" + + emersionimap "github.com/emersion/go-imap/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/testutil" +) + +func TestDraftComposeArgs(t *testing.T) { + assertions := assert.New(t) + requirements := require.New(t) + intent, err := parseDraftComposeArgs([]string{ + "draft-compose", "--source-id", "42", "--from", "owner@example.test", + "--to", "to@example.test", "--cc=copy@example.test", "--bcc", "hidden@example.test", + "--subject", "Subject", "--body=body", "--json", + }) + requirements.NoError(err) + assertions.Equal(int64(42), intent.SourceID) + assertions.Equal([]string{"to@example.test"}, intent.To) + assertions.Equal([]string{"copy@example.test"}, intent.Cc) + assertions.Equal([]string{"hidden@example.test"}, intent.Bcc) + assertions.True(intent.JSON) + + for _, args := range [][]string{ + {"draft-compose", "--source-id", "0", "--to", "to@example.test"}, + {"draft-compose", "--to", "to@example.test"}, + {"draft-compose", "--source-id", "42"}, + {"draft-compose", "--source-id", "42", "--account", "owner@example.test", "--to", "to@example.test"}, + } { + _, err := parseDraftComposeArgs(args) + assertions.Error(err) + } +} + +func TestDraftComposeEndToEnd(t *testing.T) { + assertions := assert.New(t) + requirements := require.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + events := make([]api.CLIRunEvent, 0, 1) + err := adapter.runCLIComposeDraft(t.Context(), api.CLIRunRequest{Args: []string{ + "draft-compose", "--source-id", strconv.FormatInt(fixture.source.ID, 10), + "--from", testutil.IMAPTestUsername, + "--to", "to@example.test", "--cc", "copy@example.test", "--bcc", "hidden@example.test", + "--subject", "Compose subject", "--body", "compose body", "--json", + }}, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + requirements.NoError(err) + requirements.Len(events, 1) + var result draftReplyOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &result)) + assertions.Equal(draftReplyStatusCreated, result.Status) + assertions.Equal(int64(1), result.Revision) + + message, err := fixture.store.GetMessage(result.MessageID) + requirements.NoError(err) + assertions.Equal([]string{"to@example.test"}, message.To) + assertions.Equal([]string{"copy@example.test"}, message.Cc) + assertions.Equal([]string{"hidden@example.test"}, message.Bcc) + raw, err := fixture.store.GetMessageRaw(result.MessageID) + requirements.NoError(err) + assertions.Contains(string(raw), "Bcc:") + assertions.Contains(string(raw), "hidden@example.test") + + // The backends tokenize punctuation in full email queries differently. + matches, total, err := fixture.store.SearchMessages("copy", 0, 10) + requirements.NoError(err) + requirements.Equal(int64(1), total) + requirements.Len(matches, 1) + assertions.Equal(result.MessageID, matches[0].ID) +} + +func TestDraftComposeHTTPPublishesManagedDraft(t *testing.T) { + assertions := assert.New(t) + requirements := require.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + server := httptest.NewServer(api.NewServerWithOptions(api.ServerOptions{ + Config: &config.Config{ + HomeDir: t.TempDir(), + Server: config.ServerConfig{APIKey: "owner-test-key"}, + }, + Store: adapter, + Logger: slog.New(slog.DiscardHandler), + }).Router()) + t.Cleanup(server.Close) + + args := []string{ + "draft-compose", "--source-id", strconv.FormatInt(fixture.source.ID, 10), + "--from", testutil.IMAPTestUsername, "--to", "to@example.test", + "--cc", "copy@example.test", "--bcc", "hidden@example.test", + "--subject", "Compose subject", "--body", "compose body", "--json", + } + body, err := json.Marshal(map[string]any{"args": args}) + requirements.NoError(err) + request, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/cli/run", bytes.NewReader(body)) + requirements.NoError(err) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("X-Api-Key", "owner-test-key") + response, err := http.DefaultClient.Do(request) + requirements.NoError(err) + defer func() { _ = response.Body.Close() }() + requirements.Equal(http.StatusOK, response.StatusCode) + + var events []api.CLIRunEvent + scanner := bufio.NewScanner(response.Body) + for scanner.Scan() { + var event api.CLIRunEvent + requirements.NoError(json.Unmarshal(scanner.Bytes(), &event)) + events = append(events, event) + } + requirements.NoError(scanner.Err()) + requirements.Len(events, 2) + requirements.Equal(cliStreamStdout, events[0].Type) + requirements.Equal("complete", events[1].Type) + + var result draftReplyOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &result)) + assertions.Equal(draftReplyStatusCreated, result.Status) + assertions.Equal(fixture.source.ID, result.SourceID) + assertions.Equal("Drafts", result.Mailbox) + assertions.NotZero(result.UID) + assertions.NotZero(result.UIDValidity) + assertions.Equal(int64(1), result.Revision) + + draft, err := fixture.store.GetIMAPDraft(result.DraftID) + requirements.NoError(err) + assertions.Equal(result.MessageID, draft.CurrentMessageID) + assertions.Equal(result.UID, draft.CurrentReceipt.UID) + assertions.Equal(result.UIDValidity, draft.CurrentReceipt.UIDValidity) + message, err := fixture.store.GetMessage(result.MessageID) + requirements.NoError(err) + assertions.Equal([]string{"to@example.test"}, message.To) + assertions.Equal([]string{"copy@example.test"}, message.Cc) + assertions.Equal([]string{"hidden@example.test"}, message.Bcc) + storedRaw, err := fixture.store.GetMessageRaw(result.MessageID) + requirements.NoError(err) + assertions.Contains(string(storedRaw), "Bcc:") + assertions.Contains(string(storedRaw), "hidden@example.test") + + flags, fetchedRaw := fetchDraftMailboxMessage(t, fixture.config, draft.CurrentReceipt) + assertions.Contains(flags, emersionimap.FlagDraft) + assertions.Equal(storedRaw, fetchedRaw) +} diff --git a/cmd/msgvault/cmd/serve_config.go b/cmd/msgvault/cmd/serve_config.go new file mode 100644 index 000000000..d61f6f6b8 --- /dev/null +++ b/cmd/msgvault/cmd/serve_config.go @@ -0,0 +1,62 @@ +package cmd + +import ( + "context" + "errors" + "strconv" + + "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/config" +) + +func addServeConfigFlags(cmd *cobra.Command) { + cmd.Flags().String("bind", "", "Bind address or iface:NAME (overrides environment and config)") + cmd.Flags().Int("port", 0, "HTTP API port (0 chooses an open port; overrides environment and config)") +} + +func serveRuntimeOverrides(cmd *cobra.Command) config.RuntimeOverrides { + var overrides config.RuntimeOverrides + if cmd.Name() != "serve" { + return overrides + } + if flag := cmd.Flags().Lookup("bind"); flag != nil && flag.Changed { + value, _ := cmd.Flags().GetString("bind") // Cobra has validated the declared string flag. + overrides.BindAddr = &value + } + if flag := cmd.Flags().Lookup("port"); flag != nil && flag.Changed { + value, _ := cmd.Flags().GetInt("port") // Cobra has validated the declared integer flag. + overrides.APIPort = &value + } + return overrides +} + +func resolveServeBind(address string) (string, error) { + return config.ResolveBindAddress(address) +} + +func prepareServeConfig(cfg *config.Config) error { + if cfg == nil { + return errors.New("configuration is unavailable") + } + if _, err := cfg.ResolveServerBindAddress(); err != nil { + return err + } + return cfg.ValidateServerKey() +} + +// daemonRuntimeChildEnv keeps serve flags effective when children reload config. +func daemonRuntimeChildEnv(ctx context.Context, env []string) []string { + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil { + return env + } + bind := state.cfg.Server.BindAddr + if bind == "" { + bind = defaultDaemonBindAddr + } + // exec.Cmd uses the last value for a duplicate environment key. + return append(env, + "MSGVAULT_BIND_ADDR="+bind, + "MSGVAULT_API_PORT="+strconv.Itoa(state.cfg.Server.APIPort), + ) +} diff --git a/cmd/msgvault/cmd/serve_config_test.go b/cmd/msgvault/cmd/serve_config_test.go new file mode 100644 index 000000000..a74023df8 --- /dev/null +++ b/cmd/msgvault/cmd/serve_config_test.go @@ -0,0 +1,247 @@ +package cmd + +import ( + "net" + "os" + "os/exec" + "path/filepath" + "runtime" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/config" +) + +func TestServeFlagsOverrideMalformedEnvironment(t *testing.T) { //nolint:paralleltest // process environment and logger + assert := assert.New(t) + require := require.New(t) + t.Setenv("MSGVAULT_BIND_ADDR", "") + t.Setenv("MSGVAULT_API_PORT", "invalid") + root := newRootCommand() + var got *config.Config + leaf := &cobra.Command{Use: "serve", RunE: func(cmd *cobra.Command, _ []string) error { + got = invocationFromCommand(cmd).cfg + return nil + }} + addServeConfigFlags(leaf) + root.AddCommand(leaf) + root.SetArgs([]string{"--home", t.TempDir(), "serve", "--bind", "127.0.0.1", "--port", "8181"}) + require.NoError(root.Execute()) + require.NotNil(got) + assert.Equal("127.0.0.1", got.Server.BindAddr) + assert.Equal(8181, got.Server.APIPort) + assert.Equal("--bind", got.BindAddressSource()) + if inv := invocationFromCommand(root); inv != nil && inv.logResult != nil { + inv.logResult.Close() + } +} + +func TestDaemonSubprocessesPreserveEffectiveServeConfig(t *testing.T) { //nolint:paralleltest // process environment and executable resolver + requirements := require.New(t) + repoRoot, err := filepath.Abs(filepath.Join("..", "..", "..")) + requirements.NoError(err) + binaryName := "msgvault" + if runtime.GOOS == "windows" { + binaryName += ".exe" + } + binary := filepath.Join(t.TempDir(), binaryName) + build := exec.Command("go", "build", "-tags", "fts5 sqlite_vec", "-o", binary, "./cmd/msgvault") + build.Dir = repoRoot + output, err := build.CombinedOutput() + requirements.NoError(err, "build real msgvault binary: %s", output) + savedResolver := daemonCLIExecutableResolver + daemonCLIExecutableResolver = func() (string, error) { return binary, nil } + t.Cleanup(func() { daemonCLIExecutableResolver = savedResolver }) + + home := t.TempDir() + requirements.NoError(os.WriteFile(filepath.Join(home, "config.toml"), []byte("[server]\ndaemon_auto_start = false\n[analytics]\nengine = 'sql'\n"), 0o600)) + t.Setenv("MSGVAULT_API_PORT", "invalid") + t.Setenv("MSGVAULT_BIND_ADDR", "") + t.Setenv("MSGVAULT_REMOTE_URL", "") + cfg, err := config.LoadWithOverrides("", home, config.RuntimeOverrides{BindAddr: new("127.0.0.1"), APIPort: new(8181)}) + requirements.NoError(err) + requirements.NoError(prepareServeConfig(cfg)) + for _, tc := range []struct { + name, bind string + cache bool + }{ + {"cli flags", "127.0.0.1", false}, + {"cache flags", "127.0.0.1", true}, + {"cli default bind", "", false}, + {"cache default bind", "", true}, + } { + t.Run(tc.name, func(t *testing.T) { + require := require.New(t) + childConfig := *cfg + childConfig.Server.BindAddr = tc.bind + ctx := testInvocationContext(t.Context(), &childConfig, invocationOptions{homeDir: home}) + var command *exec.Cmd + var err error + if !tc.cache { + command, err = newDaemonCLISubprocessCommand(ctx, []string{"repair-encoding"}, nil, "") + } else { + command, err = newBuildCacheSubprocessCommand(ctx, buildCacheModeDefault) + require.NoError(err) + // os.Executable returns the test runner here; keep the production + // command's arguments and environment but run the branch binary. + command.Path, command.Args[0] = binary, binary + } + require.NoError(err) + output, err := command.CombinedOutput() + require.NoError(err, "%s", output) + }) + } +} + +func TestServeBindSourceUsesConfiguredField(t *testing.T) { + t.Parallel() + path := filepath.Join(t.TempDir(), "config.toml") + for _, tc := range []struct { + content string + want string + }{ + {"[web]\ntheme = 'dark'\n", "default"}, + {"[server]\nbind_addr = '127.0.0.1'\n", path}, + } { + require.NoError(t, os.WriteFile(path, []byte(tc.content), 0o600)) + cfg, err := config.Load(path, "") + require.NoError(t, err) + assert.Equal(t, tc.want, cfg.BindAddressSource()) + } +} + +func TestServeResolvesRealLoopbackInterface(t *testing.T) { + t.Parallel() + assert := assert.New(t) + require := require.New(t) + interfaces, err := net.Interfaces() + require.NoError(err) + var name string + for _, iface := range interfaces { + if iface.Flags&net.FlagLoopback != 0 && iface.Flags&net.FlagUp != 0 { + name = iface.Name + break + } + } + if name == "" { + t.Skip("no active loopback interface") + } + bind, err := resolveServeBind("iface:" + name) + require.NoError(err) + require.NotNil(net.ParseIP(bind)) + assert.True(net.ParseIP(bind).IsLoopback()) + listener, err := listenServeAPI("iface:"+name, 0) + require.NoError(err) + t.Cleanup(func() { assert.NoError(listener.Close()) }) + addr, ok := listener.Addr().(*net.TCPAddr) + require.True(ok, "listener address should be TCP") + assert.True(addr.IP.IsLoopback()) +} + +func TestServeSavePreservesInterfaceSelector(t *testing.T) { + t.Parallel() + require := require.New(t) + assert := assert.New(t) + interfaces, err := net.Interfaces() + require.NoError(err) + var name string + for _, iface := range interfaces { + if iface.Flags&net.FlagLoopback != 0 && iface.Flags&net.FlagUp != 0 { + name = iface.Name + break + } + } + if name == "" { + t.Skip("no active loopback interface") + } + + selector := "iface:" + name + cfg := config.NewDefaultConfig() + cfg.HomeDir = t.TempDir() + cfg.Data.DataDir = cfg.HomeDir + cfg.Server.BindAddr = selector + require.NoError(prepareServeConfig(cfg)) + assert.NotEqual(selector, cfg.Server.BindAddr, "runtime bind should be resolved") + cfg.Accounts = []config.AccountSchedule{{Email: "person@example.com", Enabled: true}} + require.NoError(cfg.Save()) + + reloaded, err := config.Load("", cfg.HomeDir) + require.NoError(err) + assert.Equal(selector, reloaded.Server.BindAddr, "saved config should retain the interface selector") +} + +func TestServeUnknownInterfaceFailsBeforeMinting(t *testing.T) { + t.Parallel() + cfg := config.NewDefaultConfig() + cfg.HomeDir = t.TempDir() + cfg.Server.BindAddr = "iface:msgvault-nonexistent-test-interface" + cfg.Data.DataDir = cfg.HomeDir + require.Error(t, prepareServeConfig(cfg)) + _, err := os.Stat(cfg.ServerKeyFilePath()) + require.ErrorIs(t, err, os.ErrNotExist) +} + +func TestServeEnvironmentOnlyMintsAndReusesKey(t *testing.T) { //nolint:paralleltest // process environment + assert := assert.New(t) + require := require.New(t) + home := t.TempDir() + t.Setenv("MSGVAULT_BIND_ADDR", "0.0.0.0") + cfg, err := config.Load("", home) + require.NoError(err) + clientConfig, err := config.Load("", home) + require.NoError(err) + require.NoError(clientConfig.ResolveServerKey()) + assert.Empty(clientConfig.Server.AuthenticationKey()) + require.NoError(prepareServeConfig(cfg)) + assert.Empty(cfg.Server.AuthenticationKey(), "replacement validation must leave key creation to startup") + _, err = os.Stat(cfg.ServerKeyFilePath()) + require.ErrorIs(err, os.ErrNotExist) + require.NoError(cfg.PrepareServerKey()) + key := cfg.Server.AuthenticationKey() + assert.Len(key, 43) + assert.Empty(cfg.Server.APIKey, "runtime keys stay out of TOML") + require.NoError(cfg.Server.ValidateSecure()) + restarted, err := config.Load("", home) + require.NoError(err) + require.NoError(prepareServeConfig(restarted)) + assert.Equal(key, restarted.Server.AuthenticationKey()) + _, err = os.Stat(filepath.Join(home, "config.toml")) + require.ErrorIs(err, os.ErrNotExist) + // An already-loaded client sees a key minted by its child daemon. + rt := &DaemonRuntime{} + rt.Record.Metadata = map[string]string{runtimeAuthFingerprint: daemonAPIKeyFingerprint(key)} + require.NoError(localDaemonAuthIdentityError("http://127.0.0.1:8080", rt, clientConfig)) + assert.Equal(key, clientConfig.Server.AuthenticationKey()) +} + +func TestServeExplicitSecretFailureDoesNotMint(t *testing.T) { + t.Parallel() + cfg := config.NewDefaultConfig() + cfg.HomeDir = t.TempDir() + cfg.Server.BindAddr = "0.0.0.0" + cfg.Server.APIKeyFile = filepath.Join(cfg.HomeDir, "missing-key") + cfg.Data.DataDir = cfg.HomeDir + require.Error(t, prepareServeConfig(cfg)) + _, err := os.Stat(cfg.ServerKeyFilePath()) + require.ErrorIs(t, err, os.ErrNotExist) +} + +func TestServeExplicitInsecureDoesNotReadOrMintDefaultKey(t *testing.T) { + t.Parallel() + assert := assert.New(t) + require := require.New(t) + cfg := config.NewDefaultConfig() + cfg.HomeDir = t.TempDir() + cfg.Server.BindAddr = "0.0.0.0" + cfg.Server.AllowInsecure = true + cfg.Data.DataDir = cfg.HomeDir + require.NoError(os.MkdirAll(filepath.Dir(cfg.ServerKeyFilePath()), 0o700)) + require.NoError(os.WriteFile(cfg.ServerKeyFilePath(), nil, 0o600)) + require.NoError(prepareServeConfig(cfg)) + assert.Empty(cfg.Server.AuthenticationKey()) + data, err := os.ReadFile(cfg.ServerKeyFilePath()) + require.NoError(err) + assert.Empty(data) +} diff --git a/cmd/msgvault/cmd/serve_deletions_test.go b/cmd/msgvault/cmd/serve_deletions_test.go index 6acac6951..26bd39a30 100644 --- a/cmd/msgvault/cmd/serve_deletions_test.go +++ b/cmd/msgvault/cmd/serve_deletions_test.go @@ -1,7 +1,6 @@ package cmd import ( - "context" "testing" "github.com/stretchr/testify/assert" @@ -12,18 +11,22 @@ import ( ) func TestStoreAPIAdapterDeletionManifests(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) cfg = &config.Config{Data: config.DataConfig{DataDir: t.TempDir()}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx - adapter := &storeAPIAdapter{} + adapter := &storeAPIAdapter{config: cfg} var _ api.DeletionManifestLister = adapter var _ api.DeletionManifestCanceller = adapter - ctx := context.Background() + ctx := testCtx // Save through the existing saver path. m := deletion.NewManifest("adapter test", []string{"gm-1"}) diff --git a/cmd/msgvault/cmd/serve_draft_attachments.go b/cmd/msgvault/cmd/serve_draft_attachments.go new file mode 100644 index 000000000..7bebba4f3 --- /dev/null +++ b/cmd/msgvault/cmd/serve_draft_attachments.go @@ -0,0 +1,106 @@ +package cmd + +import ( + "context" + "errors" + "fmt" + "strings" + + "go.kenn.io/msgvault/internal/attachmentpolicy" + imaplib "go.kenn.io/msgvault/internal/imap" + msgmime "go.kenn.io/msgvault/internal/mime" + "go.kenn.io/msgvault/internal/store" +) + +// forwardDraftAttachmentWrites carries a generated forward's catalog rows onto +// its replacement message. It returns nil for other drafts, which carry no +// attachments. +func (a *storeAPIAdapter) forwardDraftAttachmentWrites(ctx context.Context, currentMessageID int64, replacement imaplib.ReplyDraft) ([]store.AttachmentWrite, error) { + if !imaplib.IsGeneratedForward(replacement.Raw) { + return nil, nil + } + refs, err := a.store.MessageMIMEAttachmentsContext(ctx, currentMessageID) + if err != nil { + return nil, err + } + return prepareIMAPDraftAttachmentWrites(ctx, replacement.Parsed, refs) +} + +// matchForwardAttachmentRef prefers the first unused keyed row with the part's +// metadata, then a keyless legacy row. Part keys are not compared: a forward +// numbers its parts differently from the message its rows came from. +func matchForwardAttachmentRef(part msgmime.Attachment, refs []store.AttachmentRef, used []bool) int { + named, legacy := -1, -1 + for i, ref := range refs { + if !strings.EqualFold(ref.ContentHash, part.ContentHash) { + continue + } + if ref.SourcePartKey != "" { + // Keyed rows record one occurrence each, with its own Content-ID. + if !used[i] && ref.Filename == part.Filename && ref.ContentID == part.ContentID { + return i + } + continue + } + // Keyless legacy rows are unique per hash, keep only the first part's + // metadata and may lack a Content-ID, so one may back several identical parts. + if named < 0 && ref.Filename == part.Filename && (ref.ContentID == "" || ref.ContentID == part.ContentID) { + named = i + } + if legacy < 0 { + legacy = i + } + } + if named >= 0 { + return named + } + return legacy +} + +// prepareIMAPDraftAttachmentWrites binds generated MIME occurrences to the +// catalog references that already own their bytes. A hash by itself never +// creates a new catalog reference. +func prepareIMAPDraftAttachmentWrites(ctx context.Context, parsed *msgmime.Message, refs []store.AttachmentRef) ([]store.AttachmentWrite, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if parsed == nil { + return nil, errors.New("parse generated draft before preparing attachments") + } + writes := make([]store.AttachmentWrite, 0, len(parsed.Attachments)) + used := make([]bool, len(refs)) + for _, part := range msgmime.DistinctAttachments(parsed.Attachments) { + if part.Size == 0 { + continue // sync stores no row for an empty part + } + refIndex := matchForwardAttachmentRef(part, refs, used) + if refIndex < 0 { + return nil, fmt.Errorf("generated attachment %q (%s) has no retained catalog reference", part.Filename, part.PartKey) + } + ref := refs[refIndex] + used[refIndex] = true + if ref.State != "" && ref.State != attachmentpolicy.StateStored { + return nil, fmt.Errorf("attachment %q is %s", ref.Filename, ref.State) + } + role, roleSource := store.AttachmentRoleFromMIME(part.Disposition, part.IsInline, part.ContentID) + writes = append(writes, store.AttachmentWrite{ + Filename: part.Filename, + MIMEType: part.ContentType, + StoragePath: ref.StoragePath, + ContentHash: strings.ToLower(ref.ContentHash), + Size: int64(part.Size), + MediaType: ref.MediaType, + Width: ref.Width, + Height: ref.Height, + DurationMS: ref.DurationMS, + Metadata: ref.Metadata, + Role: role, + RoleSource: roleSource, + SourcePartKey: part.PartKey, + ContentID: part.ContentID, + State: ref.State, + SkipReason: ref.SkipReason, + }) + } + return writes, nil +} diff --git a/cmd/msgvault/cmd/serve_draft_lifecycle.go b/cmd/msgvault/cmd/serve_draft_lifecycle.go new file mode 100644 index 000000000..954424547 --- /dev/null +++ b/cmd/msgvault/cmd/serve_draft_lifecycle.go @@ -0,0 +1,1360 @@ +package cmd + +import ( + "context" + "database/sql" + jsonv2 "encoding/json/v2" + "errors" + "fmt" + "strconv" + "strings" + "time" + "unicode/utf8" + + "go.kenn.io/msgvault/internal/agentgrant" + "go.kenn.io/msgvault/internal/api" + imaplib "go.kenn.io/msgvault/internal/imap" + msgmime "go.kenn.io/msgvault/internal/mime" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/textutil" +) + +type draftLifecycleIntent struct { + Operation string + DraftID string + Revision int64 + Body string + JSON bool + MetadataOnly bool // Set by delegated authorization, never by CLI arguments. + // ConversationID lists a conversation's local chat drafts in place of DraftID. + ConversationID int64 +} + +const draftLifecycleActive = "active" + +type draftLifecycleReceipt struct { + Mailbox string `json:"mailbox"` + UIDValidity uint32 `json:"uidvalidity"` + UID uint32 `json:"uid"` +} + +type draftLifecycleObservation struct { + State string `json:"state"` + Code string `json:"code,omitempty"` + Mailbox string `json:"mailbox,omitempty"` + UIDValidity uint32 `json:"uidvalidity,omitempty"` + UID uint32 `json:"uid,omitempty"` + Flags []string `json:"flags,omitempty"` + Present bool `json:"present"` + Draft bool `json:"draft"` + Deleted bool `json:"deleted"` + Complete bool `json:"complete"` + UIDPlus bool `json:"uidplus"` +} + +type draftLifecycleOutput struct { + Status string `json:"status"` + DraftID string `json:"draft_id"` + Revision int64 `json:"revision"` + Lifecycle string `json:"lifecycle"` + MessageID int64 `json:"message_id"` + SourceID int64 `json:"source_id"` + Receipt draftLifecycleReceipt `json:"receipt"` + Content string `json:"content,omitempty"` + RawMIME string `json:"raw_mime,omitempty"` + CandidateContent string `json:"candidate_content,omitempty"` + PendingOperation string `json:"pending_operation,omitempty"` + PendingCode string `json:"pending_code,omitempty"` + RefusalCode string `json:"refusal_code,omitempty"` + PendingReceipt *draftLifecycleReceipt `json:"pending_receipt,omitempty"` + ProviderObservation *draftLifecycleObservation `json:"provider_observation,omitempty"` + Observation *draftLifecycleObservation `json:"observation,omitempty"` + ManualReconciliation bool `json:"manual_reconciliation,omitempty"` +} + +func parseDraftLifecycleArgs(args []string) (draftLifecycleIntent, error) { + if !api.IsCLIRunDraftLifecycle(args) { + return draftLifecycleIntent{}, draftReplyError("invalid_args", errors.New("expected a draft lifecycle command")) + } + intent := draftLifecycleIntent{Operation: args[0]} + var positional string + var revisionSet, bodySet, jsonSet bool + rest := args[1:] + for len(rest) > 0 { + arg := rest[0] + rest = rest[1:] + nameValue, ok := strings.CutPrefix(arg, "--") + if !ok { + if positional != "" { + return draftLifecycleIntent{}, draftReplyError("invalid_args", errors.New("expected exactly one draft ID")) + } + positional = arg + continue + } + name, value, hasValue := strings.Cut(nameValue, "=") + switch name { + case "revision": + if !hasValue { + if len(rest) == 0 { + return draftLifecycleIntent{}, draftReplyError("invalid_args", errors.New("--revision requires a value")) + } + value, rest = rest[0], rest[1:] + } + if revisionSet { + return draftLifecycleIntent{}, draftReplyError("invalid_args", errors.New("--revision given more than once")) + } + parsed, err := strconv.ParseInt(value, 10, 64) + if err != nil || parsed <= 0 { + return draftLifecycleIntent{}, draftReplyError("invalid_args", errors.New("--revision must be a positive integer")) + } + intent.Revision, revisionSet = parsed, true + case "body": + if !hasValue { + if len(rest) == 0 { + return draftLifecycleIntent{}, draftReplyError("invalid_args", errors.New("--body requires a value")) + } + value, rest = rest[0], rest[1:] + } + if bodySet { + return draftLifecycleIntent{}, draftReplyError("invalid_args", errors.New("--body given more than once")) + } + intent.Body, bodySet = value, true + case "conversation": + if !hasValue { + if len(rest) == 0 { + return draftLifecycleIntent{}, draftReplyError("invalid_args", errors.New("--conversation requires a value")) + } + value, rest = rest[0], rest[1:] + } + parsed, err := strconv.ParseInt(value, 10, 64) + if err != nil || parsed <= 0 || intent.ConversationID != 0 || intent.Operation != api.CLIRunDraftGetCommand { + return draftLifecycleIntent{}, draftReplyError("invalid_args", errors.New("draft-get accepts one positive --conversation")) + } + intent.ConversationID = parsed + case "json": + if jsonSet || (hasValue && value != "true") { + return draftLifecycleIntent{}, draftReplyError("invalid_args", errors.New("--json accepts one flag without a value")) + } + intent.JSON, jsonSet = true, true + case "log-level", "verbose", "log-sql", "log-sql-slow-ms": + if !hasValue && name != "verbose" && name != "log-sql" && len(rest) > 0 { + rest = rest[1:] + } + default: + return draftLifecycleIntent{}, draftReplyError("invalid_args", fmt.Errorf("unknown flag --%s", name)) + } + } + if intent.ConversationID != 0 { + if positional != "" || revisionSet || bodySet { + return draftLifecycleIntent{}, draftReplyError("invalid_args", errors.New("draft-get --conversation accepts only --json")) + } + return intent, nil + } + if positional == "" || !utf8.ValidString(positional) || strings.TrimSpace(positional) == "" || strings.ContainsAny(positional, "\x00\r\n") { + return draftLifecycleIntent{}, draftReplyError("invalid_args", errors.New("draft ID is required")) + } + intent.DraftID = positional + switch intent.Operation { + case api.CLIRunDraftGetCommand: + if revisionSet || bodySet { + return draftLifecycleIntent{}, draftReplyError("invalid_args", errors.New("draft-get accepts only --json")) + } + case api.CLIRunDraftEditCommand: + if !revisionSet || !bodySet { + return draftLifecycleIntent{}, draftReplyError("invalid_args", errors.New("draft-edit requires --revision and --body")) + } + case api.CLIRunDraftDeleteCommand: + if !revisionSet || bodySet { + return draftLifecycleIntent{}, draftReplyError("invalid_args", errors.New("draft-delete requires --revision and no body")) + } + case api.CLIRunDraftRecoverCommand: + if !revisionSet || bodySet { + return draftLifecycleIntent{}, draftReplyError("invalid_args", errors.New("draft-recover requires --revision and no body")) + } + } + return intent, nil +} + +func draftLifecycleReceiptOutput(receipt store.IMAPDraftReceipt) draftLifecycleReceipt { + return draftLifecycleReceipt{Mailbox: receipt.Mailbox, UIDValidity: receipt.UIDValidity, UID: receipt.UID} +} + +func draftLifecycleObservationOutput(observation imaplib.DraftObservation) *draftLifecycleObservation { + flags := make([]string, len(observation.Flags)) + for i, flag := range observation.Flags { + flags[i] = string(flag) + } + return &draftLifecycleObservation{ + State: observation.State, Code: observation.Code, + Mailbox: observation.Mailbox, UIDValidity: observation.UIDValidity, + UID: observation.UID, Flags: flags, Present: observation.Present, + Draft: observation.Draft, Deleted: observation.Deleted, + Complete: observation.Complete, UIDPlus: observation.UIDPlus, + } +} + +func draftLifecycleObservationCode(observation imaplib.DraftObservation, fallback string) string { + if observation.Code != "" { + return observation.Code + } + return fallback +} + +func draftLifecycleMetadata( + draft store.IMAPDraft, + status string, + providerObservation *draftLifecycleObservation, + observation *draftLifecycleObservation, +) draftLifecycleOutput { + lifecycle := draftLifecycleActive + if draft.DiscardedAt != nil { + lifecycle = "discarded" + } + output := draftLifecycleOutput{ + Status: status, DraftID: draft.DraftID, Revision: draft.Revision, + Lifecycle: lifecycle, MessageID: draft.CurrentMessageID, + SourceID: draft.SourceID, Receipt: draftLifecycleReceiptOutput(draft.CurrentReceipt), + ProviderObservation: providerObservation, Observation: observation, + } + if draft.Pending != nil { + output.PendingOperation = draft.Pending.Operation + output.PendingCode = draft.Pending.Code + if draft.Pending.ReplacementReceipt != nil { + receipt := draftLifecycleReceiptOutput(*draft.Pending.ReplacementReceipt) + output.PendingReceipt = &receipt + } + } + return output +} + +func (a *storeAPIAdapter) draftLifecycleOutput( + ctx context.Context, + draft store.IMAPDraft, + status string, + providerObservation *draftLifecycleObservation, + observation *draftLifecycleObservation, +) (draftLifecycleOutput, error) { + body, raw, err := a.store.GetMessageBodyAndRawContext(ctx, draft.CurrentMessageID) + if err != nil { + return draftLifecycleOutput{}, fmt.Errorf("load managed draft MIME: %w", err) + } + output := draftLifecycleMetadata(draft, status, providerObservation, observation) + output.Content = body + output.RawMIME = string(raw) + if draft.Pending != nil { + output.CandidateContent = string(draft.Pending.Raw) + } + return output, nil +} + +func (a *storeAPIAdapter) draftRecoveryOutput( + ctx context.Context, + draft store.IMAPDraft, + status string, + providerObservation *draftLifecycleObservation, + observation *draftLifecycleObservation, + grant *agentgrant.Grant, +) (draftLifecycleOutput, error) { + if grant != nil { + return draftLifecycleMetadata(draft, status, providerObservation, observation), nil + } + return a.draftLifecycleOutput(ctx, draft, status, providerObservation, observation) +} + +func emitDraftLifecycleOutput( + emit func(api.CLIRunEvent) error, + stream string, + intent draftLifecycleIntent, + output draftLifecycleOutput, +) error { + if emit == nil { + return nil + } + if intent.MetadataOnly { + output.Content = "" + output.RawMIME = "" + output.CandidateContent = "" + } + if intent.JSON { + data, err := jsonv2.Marshal(output) + if err != nil { + return err + } + return emit(api.CLIRunEvent{Type: stream, Data: string(data) + "\n"}) + } + var data strings.Builder + fmt.Fprintf(&data, "draft %s revision %d %s\n", + textutil.SanitizeTerminal(output.DraftID), output.Revision, textutil.SanitizeTerminal(output.Lifecycle)) + fmt.Fprintf(&data, "status: %s\n", textutil.SanitizeTerminal(output.Status)) + fmt.Fprintf(&data, "receipt (revision %d): %s\n", + output.Revision, textutil.SanitizeTerminal(formatDraftLifecycleReceipt(output.Receipt))) + fmt.Fprintf(&data, "content:\n%s\n", + strings.TrimRight(textutil.SanitizeTerminalMultiline(output.Content), "\n")) + if output.PendingOperation != "" { + fmt.Fprintf(&data, "pending operation: %s\n", textutil.SanitizeTerminal(output.PendingOperation)) + } + if output.CandidateContent != "" { + fmt.Fprintf(&data, "candidate content:\n%s\n", + strings.TrimRight(textutil.SanitizeTerminalMultiline(output.CandidateContent), "\n")) + } + if output.PendingReceipt != nil { + fmt.Fprintf(&data, "pending receipt (revision %d): %s\n", + output.Revision, textutil.SanitizeTerminal(formatDraftLifecycleReceipt(*output.PendingReceipt))) + } + if output.Status == "accepted_local_failed" && output.ProviderObservation != nil && + output.ProviderObservation.State == "present" && output.ProviderObservation.Present && + output.ProviderObservation.Mailbox != "" && output.ProviderObservation.UIDValidity != 0 && + output.ProviderObservation.UID != 0 { + fmt.Fprintf(&data, "acknowledged replacement receipt: %s\n", + textutil.SanitizeTerminal(formatDraftLifecycleObservationReceipt(*output.ProviderObservation))) + } + if output.Observation != nil && output.Status == laneStatePending { + fmt.Fprintf(&data, "old provider receipt: %s\n", + textutil.SanitizeTerminal(formatDraftLifecycleObservationReceipt(*output.Observation))) + } + if output.RefusalCode != "" { + fmt.Fprintf(&data, "recovery refusal: %s\n", textutil.SanitizeTerminal(output.RefusalCode)) + } + providerOutcome := output.PendingCode + if providerOutcome == "" { + observations := []*draftLifecycleObservation{output.ProviderObservation, output.Observation} + if output.Status == laneStatePending { + observations = []*draftLifecycleObservation{output.Observation, output.ProviderObservation} + } + for _, observation := range observations { + if observation == nil { + continue + } + providerOutcome = observation.Code + if providerOutcome == "" { + providerOutcome = observation.State + } + if providerOutcome != "" { + break + } + } + } + if providerOutcome != "" { + fmt.Fprintf(&data, "provider outcome: %s\n", textutil.SanitizeTerminal(providerOutcome)) + } + if output.Status == laneStatePending || output.Status == "accepted_local_failed" || output.ManualReconciliation { + fmt.Fprintf(&data, "old draft ID remains blocked at revision %d\n", output.Revision) + fmt.Fprintln(&data, "manual action: reconcile the provider receipt and local state before retrying") + } + return emit(api.CLIRunEvent{Type: stream, Data: data.String()}) +} + +func formatDraftLifecycleReceipt(receipt draftLifecycleReceipt) string { + return fmt.Sprintf("%s uidvalidity=%d uid=%d", receipt.Mailbox, receipt.UIDValidity, receipt.UID) +} + +func formatDraftLifecycleObservationReceipt(observation draftLifecycleObservation) string { + return fmt.Sprintf("%s uidvalidity=%d uid=%d (%s)", + observation.Mailbox, observation.UIDValidity, observation.UID, observation.Code) +} + +func (a *storeAPIAdapter) loadManagedDraftSource(ctx context.Context, draft store.IMAPDraft) (*store.Source, error) { + source, err := a.store.GetSourceByIDContext(ctx, draft.SourceID) + if err != nil { + return nil, draftReplyError("invalid_source", err) + } + if err := a.validateManagedDraftSource(draft, source); err != nil { + return nil, err + } + return source, nil +} + +func (a *storeAPIAdapter) validateManagedDraftSource(draft store.IMAPDraft, source *store.Source) error { + mailbox, err := authorizeIMAPDraft(a.draftPolicy, source.ID, source.SourceType) + if err != nil { + return err + } + if mailbox != draft.CurrentReceipt.Mailbox { + return draftReplyError("invalid_mailbox", errors.New("draft receipt mailbox is outside the current owner grant")) + } + if !source.SyncConfig.Valid { + return draftReplyError("invalid_source", errors.New("source has no sync config")) + } + config, err := imaplib.ConfigFromJSON(source.SyncConfig.String) + if err != nil || config.Identifier() != source.Identifier { + return draftReplyError("invalid_source", errors.New("source sync config identity does not match the source")) + } + return nil +} + +func localDraftEvidenceContext(ctx context.Context) (context.Context, context.CancelFunc) { + return context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) +} + +func (a *storeAPIAdapter) releaseDraftSourceAndRefreshCache(ctx context.Context, source *store.Source, execution *store.SyncExecution) { + if err := execution.Release(); err != nil { + if a.logger != nil { + a.logger.Error("release source after draft write", "source_id", source.ID, "error", err) + } + } + // Committed changes must reach the cache even if cleanup or output fails. + refreshCtx, cancel := localDraftEvidenceContext(ctx) + defer cancel() + a.refreshDraftCache(refreshCtx, source) +} + +func (a *storeAPIAdapter) runCLIDraftLifecycle( + ctx context.Context, + req api.CLIRunRequest, + emit func(api.CLIRunEvent) error, +) error { + if len(req.Env) != 0 || req.Cwd != "" { + return draftReplyError("invalid_args", errors.New("draft lifecycle commands accept no environment or working directory")) + } + intent, err := parseDraftLifecycleArgs(req.Args) + if err != nil { + return err + } + if intent.ConversationID != 0 || strings.HasPrefix(intent.DraftID, store.ChatDraftIDPrefix) { + return a.runCLIChatDraftLifecycle(ctx, intent, req.Grant, emit) + } + draft, err := a.store.GetIMAPDraftContext(ctx, intent.DraftID) + if err != nil { + if errors.Is(err, store.ErrIMAPDraftNotFound) { + beeperDraft, beeperErr := a.store.GetBeeperDraftContext(ctx, intent.DraftID) + if beeperErr == nil { + return a.runBeeperDraftLifecycle(ctx, intent, req.Grant, beeperDraft, emit) + } + if !errors.Is(beeperErr, store.ErrBeeperDraftNotFound) { + if req.Grant != nil { + return draftReplyNotPermitted(beeperErr) + } + return draftReplyError("draft_read_failed", beeperErr) + } + } + if req.Grant != nil { + return a.runDelegatedGmailDraftLifecycle(ctx, intent, req.Grant, err, emit) + } + if errors.Is(err, store.ErrIMAPDraftNotFound) { + gmailDraft, gmailErr := a.store.GetGmailDraftContext(ctx, intent.DraftID) + if gmailErr == nil { + if intent.Operation == api.CLIRunDraftRecoverCommand { + return draftReplyError("not_supported", errors.New("draft-recover supports IMAP drafts only")) + } + return a.runCLIGmailDraftLifecycle(ctx, intent, gmailDraft, nil, emit) + } + if !errors.Is(gmailErr, store.ErrGmailDraftNotFound) { + return draftReplyError("draft_read_failed", gmailErr) + } + } + return draftReplyError("draft_not_found", err) + } + if req.Grant != nil && intent.Operation != api.CLIRunDraftRecoverCommand { + canReadContent, err := a.authorizeDelegatedDraftLifecycle(ctx, intent, req.Grant, draft.SourceID, draft.CurrentMessageID) + if err != nil { + return err + } + intent.MetadataOnly = !canReadContent + } + if intent.Operation == api.CLIRunDraftGetCommand { + provider := &draftLifecycleObservation{State: "not_checked", Code: "not_checked"} + output, err := a.draftLifecycleOutput(ctx, draft, "ok", provider, nil) + if err != nil { + return draftReplyError("draft_read_failed", err) + } + if req.Grant != nil { + if err := a.authorizeDelegatedDraftOutput(ctx, intent, req.Grant, draft.SourceID, output.RawMIME, output.CandidateContent); err != nil { + return err + } + } + return emitDraftLifecycleOutput(emit, cliStreamStdout, intent, output) + } + if intent.Operation == api.CLIRunDraftRecoverCommand { + return a.runDraftRecover(ctx, intent, draft, req.Grant, emit) + } + if draft.Revision != intent.Revision { + return draftReplyError("revision_mismatch", fmt.Errorf("expected revision %d, found %d", intent.Revision, draft.Revision)) + } + if draft.DiscardedAt != nil { + if intent.Operation == api.CLIRunDraftDeleteCommand { + output, err := a.draftLifecycleOutput(ctx, draft, "already_discarded", nil, nil) + if err != nil { + return draftReplyError("draft_read_failed", err) + } + if req.Grant != nil { + if err := a.authorizeDelegatedDraftOutput(ctx, intent, req.Grant, draft.SourceID, output.RawMIME, output.CandidateContent); err != nil { + return err + } + } + return emitDraftLifecycleOutput(emit, cliStreamStdout, intent, output) + } + return draftReplyError("draft_discarded", errors.New("discarded drafts cannot be edited")) + } + if draft.Pending != nil && draft.Pending.Code != store.IMAPDraftCodeRemoved { + return draftReplyError("pending_operation", store.ErrIMAPDraftPending) + } + source, err := a.loadManagedDraftSource(ctx, draft) + if err != nil { + return err + } + execution, err := a.store.AcquireSyncExecutionContext(ctx, source.ID) + if err != nil { + if errors.Is(err, store.ErrSyncAlreadyActive) { + return draftReplyError("sync_active", err) + } + return draftReplyError("sync_lock_failed", err) + } + defer func() { _ = execution.Release() }() + draft, err = a.store.GetIMAPDraftContext(ctx, intent.DraftID) + if err != nil { + return draftReplyError("draft_not_found", err) + } + // Sync can replace the archived sender without advancing the draft revision. + if req.Grant != nil { + if _, err := a.authorizeDelegatedDraftLifecycle(ctx, intent, req.Grant, draft.SourceID, draft.CurrentMessageID); err != nil { + return err + } + } + if draft.Revision != intent.Revision { + return draftReplyError("revision_mismatch", errors.New("draft changed while acquiring source ownership")) + } + if draft.Pending != nil && (draft.Pending.Code != store.IMAPDraftCodeRemoved || "draft-"+draft.Pending.Operation != intent.Operation) { + return draftReplyError("pending_operation", store.ErrIMAPDraftPending) + } + source, err = a.loadManagedDraftSource(ctx, draft) + if err != nil { + return err + } + currentRaw, err := a.store.GetMessageRawContext(ctx, draft.CurrentMessageID) + if err != nil { + return draftReplyError("draft_read_failed", err) + } + var replacement imaplib.ReplyDraft + if intent.Operation == api.CLIRunDraftEditCommand { + replacement, err = imaplib.BuildIMAPDraftReplacement(currentRaw, intent.Body, time.Now(), "") + if err != nil { + return draftReplyError("invalid_draft", err) + } + } + if draft.Pending != nil { + if intent.Operation == api.CLIRunDraftEditCommand { + current, err := msgmime.Parse(currentRaw) + if err != nil { + return draftReplyError("draft_read_failed", err) + } + if current.BodyText != replacement.Parsed.BodyText { + return draftReplyError("pending_operation", errors.New("retry must use the already published edit body")) + } + } + finished, err := a.store.FinishIMAPDraftRemovalContext(ctx, intent.DraftID, intent.Revision) + if err != nil { + return draftReplyError("cleanup_local_failed", err) + } + defer a.releaseDraftSourceAndRefreshCache(ctx, source, execution) + status := "edited" + if intent.Operation == api.CLIRunDraftDeleteCommand { + status = "deleted" + } + output, err := a.draftLifecycleOutput(ctx, finished, status, nil, nil) + if err != nil { + return draftReplyError("draft_read_failed", err) + } + if err := emitDraftLifecycleOutput(emit, cliStreamStdout, intent, output); err != nil { + return draftReplyError("output_failed", err) + } + return nil + } + var attachmentWrites []store.AttachmentWrite + if intent.Operation == api.CLIRunDraftEditCommand { + attachmentWrites, err = a.forwardDraftAttachmentWrites(ctx, draft.CurrentMessageID, replacement) + if err != nil { + return draftReplyError("invalid_draft", err) + } + } + clientFactory := a.draftClientFactory + if clientFactory == nil { + clientFactory = defaultDraftClientFactory + } + client, err := clientFactory(ctx, source) + if err != nil { + return draftReplyError("invalid_source", err) + } + defer func() { _ = client.Close() }() + + providerReceipt := imaplib.DraftReceipt{ + Mailbox: draft.CurrentReceipt.Mailbox, + UIDValidity: draft.CurrentReceipt.UIDValidity, + UID: draft.CurrentReceipt.UID, + } + inspection, err := client.InspectDraft(ctx, providerReceipt) + if err != nil || !inspection.Present || inspection.Deleted || !inspection.Draft || !inspection.UIDPlus { + provider := draftLifecycleObservationOutput(inspection) + output, outputErr := a.draftLifecycleOutput(ctx, draft, "refused", provider, nil) + if outputErr == nil { + _ = emitDraftLifecycleOutput(emit, cliStreamStderr, intent, output) + } + if err != nil { + code := inspection.Code + if code == "" { + code = "provider_refused" + } + return draftReplyError(code, err) + } + code := inspection.Code + if code == "" { + code = "provider_refused" + } + return draftReplyError(code, errors.New("provider draft inspection refused mutation")) + } + if intent.Operation == api.CLIRunDraftEditCommand { + return a.runDraftEdit(ctx, intent, draft, source, client, currentRaw, replacement, attachmentWrites, inspection, execution, emit) + } + return a.runDraftDelete(ctx, intent, draft, source, client, inspection, execution, emit) +} + +func (a *storeAPIAdapter) runDraftEdit( + ctx context.Context, + intent draftLifecycleIntent, + draft store.IMAPDraft, + source *store.Source, + client *imaplib.Client, + currentRaw []byte, + replacement imaplib.ReplyDraft, + attachmentWrites []store.AttachmentWrite, + inspection imaplib.DraftObservation, + execution *store.SyncExecution, + emit func(api.CLIRunEvent) error, +) error { + claimed, err := a.store.ClaimIMAPDraftContext(ctx, intent.DraftID, intent.Revision, store.IMAPDraftOperationEdit, replacement.Raw) + if err != nil { + return draftReplyError("claim_failed", err) + } + appendResult, err := client.AppendDraft(ctx, draft.CurrentReceipt.Mailbox, replacement.Raw) + if err != nil { + code := appendResult.Code + if code == "" { + code = "append_failed" + } + evidenceCtx, cancel := localDraftEvidenceContext(ctx) + defer cancel() + latest := claimed + var persistenceErr error + if appendResult.State == imaplib.DraftStateRejected || appendResult.State == imaplib.DraftStateCancelled { + latest, persistenceErr = a.store.AbortIMAPDraftContext(evidenceCtx, intent.DraftID, intent.Revision, appendResult.State) + if persistenceErr != nil { + latest = claimed + } + } else { + persistenceErr = a.store.RecordIMAPDraftOutcomeContext(evidenceCtx, intent.DraftID, intent.Revision, code, nil) + if loaded, loadErr := a.store.GetIMAPDraftContext(evidenceCtx, intent.DraftID); loadErr == nil { + latest = loaded + } + } + status := laneStatePending + if latest.Pending == nil && persistenceErr == nil { + status = draftLifecycleActive + } + output, outputErr := a.draftLifecycleOutput(evidenceCtx, latest, status, draftLifecycleObservationOutput(inspection), nil) + if outputErr != nil { + output = draftLifecycleOutput{ + Status: status, DraftID: latest.DraftID, Revision: latest.Revision, Lifecycle: draftLifecycleActive, + MessageID: latest.CurrentMessageID, SourceID: latest.SourceID, + Receipt: draftLifecycleReceiptOutput(latest.CurrentReceipt), RawMIME: string(currentRaw), + } + if latest.Pending != nil { + output.PendingOperation = latest.Pending.Operation + output.CandidateContent = string(latest.Pending.Raw) + } + } + output.PendingCode = code + output.ManualReconciliation = latest.Pending != nil || persistenceErr != nil + _ = emitDraftLifecycleOutput(emit, cliStreamStderr, intent, output) + if persistenceErr != nil { + return draftReplyError("local_persistence_failed", errors.Join(err, persistenceErr)) + } + if coded, ok := errors.AsType[*imaplib.DraftAppendError](err); ok { + err = coded.Err + } + return draftReplyError(code, err) + } + receipt := store.IMAPDraftReceipt{SourceID: source.ID, Mailbox: draft.CurrentReceipt.Mailbox, UIDValidity: appendResult.UIDValidity, UID: appendResult.UID} + evidenceCtx, cancel := localDraftEvidenceContext(ctx) + defer cancel() + if err := a.store.RecordIMAPDraftOutcomeContext(evidenceCtx, intent.DraftID, intent.Revision, appendResult.Code, &receipt); err != nil { + appendObservation := &draftLifecycleObservation{ + State: "present", Code: appendResult.Code, Mailbox: receipt.Mailbox, + UIDValidity: receipt.UIDValidity, UID: receipt.UID, Present: true, + } + output, outputErr := a.draftLifecycleOutput(evidenceCtx, claimed, "accepted_local_failed", appendObservation, nil) + if outputErr == nil { + output.ManualReconciliation = true + _ = emitDraftLifecycleOutput(emit, cliStreamStderr, intent, output) + } + return draftReplyError("accepted_local_failed", err) + } + reportAcceptedLocalFailure := func(cause error) error { + output := draftLifecycleOutput{ + Status: "accepted_local_failed", DraftID: claimed.DraftID, Revision: claimed.Revision, + Lifecycle: draftLifecycleActive, MessageID: claimed.CurrentMessageID, SourceID: claimed.SourceID, + Receipt: draftLifecycleReceiptOutput(claimed.CurrentReceipt), RawMIME: string(currentRaw), + PendingOperation: store.IMAPDraftOperationEdit, PendingCode: appendResult.Code, + CandidateContent: string(replacement.Raw), ManualReconciliation: true, + } + if loaded, loadErr := a.draftLifecycleOutput(evidenceCtx, claimed, "accepted_local_failed", nil, nil); loadErr == nil { + output = loaded + } + output.PendingCode = appendResult.Code + output.ManualReconciliation = true + pendingReceipt := draftLifecycleReceiptOutput(receipt) + output.PendingReceipt = &pendingReceipt + _ = emitDraftLifecycleOutput(emit, cliStreamStderr, intent, output) + return draftReplyError("accepted_local_failed", cause) + } + currentMessage, currentMessageErr := a.store.GetMessageContext(evidenceCtx, draft.CurrentMessageID) + if currentMessageErr != nil { + return reportAcceptedLocalFailure(currentMessageErr) + } + replyTo, replyToErr := a.store.GetMessageReplyToMessageIDContext(evidenceCtx, draft.CurrentMessageID) + if replyToErr != nil { + return reportAcceptedLocalFailure(replyToErr) + } + participants, build := draftLifecyclePersistData(currentMessage.ConversationID, replyTo, replacement, receipt, attachmentWrites) + published, err := a.store.PublishIMAPDraftReplacementContext(evidenceCtx, intent.DraftID, intent.Revision, participants, build) + if err != nil { + return reportAcceptedLocalFailure(err) + } + defer a.releaseDraftSourceAndRefreshCache(ctx, source, execution) + if ctx.Err() != nil { + output, outputErr := a.draftLifecycleOutput(evidenceCtx, published, laneStatePending, nil, nil) + if outputErr == nil { + output.ManualReconciliation = true + _ = emitDraftLifecycleOutput(emit, cliStreamStderr, intent, output) + } + return draftReplyError("cancelled", ctx.Err()) + } + removed, err := client.RemoveDraft(ctx, imaplib.DraftReceipt{Mailbox: draft.CurrentReceipt.Mailbox, UIDValidity: draft.CurrentReceipt.UIDValidity, UID: draft.CurrentReceipt.UID}) + evidenceCtx, cancelCleanupEvidence := localDraftEvidenceContext(ctx) + defer cancelCleanupEvidence() + if err != nil || !removed.Complete { + code := draftLifecycleObservationCode(removed, "cleanup_incomplete") + recordErr := a.store.RecordIMAPDraftOutcomeContext(evidenceCtx, intent.DraftID, published.Revision, code, nil) + a.emitDraftLifecyclePending(evidenceCtx, intent, published, nil, nil, removed, emit) + if err == nil { + err = errors.New("provider cleanup is incomplete") + } + if recordErr != nil { + return draftReplyError("local_persistence_failed", errors.Join(err, recordErr)) + } + return draftReplyError(code, err) + } + if err := a.store.RecordIMAPDraftOutcomeContext( + evidenceCtx, intent.DraftID, published.Revision, store.IMAPDraftCodeRemoved, nil, + ); err != nil { + a.emitDraftLifecyclePending(evidenceCtx, intent, published, nil, nil, removed, emit) + return draftReplyError("local_persistence_failed", err) + } + finished, err := a.store.FinishIMAPDraftRemovalContext(evidenceCtx, intent.DraftID, published.Revision) + if err != nil { + a.emitDraftLifecyclePending(evidenceCtx, intent, published, nil, nil, removed, emit) + return draftReplyError("cleanup_local_failed", err) + } + output, err := a.draftLifecycleOutput(evidenceCtx, finished, "edited", nil, draftLifecycleObservationOutput(removed)) + if err != nil { + return draftReplyError("draft_read_failed", err) + } + if err := emitDraftLifecycleOutput(emit, cliStreamStdout, intent, output); err != nil { + return draftReplyError("output_failed", err) + } + return nil +} + +func (a *storeAPIAdapter) emitDraftLifecyclePending( + ctx context.Context, + intent draftLifecycleIntent, + draft store.IMAPDraft, + grant *agentgrant.Grant, + providerObservation *draftLifecycleObservation, + observation imaplib.DraftObservation, + emit func(api.CLIRunEvent) error, +) { + if latest, err := a.store.GetIMAPDraftContext(ctx, draft.DraftID); err == nil { + draft = latest + } + output, err := a.draftRecoveryOutput(ctx, draft, laneStatePending, providerObservation, draftLifecycleObservationOutput(observation), grant) + if err != nil { + output = draftLifecycleMetadata(draft, laneStatePending, providerObservation, draftLifecycleObservationOutput(observation)) + if grant == nil && draft.Pending != nil { + output.CandidateContent = string(draft.Pending.Raw) + } + } + output.ManualReconciliation = true + _ = emitDraftLifecycleOutput(emit, cliStreamStderr, intent, output) +} + +func (a *storeAPIAdapter) runDraftDelete( + ctx context.Context, + intent draftLifecycleIntent, + draft store.IMAPDraft, + source *store.Source, + client *imaplib.Client, + inspection imaplib.DraftObservation, + execution *store.SyncExecution, + emit func(api.CLIRunEvent) error, +) error { + claimed, err := a.store.ClaimIMAPDraftContext(ctx, intent.DraftID, intent.Revision, store.IMAPDraftOperationDelete, nil) + if err != nil { + return draftReplyError("claim_failed", err) + } + removed, err := client.RemoveDraft(ctx, imaplib.DraftReceipt{Mailbox: draft.CurrentReceipt.Mailbox, UIDValidity: draft.CurrentReceipt.UIDValidity, UID: draft.CurrentReceipt.UID}) + if err != nil || !removed.Complete { + evidenceCtx, cancel := localDraftEvidenceContext(ctx) + defer cancel() + code := draftLifecycleObservationCode(removed, "cleanup_incomplete") + if !removed.WriteAttempted { + active, abortErr := a.store.AbortIMAPDraftContext(evidenceCtx, intent.DraftID, intent.Revision, "not_attempted") + if abortErr != nil { + a.emitDraftLifecyclePending(evidenceCtx, intent, claimed, nil, draftLifecycleObservationOutput(inspection), removed, emit) + return draftReplyError("local_persistence_failed", errors.Join(err, abortErr)) + } + output, outputErr := a.draftLifecycleOutput(evidenceCtx, active, draftLifecycleActive, draftLifecycleObservationOutput(inspection), draftLifecycleObservationOutput(removed)) + if outputErr != nil { + return draftReplyError("draft_read_failed", outputErr) + } + _ = emitDraftLifecycleOutput(emit, cliStreamStderr, intent, output) + return draftReplyError(code, err) + } + recordErr := a.store.RecordIMAPDraftOutcomeContext(evidenceCtx, intent.DraftID, intent.Revision, code, nil) + a.emitDraftLifecyclePending(evidenceCtx, intent, claimed, nil, draftLifecycleObservationOutput(inspection), removed, emit) + if err == nil { + err = errors.New("provider cleanup is incomplete") + } + if recordErr != nil { + return draftReplyError("local_persistence_failed", errors.Join(err, recordErr)) + } + return draftReplyError(code, err) + } + evidenceCtx, cancel := localDraftEvidenceContext(ctx) + defer cancel() + if err := a.store.RecordIMAPDraftOutcomeContext( + evidenceCtx, intent.DraftID, intent.Revision, store.IMAPDraftCodeRemoved, nil, + ); err != nil { + a.emitDraftLifecyclePending(evidenceCtx, intent, claimed, nil, draftLifecycleObservationOutput(inspection), removed, emit) + return draftReplyError("local_persistence_failed", err) + } + finished, err := a.store.FinishIMAPDraftRemovalContext(evidenceCtx, intent.DraftID, intent.Revision) + if err != nil { + a.emitDraftLifecyclePending(evidenceCtx, intent, claimed, nil, draftLifecycleObservationOutput(inspection), removed, emit) + return draftReplyError("cleanup_local_failed", err) + } + defer a.releaseDraftSourceAndRefreshCache(ctx, source, execution) + output, err := a.draftLifecycleOutput(evidenceCtx, finished, "deleted", nil, draftLifecycleObservationOutput(removed)) + if err != nil { + return draftReplyError("draft_read_failed", err) + } + if err := emitDraftLifecycleOutput(emit, cliStreamStdout, intent, output); err != nil { + return draftReplyError("output_failed", err) + } + return nil +} + +func draftRecoveryPermission(draft store.IMAPDraft) (agentgrant.Permission, error) { + if draft.DiscardedAt != nil { + return agentgrant.PermissionDraftDelete, nil + } + if draft.Pending == nil { + return agentgrant.PermissionDraftEdit, nil + } + switch draft.Pending.Operation { + case store.IMAPDraftOperationEdit: + return agentgrant.PermissionDraftEdit, nil + case store.IMAPDraftOperationDelete: + return agentgrant.PermissionDraftDelete, nil + default: + return "", draftReplyError("invalid_state", fmt.Errorf("unknown pending draft operation %q", draft.Pending.Operation)) + } +} + +func draftProviderReceipt(receipt store.IMAPDraftReceipt) imaplib.DraftReceipt { + return imaplib.DraftReceipt{ + Mailbox: receipt.Mailbox, UIDValidity: receipt.UIDValidity, UID: receipt.UID, + } +} + +// authorizeDelegatedDraftLifecycle checks a delegated Gmail or IMAP get, edit, +// or delete before any draft work and reports whether content may be returned. +func (a *storeAPIAdapter) authorizeDelegatedDraftLifecycle( + ctx context.Context, + intent draftLifecycleIntent, + grant *agentgrant.Grant, + sourceID, currentMessageID int64, +) (bool, error) { + raw, err := a.store.GetMessageRawContext(ctx, currentMessageID) + if err != nil { + return false, draftReplyNotPermitted(fmt.Errorf("load managed draft sender: %w", err)) + } + return a.authorizeDelegatedDraftContent(ctx, intent, grant, sourceID, raw) +} + +// authorizeDelegatedDraftOutput checks the snapshots actually returned. Sync +// may replace current MIME while a pending edit still holds a different sender. +func (a *storeAPIAdapter) authorizeDelegatedDraftOutput( + ctx context.Context, + intent draftLifecycleIntent, + grant *agentgrant.Grant, + sourceID int64, + raw, candidate string, +) error { + canReadContent, err := a.authorizeDelegatedDraftContent(ctx, intent, grant, sourceID, []byte(raw)) + if err != nil { + return err + } + if canReadContent && candidate != "" { + _, err = a.authorizeDelegatedDraftContent(ctx, intent, grant, sourceID, []byte(candidate)) + } + return err +} + +func (a *storeAPIAdapter) authorizeDelegatedDraftContent( + ctx context.Context, + intent draftLifecycleIntent, + grant *agentgrant.Grant, + sourceID int64, + raw []byte, +) (bool, error) { + source, err := a.store.GetSourceByIDContext(ctx, sourceID) + if err != nil { + return false, draftReplyNotPermitted(fmt.Errorf("load source %d: %w", sourceID, err)) + } + ref := draftSourceRef(source) + for _, permission := range api.CLIRunDraftLifecyclePermissions(intent.Operation) { + if !grant.Allows(permission, ref) { + continue + } + senderKey, err := managedDraftSenderKey(raw) + if err != nil { + return false, draftReplyNotPermitted(fmt.Errorf("load managed draft sender: %w", err)) + } + if grant.AllowsSender(permission, ref, senderKey) { + return grant.AllowsSender(agentgrant.PermissionDraftCreate, ref, senderKey) || + grant.AllowsSender(agentgrant.PermissionDraftEdit, ref, senderKey), nil + } + } + return false, draftReplyNotPermitted(fmt.Errorf("source %d or sender is not in grant %s for %s", source.ID, grant.ID, intent.Operation)) +} + +func managedDraftSenderKey(raw []byte) (string, error) { + parsed, err := msgmime.Parse(raw) + if err != nil { + return "", err + } + if len(parsed.From) != 1 { + return "", errors.New("managed draft must have exactly one From address") + } + _, senderKey, err := parseDraftSender(parsed.From[0].Email) + return senderKey, err +} + +func (a *storeAPIAdapter) authorizeDraftRecovery( + ctx context.Context, + intent draftLifecycleIntent, + draft store.IMAPDraft, + grant *agentgrant.Grant, +) (*store.Source, error) { + permission, err := draftRecoveryPermission(draft) + if err != nil { + return nil, err + } + source, err := a.store.GetSourceByIDContext(ctx, draft.SourceID) + if err != nil { + if grant != nil { + return nil, draftReplyNotPermitted(fmt.Errorf("load source %d: %w", draft.SourceID, err)) + } + return nil, draftReplyError("invalid_source", err) + } + if grant != nil { + ref := agentgrant.SourceRef{ID: source.ID, Type: source.SourceType, Identifier: source.Identifier} + if !grant.Allows(permission, ref) { + return nil, draftReplyNotPermitted(fmt.Errorf("source %d is not in grant %s", source.ID, grant.ID)) + } + } + if draft.Revision != intent.Revision { + return nil, draftReplyError("revision_mismatch", fmt.Errorf("expected revision %d, found %d", intent.Revision, draft.Revision)) + } + if err := a.validateManagedDraftSource(draft, source); err != nil { + return nil, err + } + return source, nil +} + +func (a *storeAPIAdapter) settleDraftRecovery( + ctx context.Context, + intent draftLifecycleIntent, + draft store.IMAPDraft, + grant *agentgrant.Grant, + emit func(api.CLIRunEvent) error, +) (bool, error) { + var status string + switch { + case draft.DiscardedAt != nil: + status = "already_discarded" + case draft.Pending == nil: + status = draftLifecycleActive + case draft.Pending.Operation == store.IMAPDraftOperationEdit && draft.Pending.ReplacementReceipt == nil: + output, err := a.draftRecoveryOutput(ctx, draft, "unknown_replacement", nil, nil, grant) + if err != nil { + return true, draftReplyError("draft_read_failed", err) + } + output.RefusalCode = "unknown_replacement" + output.ManualReconciliation = true + if err := emitDraftLifecycleOutput(emit, cliStreamStderr, intent, output); err != nil { + return true, draftReplyError("output_failed", err) + } + return true, draftReplyError("unknown_replacement", errors.New("pending edit has no recorded replacement receipt")) + default: + return false, nil + } + output, err := a.draftRecoveryOutput(ctx, draft, status, nil, nil, grant) + if err != nil { + return true, draftReplyError("draft_read_failed", err) + } + if err := emitDraftLifecycleOutput(emit, cliStreamStdout, intent, output); err != nil { + return true, draftReplyError("output_failed", err) + } + return true, nil +} + +func (a *storeAPIAdapter) refuseDraftRecovery( + ctx context.Context, + intent draftLifecycleIntent, + draft store.IMAPDraft, + grant *agentgrant.Grant, + code string, + cause error, + observation *imaplib.DraftObservation, + emit func(api.CLIRunEvent) error, +) error { + var providerObservation *draftLifecycleObservation + if observation != nil { + providerObservation = draftLifecycleObservationOutput(*observation) + } + evidenceCtx, cancel := localDraftEvidenceContext(ctx) + defer cancel() + output, outputErr := a.draftRecoveryOutput(evidenceCtx, draft, "refused", providerObservation, nil, grant) + if outputErr != nil { + return draftReplyError("draft_read_failed", errors.Join(cause, outputErr)) + } + output.RefusalCode = code + output.ManualReconciliation = true + if emitErr := emitDraftLifecycleOutput(emit, cliStreamStderr, intent, output); emitErr != nil { + return draftReplyError("output_failed", errors.Join(cause, emitErr)) + } + if cause == nil { + cause = errors.New("provider refused draft recovery") + } + return draftReplyError(code, cause) +} + +func (a *storeAPIAdapter) publishRecoveredDraftReplacement( + ctx context.Context, + draft store.IMAPDraft, +) (store.IMAPDraft, error) { + if draft.Pending == nil || draft.Pending.Operation != store.IMAPDraftOperationEdit || draft.Pending.ReplacementReceipt == nil { + return store.IMAPDraft{}, errors.New("known replacement is required for draft publication") + } + parsed, err := msgmime.Parse(draft.Pending.Raw) + if err != nil { + return store.IMAPDraft{}, fmt.Errorf("parse recorded draft replacement: %w", err) + } + replacement := imaplib.ReplyDraft{ + Raw: append([]byte(nil), draft.Pending.Raw...), + Parsed: parsed, + } + message, err := a.store.GetMessageContext(ctx, draft.CurrentMessageID) + if err != nil { + return store.IMAPDraft{}, fmt.Errorf("load draft message for replacement: %w", err) + } + replyTo, err := a.store.GetMessageReplyToMessageIDContext(ctx, draft.CurrentMessageID) + if err != nil { + return store.IMAPDraft{}, fmt.Errorf("load draft reply link for replacement: %w", err) + } + receipt := *draft.Pending.ReplacementReceipt + attachmentWrites, err := a.forwardDraftAttachmentWrites(ctx, draft.CurrentMessageID, replacement) + if err != nil { + return store.IMAPDraft{}, err + } + participants, build := draftLifecyclePersistData(message.ConversationID, replyTo, replacement, receipt, attachmentWrites) + return a.store.PublishIMAPDraftReplacementContext(ctx, draft.DraftID, draft.Revision, participants, build) +} + +func (a *storeAPIAdapter) runDraftRecover( + ctx context.Context, + intent draftLifecycleIntent, + draft store.IMAPDraft, + grant *agentgrant.Grant, + emit func(api.CLIRunEvent) error, +) error { + source, err := a.authorizeDraftRecovery(ctx, intent, draft, grant) + if err != nil { + return err + } + settled, err := a.settleDraftRecovery(ctx, intent, draft, grant, emit) + if settled || err != nil { + return err + } + execution, err := a.store.AcquireSyncExecutionContext(ctx, source.ID) + if err != nil { + if errors.Is(err, store.ErrSyncAlreadyActive) { + return draftReplyError("sync_active", err) + } + return draftReplyError("sync_lock_failed", err) + } + refreshScheduled := false + defer func() { + if refreshScheduled { + a.releaseDraftSourceAndRefreshCache(ctx, source, execution) + return + } + _ = execution.Release() + }() + + draft, err = a.store.GetIMAPDraftContext(ctx, intent.DraftID) + if err != nil { + if grant != nil { + return draftReplyNotPermitted(err) + } + return draftReplyError("draft_not_found", err) + } + source, err = a.authorizeDraftRecovery(ctx, intent, draft, grant) + if err != nil { + return err + } + settled, err = a.settleDraftRecovery(ctx, intent, draft, grant, emit) + if settled || err != nil { + return err + } + + if draft.Pending.Code == store.IMAPDraftCodeRemoved { + evidenceCtx, cancel := localDraftEvidenceContext(ctx) + defer cancel() + finished, finishErr := a.store.FinishIMAPDraftRemovalContext(evidenceCtx, draft.DraftID, draft.Revision) + if finishErr != nil { + return draftReplyError("cleanup_local_failed", finishErr) + } + refreshScheduled = true + status := "edited" + if finished.DiscardedAt != nil { + status = "deleted" + } + output, outputErr := a.draftRecoveryOutput(evidenceCtx, finished, status, nil, nil, grant) + if outputErr != nil { + return draftReplyError("draft_read_failed", outputErr) + } + if outputErr := emitDraftLifecycleOutput(emit, cliStreamStdout, intent, output); outputErr != nil { + return draftReplyError("output_failed", outputErr) + } + return nil + } + clientFactory := a.draftClientFactory + if clientFactory == nil { + clientFactory = defaultDraftClientFactory + } + client, err := clientFactory(ctx, source) + if err != nil { + return draftReplyError("invalid_source", err) + } + defer func() { _ = client.Close() }() + + originalObservation, err := client.InspectDraft(ctx, draftProviderReceipt(draft.Pending.OriginalReceipt)) + if err != nil { + code := draftLifecycleObservationCode(originalObservation, "provider_refused") + return a.refuseDraftRecovery(ctx, intent, draft, grant, code, err, &originalObservation, emit) + } + published := false + if draft.Pending.Operation == store.IMAPDraftOperationEdit { + replacementReceipt := *draft.Pending.ReplacementReceipt + published = draft.CurrentReceipt == replacementReceipt + if !published { + if draft.Pending.OriginalReceipt.UIDValidity != replacementReceipt.UIDValidity { + return a.refuseDraftRecovery( + ctx, intent, draft, grant, "uidvalidity_mismatch", + errors.New("recorded original and replacement generations differ"), + &originalObservation, emit, + ) + } + replacementObservation, replacementErr := client.InspectDraft(ctx, draftProviderReceipt(replacementReceipt)) + if replacementErr != nil || !replacementObservation.Present || !replacementObservation.Draft || replacementObservation.Deleted { + code := draftLifecycleObservationCode(replacementObservation, "provider_refused") + if replacementErr == nil { + switch { + case !replacementObservation.Present: + code = "absent" + case replacementObservation.Deleted: + code = "already_deleted" + case !replacementObservation.Draft: + code = "not_draft" + } + replacementErr = errors.New("recorded replacement is unavailable for publication") + } + return a.refuseDraftRecovery(ctx, intent, draft, grant, code, replacementErr, &replacementObservation, emit) + } + publicationCtx, cancel := localDraftEvidenceContext(ctx) + publishedDraft, publishErr := a.publishRecoveredDraftReplacement(publicationCtx, draft) + if publishErr != nil { + output, outputErr := a.draftRecoveryOutput(publicationCtx, draft, "accepted_local_failed", draftLifecycleObservationOutput(replacementObservation), nil, grant) + if outputErr == nil { + output.ManualReconciliation = true + _ = emitDraftLifecycleOutput(emit, cliStreamStderr, intent, output) + } + cancel() + return draftReplyError("accepted_local_failed", publishErr) + } + cancel() + draft = publishedDraft + published = true + refreshScheduled = true + } + } + + cleanupObservation := originalObservation + if originalObservation.Present { + cleanupObservation, err = client.RemoveDraft(ctx, draftProviderReceipt(draft.Pending.OriginalReceipt)) + if err != nil || !cleanupObservation.Complete { + code := draftLifecycleObservationCode(cleanupObservation, "cleanup_incomplete") + if !cleanupObservation.WriteAttempted { + if published { + evidenceCtx, cancel := localDraftEvidenceContext(ctx) + defer cancel() + a.emitDraftLifecyclePending(evidenceCtx, intent, draft, grant, nil, cleanupObservation, emit) + if err == nil { + err = errors.New("provider cleanup was not attempted") + } + return draftReplyError(code, err) + } + return a.refuseDraftRecovery(ctx, intent, draft, grant, code, err, &cleanupObservation, emit) + } + evidenceCtx, cancel := localDraftEvidenceContext(ctx) + defer cancel() + recordErr := a.store.RecordIMAPDraftOutcomeContext(evidenceCtx, draft.DraftID, draft.Revision, code, nil) + a.emitDraftLifecyclePending(evidenceCtx, intent, draft, grant, nil, cleanupObservation, emit) + if recordErr != nil { + if err == nil { + err = errors.New("provider cleanup is incomplete") + } + return draftReplyError("local_persistence_failed", errors.Join(err, recordErr)) + } + if err == nil { + err = errors.New("provider cleanup is incomplete") + } + return draftReplyError(code, err) + } + } + + evidenceCtx, cancel := localDraftEvidenceContext(ctx) + defer cancel() + if err := a.store.RecordIMAPDraftOutcomeContext(evidenceCtx, draft.DraftID, draft.Revision, store.IMAPDraftCodeRemoved, nil); err != nil { + a.emitDraftLifecyclePending(evidenceCtx, intent, draft, grant, nil, cleanupObservation, emit) + return draftReplyError("local_persistence_failed", err) + } + finished, err := a.store.FinishIMAPDraftRemovalContext(evidenceCtx, draft.DraftID, draft.Revision) + if err != nil { + a.emitDraftLifecyclePending(evidenceCtx, intent, draft, grant, nil, cleanupObservation, emit) + return draftReplyError("cleanup_local_failed", err) + } + refreshScheduled = true + status := "edited" + if finished.DiscardedAt != nil { + status = "deleted" + } + output, err := a.draftRecoveryOutput(evidenceCtx, finished, status, nil, draftLifecycleObservationOutput(cleanupObservation), grant) + if err != nil { + return draftReplyError("draft_read_failed", err) + } + if err := emitDraftLifecycleOutput(emit, cliStreamStdout, intent, output); err != nil { + return draftReplyError("output_failed", err) + } + return nil +} + +func draftLifecyclePersistData( + conversationID int64, + replyTo sql.NullInt64, + replacement imaplib.ReplyDraft, + receipt store.IMAPDraftReceipt, + attachmentWrites []store.AttachmentWrite, +) ([]store.ParticipantPersistData, func([]int64) *store.MessagePersistData) { + parsed := replacement.Parsed + addresses := append([]msgmime.Address(nil), parsed.From...) + addresses = append(addresses, parsed.To...) + addresses = append(addresses, parsed.Cc...) + addresses = append(addresses, parsed.Bcc...) + participants := make([]store.ParticipantPersistData, len(addresses)) + for i, address := range addresses { + participants[i] = store.ParticipantPersistData{EmailAddress: address.Email, DisplayName: address.Name, Domain: address.Domain} + } + fromCount, toCount := len(parsed.From), len(parsed.To) + ccCount, bccCount := len(parsed.Cc), len(parsed.Bcc) + build := func(ids []int64) *store.MessagePersistData { + at := 0 + fromIDs := ids[at : at+fromCount] + at += fromCount + toIDs := ids[at : at+toCount] + at += toCount + ccIDs := ids[at : at+ccCount] + at += ccCount + bccIDs := ids[at : at+bccCount] + toAddresses := addressStrings(parsed.To) + ccAddresses := addressStrings(parsed.Cc) + bccAddresses := addressStrings(parsed.Bcc) + fromAddresses := addressStrings(parsed.From) + rfc822 := msgmime.NormalizeMessageID(parsed.MessageID) + if rfc822 != "" { + rfc822 = "<" + rfc822 + ">" + } + message := &store.Message{ + SourceID: receipt.SourceID, SourceMessageID: store.IMAPDraftSourceMessageID(receipt), + RFC822MessageID: sql.NullString{String: rfc822, Valid: rfc822 != ""}, + ConversationID: conversationID, + MessageType: store.MessageTypeEmail, IsFromMe: true, IdentityDerivedIsFromMe: true, + SenderID: sql.NullInt64{Int64: fromIDs[0], Valid: len(fromIDs) > 0}, + ReplyToMessageID: replyTo, + ListID: sql.NullString{String: parsed.ListID, Valid: parsed.ListID != ""}, + Subject: sql.NullString{String: parsed.Subject, Valid: parsed.Subject != ""}, + Snippet: sql.NullString{String: strings.TrimSpace(parsed.BodyText), Valid: parsed.BodyText != ""}, + SentAt: sql.NullTime{Time: parsed.Date, Valid: !parsed.Date.IsZero()}, + InternalDate: sql.NullTime{Time: parsed.Date, Valid: !parsed.Date.IsZero()}, + SizeEstimate: int64(len(replacement.Raw)), ArchivedAt: time.Now(), + } + data := &store.MessagePersistData{ + Message: message, BodyText: sql.NullString{String: parsed.BodyText, Valid: true}, + RawMIME: replacement.Raw, RawFormat: "mime", + Recipients: []store.RecipientSet{ + {Type: "from", ParticipantIDs: fromIDs, EmailAddresses: fromAddresses}, + {Type: "to", ParticipantIDs: toIDs, EmailAddresses: toAddresses}, + {Type: "cc", ParticipantIDs: ccIDs, EmailAddresses: ccAddresses}, + {Type: "bcc", ParticipantIDs: bccIDs, EmailAddresses: bccAddresses}, + }, + FTS: &store.FTSDoc{ + Subject: parsed.Subject, + Body: parsed.BodyText, + FromAddr: firstAddress(parsed.From), + ToAddrs: strings.Join(toAddresses, " "), + CcAddrs: strings.Join(ccAddresses, " "), + }, + } + if attachmentWrites != nil { + data.MIMEAttachmentReplacement = &attachmentWrites + } + return data + } + return participants, build +} + +func addressStrings(addresses []msgmime.Address) []string { + result := make([]string, len(addresses)) + for i, address := range addresses { + result[i] = address.Email + } + return result +} + +func firstAddress(addresses []msgmime.Address) string { + if len(addresses) == 0 { + return "" + } + return addresses[0].Email +} diff --git a/cmd/msgvault/cmd/serve_draft_lifecycle_cache_test.go b/cmd/msgvault/cmd/serve_draft_lifecycle_cache_test.go new file mode 100644 index 000000000..316aac33d --- /dev/null +++ b/cmd/msgvault/cmd/serve_draft_lifecycle_cache_test.go @@ -0,0 +1,370 @@ +package cmd + +import ( + "bytes" + "context" + "encoding/json/v2" + "errors" + "log/slog" + "os" + "path/filepath" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/api" + imaplib "go.kenn.io/msgvault/internal/imap" + "go.kenn.io/msgvault/internal/query" + "go.kenn.io/msgvault/internal/search" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil" +) + +func TestDraftReplyOutputFailureRefreshesCache(t *testing.T) { + testutil.SkipIfPostgres(t, "analytics cache rebuild uses a SQLite snapshot") + requirements := require.New(t) + assertions := assert.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + cacheRoot := t.TempDir() + cacheDB := filepath.Join(cacheRoot, "cache.db") + analyticsDir := filepath.Join(cacheRoot, "analytics") + checkSourceUnlocked := adapter.draftCacheRefresh + adapter.draftCacheRefresh = func(ctx context.Context, label string) error { + if err := checkSourceUnlocked(ctx, label); err != nil { + return err + } + if err := os.Remove(cacheDB); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + if err := fixture.store.BackupDatabase(cacheDB); err != nil { + return err + } + _, err := buildCache(cacheDB, analyticsDir, true) + return err + } + requirements.NoError(adapter.draftCacheRefresh(t.Context(), fixture.source.Identifier)) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + var created draftReplyOutput + err := adapter.runCLIReplyDraft(ctx, api.CLIRunRequest{ + Args: []string{"draft-reply", strconv.FormatInt(fixture.parentID, 10), "--from", testutil.IMAPTestUsername, "--body", "createdcache", "--json"}, + }, func(event api.CLIRunEvent) error { + requirements.NoError(json.Unmarshal([]byte(event.Data), &created)) + cancel() + return errors.New("output disconnected") + }) + requirements.ErrorContains(err, "output_failed") + draft, err := fixture.store.GetIMAPDraftContext(t.Context(), created.DraftID) + requirements.NoError(err) + engine, err := query.NewDuckDBEngine(analyticsDir, "", nil) + requirements.NoError(err) + t.Cleanup(func() { _ = engine.Close() }) + results, err := engine.SearchFast(t.Context(), search.Parse("createdcache"), query.MessageFilter{}, 100, 0) + requirements.NoError(err) + requirements.Len(results, 1, "created draft must reach search despite output failure and cancellation") + assertions.Equal(draft.CurrentMessageID, results[0].ID) +} + +func TestDraftEditCancelledAfterPublicationRefreshesCache(t *testing.T) { + testutil.SkipIfPostgres(t, "analytics cache rebuild uses a SQLite snapshot") + requirements := require.New(t) + assertions := assert.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + created := createReviewDraft(t, fixture, adapter, "original body") + cacheRoot := t.TempDir() + cacheDB := filepath.Join(cacheRoot, "cache.db") + analyticsDir := filepath.Join(cacheRoot, "analytics") + checkSourceUnlocked := adapter.draftCacheRefresh + adapter.draftCacheRefresh = func(ctx context.Context, label string) error { + if err := checkSourceUnlocked(ctx, label); err != nil { + return err + } + if err := os.Remove(cacheDB); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + if err := fixture.store.BackupDatabase(cacheDB); err != nil { + return err + } + _, err := buildCache(cacheDB, analyticsDir, true) + return err + } + requirements.NoError(adapter.draftCacheRefresh(t.Context(), fixture.source.Identifier)) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + previous := slog.Default() + slog.SetDefault(slog.New(reviewDraftCommitHandler{ + Handler: slog.NewTextHandler(&bytes.Buffer{}, &slog.HandlerOptions{Level: slog.LevelDebug}), + onCommit: func() { + draft, err := fixture.store.GetIMAPDraftContext(t.Context(), created.DraftID) + if err == nil && draft.Revision == 2 { + cancel() + } + }, + })) + defer slog.SetDefault(previous) + err := adapter.runCLIDraftLifecycle(ctx, api.CLIRunRequest{ + Args: []string{api.CLIRunDraftEditCommand, created.DraftID, "--revision", "1", "--body", "publishedcache", "--json"}, + }, func(api.CLIRunEvent) error { return nil }) + requirements.ErrorContains(err, "cancelled") + requirements.ErrorIs(ctx.Err(), context.Canceled) + slog.SetDefault(previous) + draft, err := fixture.store.GetIMAPDraftContext(t.Context(), created.DraftID) + requirements.NoError(err) + requirements.NotNil(draft.Pending) + assertions.Equal(store.IMAPDraftCodeCleanup, draft.Pending.Code) + assertions.Equal(int64(2), draft.Revision) + engine, err := query.NewDuckDBEngine(analyticsDir, "", nil) + requirements.NoError(err) + t.Cleanup(func() { _ = engine.Close() }) + results, err := engine.SearchFast(t.Context(), search.Parse("publishedcache"), query.MessageFilter{}, 100, 0) + requirements.NoError(err) + requirements.Len(results, 1) + assertions.Equal(draft.CurrentMessageID, results[0].ID) + rows, err := engine.Aggregate(t.Context(), query.ViewLabels, query.DefaultAggregateOptions()) + requirements.NoError(err) + var draftCount int64 + for _, row := range rows { + if row.Key == "Drafts" { + draftCount = row.Count + } + } + assertions.Equal(int64(2), draftCount, "both published and pending predecessor drafts remain visible") +} + +func TestDraftDeleteOutputFailureRefreshesCache(t *testing.T) { + testutil.SkipIfPostgres(t, "analytics cache rebuild uses a SQLite snapshot") + for _, resume := range []bool{false, true} { + name := "normal deletion" + if resume { + name = "pending completion" + } + t.Run(name, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + created := createReviewDraft(t, fixture, adapter, "deletedcache") + if resume { + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), created.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + requirements.NoError(fixture.store.RecordIMAPDraftOutcomeContext(t.Context(), created.DraftID, 1, store.IMAPDraftCodeRemoved, nil)) + } + cacheRoot := t.TempDir() + cacheDB := filepath.Join(cacheRoot, "cache.db") + analyticsDir := filepath.Join(cacheRoot, "analytics") + checkSourceUnlocked := adapter.draftCacheRefresh + adapter.draftCacheRefresh = func(ctx context.Context, label string) error { + if err := checkSourceUnlocked(ctx, label); err != nil { + return err + } + if err := os.Remove(cacheDB); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + if err := fixture.store.BackupDatabase(cacheDB); err != nil { + return err + } + _, err := buildCache(cacheDB, analyticsDir, true) + return err + } + requirements.NoError(adapter.draftCacheRefresh(t.Context(), fixture.source.Identifier)) + engine, err := query.NewDuckDBEngine(analyticsDir, "", nil) + requirements.NoError(err) + results, err := engine.SearchFast(t.Context(), search.Parse("deletedcache"), query.MessageFilter{HideDeletedFromSource: true}, 100, 0) + requirements.NoError(err) + requirements.Len(results, 1) + requirements.NoError(engine.Close()) + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + err = adapter.runCLIDraftLifecycle(ctx, api.CLIRunRequest{ + Args: []string{api.CLIRunDraftDeleteCommand, created.DraftID, "--revision", "1", "--json"}, + }, func(api.CLIRunEvent) error { + cancel() + return errors.New("output disconnected") + }) + requirements.ErrorContains(err, "output_failed") + draft, err := fixture.store.GetIMAPDraftContext(t.Context(), created.DraftID) + requirements.NoError(err) + requirements.NotNil(draft.DiscardedAt) + engine, err = query.NewDuckDBEngine(analyticsDir, "", nil) + requirements.NoError(err) + t.Cleanup(func() { _ = engine.Close() }) + results, err = engine.SearchFast(t.Context(), search.Parse("deletedcache"), query.MessageFilter{HideDeletedFromSource: true}, 100, 0) + requirements.NoError(err) + assertions.Empty(results, "completed deletion must reach search despite output failure and cancellation") + }) + } +} + +func TestDraftRecoverRefreshesCache(t *testing.T) { + testutil.SkipIfPostgres(t, "analytics cache rebuild uses a SQLite snapshot") + requirements := require.New(t) + assertions := assert.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + created := createReviewDraft(t, fixture, adapter, "recovercache") + draft, err := fixture.store.GetIMAPDraftContext(t.Context(), created.DraftID) + requirements.NoError(err) + _, err = fixture.store.ClaimIMAPDraftContext(t.Context(), created.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + client := imaplib.NewClient(fixture.config, testutil.IMAPTestPassword) + removed, err := client.RemoveDraft(t.Context(), recoveryTestReceipt(draft.CurrentReceipt)) + requirements.NoError(err) + requirements.True(removed.Complete) + requirements.NoError(client.Close()) + + cacheRoot := t.TempDir() + cacheDB := filepath.Join(cacheRoot, "cache.db") + analyticsDir := filepath.Join(cacheRoot, "analytics") + checkSourceUnlocked := adapter.draftCacheRefresh + adapter.draftCacheRefresh = func(ctx context.Context, label string) error { + if err := checkSourceUnlocked(ctx, label); err != nil { + return err + } + if err := os.Remove(cacheDB); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + if err := fixture.store.BackupDatabase(cacheDB); err != nil { + return err + } + _, err := buildCache(cacheDB, analyticsDir, true) + return err + } + requirements.NoError(adapter.draftCacheRefresh(t.Context(), fixture.source.Identifier)) + baselineEngine, err := query.NewDuckDBEngine(analyticsDir, "", nil) + requirements.NoError(err) + baselineResults, err := baselineEngine.SearchFast(t.Context(), search.Parse("recovercache"), query.MessageFilter{HideDeletedFromSource: true}, 100, 0) + requirements.NoError(err) + requirements.Len(baselineResults, 1) + baselineRows, err := baselineEngine.Aggregate(t.Context(), query.ViewLabels, query.DefaultAggregateOptions()) + requirements.NoError(err) + var baselineDraftCount int64 + for _, row := range baselineRows { + if row.Key == "Drafts" { + baselineDraftCount = row.Count + } + } + assertions.Equal(int64(1), baselineDraftCount) + requirements.NoError(baselineEngine.Close()) + + events, err := runReviewLifecycle(t, adapter, "draft-recover", created.DraftID, "--revision", "1", "--json") + requirements.NoError(err) + requirements.Len(events, 1) + engine, err := query.NewDuckDBEngine(analyticsDir, "", nil) + requirements.NoError(err) + t.Cleanup(func() { _ = engine.Close() }) + results, err := engine.SearchFast(t.Context(), search.Parse("recovercache"), query.MessageFilter{HideDeletedFromSource: true}, 100, 0) + requirements.NoError(err) + assertions.Empty(results) + rows, err := engine.Aggregate(t.Context(), query.ViewLabels, query.DefaultAggregateOptions()) + requirements.NoError(err) + var draftCount int64 + for _, row := range rows { + if row.Key == "Drafts" { + draftCount = row.Count + } + } + assertions.Equal(int64(0), draftCount) +} + +func TestDraftRecoverPublishedEditCleanupRefreshesCache(t *testing.T) { + testutil.SkipIfPostgres(t, "analytics cache rebuild uses a SQLite snapshot") + requirements := require.New(t) + assertions := assert.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + created := createReviewDraft(t, fixture, adapter, "original") + draft, err := fixture.store.GetIMAPDraftContext(t.Context(), created.DraftID) + requirements.NoError(err) + adapter.draftCacheRefresh = func(context.Context, string) error { return nil } + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + previous := slog.Default() + slog.SetDefault(slog.New(reviewDraftCommitHandler{ + Handler: slog.NewTextHandler(&bytes.Buffer{}, &slog.HandlerOptions{Level: slog.LevelDebug}), + onCommit: func() { + latest, loadErr := fixture.store.GetIMAPDraftContext(t.Context(), created.DraftID) + if loadErr == nil && latest.Revision == 2 { + cancel() + } + }, + })) + var editEvents []api.CLIRunEvent + err = adapter.runCLIDraftLifecycle(ctx, api.CLIRunRequest{ + Args: []string{api.CLIRunDraftEditCommand, created.DraftID, "--revision", "1", "--body", "recoverpublishedcache", "--json"}, + }, func(event api.CLIRunEvent) error { + editEvents = append(editEvents, event) + return nil + }) + slog.SetDefault(previous) + requirements.ErrorContains(err, "cancelled") + requirements.NotEmpty(editEvents) + published, err := fixture.store.GetIMAPDraftContext(t.Context(), created.DraftID) + requirements.NoError(err) + requirements.Equal(int64(2), published.Revision) + requirements.NotNil(published.Pending) + client := imaplib.NewClient(fixture.config, testutil.IMAPTestPassword) + removed, err := client.RemoveDraft(t.Context(), recoveryTestReceipt(draft.CurrentReceipt)) + requirements.NoError(err) + requirements.True(removed.Complete) + requirements.NoError(client.Close()) + + cacheRoot := t.TempDir() + cacheDB := filepath.Join(cacheRoot, "cache.db") + analyticsDir := filepath.Join(cacheRoot, "analytics") + checkSourceUnlocked := adapter.draftCacheRefresh + adapter.draftCacheRefresh = func(ctx context.Context, label string) error { + if err := checkSourceUnlocked(ctx, label); err != nil { + return err + } + if err := os.Remove(cacheDB); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + if err := fixture.store.BackupDatabase(cacheDB); err != nil { + return err + } + _, err := buildCache(cacheDB, analyticsDir, true) + return err + } + requirements.NoError(adapter.draftCacheRefresh(t.Context(), fixture.source.Identifier)) + baselineEngine, err := query.NewDuckDBEngine(analyticsDir, "", nil) + requirements.NoError(err) + baselineResults, err := baselineEngine.SearchFast(t.Context(), search.Parse("recoverpublishedcache"), query.MessageFilter{HideDeletedFromSource: true}, 100, 0) + requirements.NoError(err) + assertions.Len(baselineResults, 1) + baselineRows, err := baselineEngine.Aggregate(t.Context(), query.ViewLabels, query.DefaultAggregateOptions()) + requirements.NoError(err) + var baselineDraftCount int64 + for _, row := range baselineRows { + if row.Key == "Drafts" { + baselineDraftCount = row.Count + } + } + assertions.Equal(int64(2), baselineDraftCount) + requirements.NoError(baselineEngine.Close()) + + events, err := runReviewLifecycle(t, adapter, api.CLIRunDraftRecoverCommand, created.DraftID, "--revision", "2", "--json") + requirements.NoError(err) + requirements.Len(events, 1) + engine, err := query.NewDuckDBEngine(analyticsDir, "", nil) + requirements.NoError(err) + t.Cleanup(func() { _ = engine.Close() }) + results, err := engine.SearchFast(t.Context(), search.Parse("recoverpublishedcache"), query.MessageFilter{HideDeletedFromSource: true}, 100, 0) + requirements.NoError(err) + latest, err := fixture.store.GetIMAPDraftContext(t.Context(), created.DraftID) + requirements.NoError(err) + assertions.Len(results, 1) + assertions.Equal(latest.CurrentMessageID, results[0].ID) + rows, err := engine.Aggregate(t.Context(), query.ViewLabels, query.DefaultAggregateOptions()) + requirements.NoError(err) + var draftCount int64 + for _, row := range rows { + if row.Key == "Drafts" { + draftCount = row.Count + } + } + assertions.Equal(int64(1), draftCount) +} diff --git a/cmd/msgvault/cmd/serve_draft_lifecycle_retry_test.go b/cmd/msgvault/cmd/serve_draft_lifecycle_retry_test.go new file mode 100644 index 000000000..90807f714 --- /dev/null +++ b/cmd/msgvault/cmd/serve_draft_lifecycle_retry_test.go @@ -0,0 +1,58 @@ +package cmd + +import ( + "bytes" + "context" + "encoding/json/v2" + "log/slog" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/api" +) + +func TestDraftDeleteCancelledBeforeStoreCanRetry(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + created := createReviewDraft(t, fixture, adapter, "keep until retry") + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + previous := slog.Default() + slog.SetDefault(slog.New(reviewDraftCommitHandler{ + Handler: slog.NewTextHandler(&bytes.Buffer{}, &slog.HandlerOptions{Level: slog.LevelDebug}), + onCommit: func() { + draft, err := fixture.store.GetIMAPDraftContext(context.Background(), created.DraftID) + if err == nil && draft.Pending != nil { + cancel() + } + }, + })) + t.Cleanup(func() { slog.SetDefault(previous) }) + args := []string{api.CLIRunDraftDeleteCommand, created.DraftID, "--revision", "1", "--json"} + var events []api.CLIRunEvent + err := adapter.runCLIDraftLifecycle(ctx, api.CLIRunRequest{Args: args}, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + requirements.ErrorContains(err, "cancelled") + requirements.ErrorIs(ctx.Err(), context.Canceled) + draft, err := fixture.store.GetIMAPDraftContext(t.Context(), created.DraftID) + requirements.NoError(err) + requirements.Nil(draft.Pending) + assertions.Equal(int64(1), draft.Revision) + assertions.Nil(draft.DiscardedAt) + requirements.Len(events, 1) + var output draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Equal("active", output.Status) + assertions.False(output.ManualReconciliation) + slog.SetDefault(previous) + _, err = runReviewLifecycle(t, adapter, args...) + requirements.NoError(err) + draft, err = fixture.store.GetIMAPDraftContext(t.Context(), created.DraftID) + requirements.NoError(err) + assertions.NotNil(draft.DiscardedAt) +} diff --git a/cmd/msgvault/cmd/serve_draft_lifecycle_review_test.go b/cmd/msgvault/cmd/serve_draft_lifecycle_review_test.go new file mode 100644 index 000000000..a0adf4513 --- /dev/null +++ b/cmd/msgvault/cmd/serve_draft_lifecycle_review_test.go @@ -0,0 +1,1656 @@ +package cmd + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + emersionimap "github.com/emersion/go-imap/v2" + "github.com/emersion/go-imap/v2/imapclient" + "github.com/emersion/go-imap/v2/imapserver" + "github.com/emersion/go-imap/v2/imapserver/imapmemserver" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/gmail" + imaplib "go.kenn.io/msgvault/internal/imap" + "go.kenn.io/msgvault/internal/query" + "go.kenn.io/msgvault/internal/search" + "go.kenn.io/msgvault/internal/store" + msgsync "go.kenn.io/msgvault/internal/sync" + "go.kenn.io/msgvault/internal/testutil" +) + +type reviewDraftCommitHandler struct { + slog.Handler + + onCommit func() +} + +func (h reviewDraftCommitHandler) Handle(_ context.Context, record slog.Record) error { + if record.Message == "sql tx commit" || record.Message == "sql tx slow" { + h.onCommit() + } + return nil +} + +func TestDraftLifecycleCancelledAfterClaimBeforeAppend(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + barrier := &reviewCmdStoreBarrier{stored: make(chan struct{}), release: make(chan struct{})} + addr, user := startReviewCmdStoreBarrierServer(t, barrier) + fixture := newReviewManagedLifecycleFixtureOnServer(t, addr, func() { + testutil.AppendIMAPRawMessage(t, user, "Drafts", []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Original\r\n\r\noriginal\r\n")) + }) + barrier.appendCalls.Store(0) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + previous := slog.Default() + slog.SetDefault(slog.New(reviewDraftCommitHandler{ + Handler: slog.NewTextHandler(&bytes.Buffer{}, &slog.HandlerOptions{Level: slog.LevelDebug}), + onCommit: func() { + if draft, err := fixture.store.GetIMAPDraftContext(context.Background(), fixture.draft.DraftID); err == nil && draft.Pending != nil { + cancel() + } + }, + })) + defer slog.SetDefault(previous) + var events []api.CLIRunEvent + err := fixture.adapter.runCLIDraftLifecycle(ctx, api.CLIRunRequest{Args: []string{"draft-edit", fixture.draft.DraftID, "--revision", "1", "--body", "unsent", "--json"}}, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + requirements.ErrorContains(err, "cancelled") + requirements.ErrorIs(ctx.Err(), context.Canceled) + assertions.Zero(barrier.appendCalls.Load()) + pending, err := fixture.store.GetIMAPDraftContext(context.Background(), fixture.draft.DraftID) + requirements.NoError(err) + assertions.Nil(pending.Pending) + assertions.Equal(int64(1), pending.Revision) + assertions.Equal(uint32(1), reviewDraftMailboxCount(t, addr)) + requirements.Len(events, 1) + var output draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Equal(draftLifecycleActive, output.Status) + assertions.False(output.ManualReconciliation) +} + +func TestDraftLifecycleRejectedClaimPersistenceEvidence(t *testing.T) { + testutil.SkipIfPostgres(t, "claim failure injection uses SQLite triggers") + for _, fault := range []string{"record", "clear"} { + t.Run(fault, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + barrier := &reviewCmdStoreBarrier{stored: make(chan struct{}), release: make(chan struct{}), appendErr: &emersionimap.Error{Type: emersionimap.StatusResponseTypeNo, Text: "injected rejection"}} + addr, user := startReviewCmdStoreBarrierServer(t, barrier) + fixture := newReviewManagedLifecycleFixtureOnServer(t, addr, func() { + testutil.AppendIMAPRawMessage(t, user, "Drafts", []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Original\r\n\r\noriginal\r\n")) + }) + barrier.appendCalls.Store(0) + condition := "NEW.pending_code = 'append_rejected'" + if fault == "clear" { + condition = "OLD.pending_operation IS NOT NULL AND NEW.pending_operation IS NULL" + } + _, err := fixture.store.DB().Exec("CREATE TRIGGER fail_rejected_claim BEFORE UPDATE ON imap_drafts WHEN " + condition + " BEGIN SELECT RAISE(FAIL, 'injected rejection persistence failure'); END") + requirements.NoError(err) + events, err := runReviewLifecycle(t, fixture.adapter, "draft-edit", fixture.draft.DraftID, "--revision", "1", "--body", "candidate", "--json") + requirements.Error(err) + requirements.Len(events, 1) + var output draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + if fault == "clear" { + assertions.Equal("pending", output.Status) + assertions.True(output.ManualReconciliation) + assertions.Contains(output.CandidateContent, "candidate") + assertions.Equal("local_persistence_failed", err.Error()) + coded, ok := errors.AsType[*api.CLIRunCodedError](err) + requirements.True(ok) + requirements.ErrorContains(coded.Err, "injected rejection persistence failure") + } else { + assertions.Equal(draftLifecycleActive, output.Status) + assertions.False(output.ManualReconciliation) + } + pending, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + if fault == "clear" { + requirements.NotNil(pending.Pending) + assertions.Contains(string(pending.Pending.Raw), "candidate") + } else { + assertions.Nil(pending.Pending) + } + assertions.Equal(int64(1), pending.Revision) + assertions.Equal(int32(1), barrier.appendCalls.Load()) + assertions.Equal(uint32(1), reviewDraftMailboxCount(t, addr)) + }) + } +} + +func TestDraftReplyHumanLifecycleHandle(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + server := httptest.NewServer(api.NewServerWithOptions(api.ServerOptions{Config: &config.Config{HomeDir: t.TempDir()}, Store: adapter, Logger: slog.New(slog.DiscardHandler)}).Router()) + t.Cleanup(server.Close) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx + root := &cobra.Command{Use: "msgvault"} + root.SetContext(testCtx) + root.AddCommand(newDraftReplyCommand()) + silenceUsageInRunE(root) + var stdout bytes.Buffer + root.SetOut(&stdout) + root.SetArgs([]string{"draft-reply", strconv.FormatInt(fixture.parentID, 10), "--from", testutil.IMAPTestUsername, "--body", "handle"}) + requirements.NoError(root.ExecuteContext(testCtx)) + var draftID string + requirements.NoError(fixture.store.DB().QueryRow("SELECT draft_id FROM imap_drafts").Scan(&draftID)) + assertions.Contains(stdout.String(), "draft "+draftID+" revision 1") + draft, err := fixture.store.GetIMAPDraftContext(testCtx, draftID) + requirements.NoError(err) + assertions.Equal(int64(1), draft.Revision) +} + +func TestEmitDraftLifecycleOutputHuman(t *testing.T) { + tests := []struct { + name string + output draftLifecycleOutput + want []string + }{ + { + name: "sanitizes metadata and multiline content", + output: draftLifecycleOutput{ + Status: "status\x1b[31m\r\ninjected", + DraftID: "draft-ID\x1b[31m\r\nspoofed", + Revision: 7, + Lifecycle: "life\x1b]0;title\x07cycle", + Receipt: draftLifecycleReceipt{Mailbox: "Mailbox\x1b]0;x\x07\r\nreceipt", UIDValidity: 11, UID: 12}, + Content: "line one\r\nline \x1b[31mred\x1b[0m\nline\u009bthree", + RawMIME: "raw-only-value", + CandidateContent: "candidate one\ncandidate \x1b]0;evil\x07two\r\nend", + PendingOperation: "edit\x1b[31m\r\noperation", + PendingCode: "pending\x1b[31m\r\ncode", + PendingReceipt: &draftLifecycleReceipt{Mailbox: "Pending\x1b[31m\r\nmail", UIDValidity: 13, UID: 14}, + ManualReconciliation: true, + }, + want: []string{ + "draft draft-ID spoofed revision 7 lifecycle", + "status: status injected", + "receipt (revision 7): Mailbox receipt uidvalidity=11 uid=12", + "content:\nline one\nline red\nlinethree\n", + "pending operation: edit operation", + "candidate content:\ncandidate one\ncandidate two\nend\n", + "pending receipt (revision 7): Pending mail uidvalidity=13 uid=14", + "provider outcome: pending code", + "old draft ID remains blocked at revision 7", + "manual action: reconcile the provider receipt and local state before retrying", + }, + }, + { + name: "sanitizes pending old provider receipt", + output: draftLifecycleOutput{ + Status: "pending", + DraftID: "pending-id", + Revision: 8, + Lifecycle: "active", + Receipt: draftLifecycleReceipt{Mailbox: "Drafts", UIDValidity: 21, UID: 22}, + Content: "stored\nbody", + CandidateContent: "candidate\nbody", + PendingOperation: "edit", + PendingReceipt: &draftLifecycleReceipt{Mailbox: "replacement", UIDValidity: 21, UID: 23}, + Observation: &draftLifecycleObservation{ + State: "present", Code: "old\x1b[31mcode", Mailbox: "old\x1b]0;evil\x07mail", + UIDValidity: 21, UID: 22, Present: true, + }, + }, + want: []string{ + "old provider receipt: oldmail uidvalidity=21 uid=22 (oldcode)", + "provider outcome: oldcode", + }, + }, + { + name: "sanitizes accepted replacement receipt", + output: draftLifecycleOutput{ + Status: "accepted_local_failed", + DraftID: "accepted-id", + Revision: 9, + Lifecycle: "active", + Receipt: draftLifecycleReceipt{Mailbox: "Drafts", UIDValidity: 31, UID: 32}, + Content: "stored", + PendingOperation: "edit", + PendingReceipt: &draftLifecycleReceipt{Mailbox: "replacement", UIDValidity: 31, UID: 33}, + ProviderObservation: &draftLifecycleObservation{ + State: "present", Code: "replacement\x1b[31mcode", Mailbox: "new\x1b]0;evil\x07mail", + UIDValidity: 31, UID: 33, Present: true, + }, + Observation: &draftLifecycleObservation{ + State: "missing", Code: "old\x1b[31mcode", Mailbox: "old\x1b]0;evil\x07mail", + }, + }, + want: []string{ + "acknowledged replacement receipt: newmail uidvalidity=31 uid=33 (replacementcode)", + "provider outcome: replacementcode", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + var events []api.CLIRunEvent + err := emitDraftLifecycleOutput(func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }, cliStreamStderr, draftLifecycleIntent{}, tt.output) + requirements.NoError(err) + requirements.Len(events, 1) + for _, want := range tt.want { + assertions.Contains(events[0].Data, want) + } + if tt.output.RawMIME != "" { + assertions.NotContains(events[0].Data, tt.output.RawMIME) + } + for _, control := range []string{"\x00", "\x07", "\x1b", "\r", "\u009b"} { + assertions.NotContains(events[0].Data, control) + } + }) + } +} + +func TestEmitDraftLifecycleOutputJSONRoundTrip(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + output := draftLifecycleOutput{ + Status: "accepted_local_failed\x1b[31m\r\nstatus", + DraftID: "draft\x1b]0;id\x07value", + Revision: 42, + Lifecycle: "active\u009bstate", + MessageID: 43, + SourceID: 44, + Receipt: draftLifecycleReceipt{Mailbox: "Drafts\r\nmailbox", UIDValidity: 45, UID: 46}, + Content: "stored\r\n\x1b[31mcontent\x1b[0m\u009b", + RawMIME: "From: sender@example.com\r\n\r\n\x1b[31mraw\x1b[0m\u009b", + CandidateContent: "candidate\r\n\x1b]0;candidate\x07content", + PendingOperation: "edit\r\noperation", + PendingCode: "pending\u009bcode", + PendingReceipt: &draftLifecycleReceipt{Mailbox: "Replacement\x1b[31m\r\nmailbox", UIDValidity: 47, UID: 48}, + ProviderObservation: &draftLifecycleObservation{ + State: "present\x1b[31m", Code: "replacement\r\ncode", Mailbox: "Replacement\x1b]0;mail\x07box", + UIDValidity: 49, UID: 50, Flags: []string{"\\Seen", "flag\u009bvalue"}, Present: true, + Draft: true, Deleted: false, Complete: true, UIDPlus: true, + }, + Observation: &draftLifecycleObservation{ + State: "missing\x1b[31m", Code: "old\r\ncode", Mailbox: "Old\x1b]0;mail\x07box", + UIDValidity: 51, UID: 52, Flags: []string{"\\Draft", "old\u009bflag"}, Present: false, + Draft: false, Deleted: true, Complete: false, UIDPlus: false, + }, + ManualReconciliation: true, + } + + var events []api.CLIRunEvent + err := emitDraftLifecycleOutput(func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }, cliStreamStdout, draftLifecycleIntent{JSON: true}, output) + requirements.NoError(err) + requirements.Len(events, 1) + var decoded draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &decoded)) + assertions.Equal(output, decoded) +} + +func TestDraftLifecycleAcceptedReadFailureEvidence(t *testing.T) { + testutil.SkipIfPostgres(t, "local read fault injection alters SQLite tables") + for _, fault := range []string{"message", "reply_link"} { + for _, asJSON := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/json=%t", fault, asJSON), func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + barrier := &reviewCmdStoreBarrier{stored: make(chan struct{}), release: make(chan struct{})} + addr, user := startReviewCmdStoreBarrierServer(t, barrier) + fixture := newReviewManagedLifecycleFixtureOnServer(t, addr, func() { + testutil.AppendIMAPRawMessage(t, user, "Drafts", []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Original\r\n\r\noriginal\r\n")) + }) + barrier.appendCalls.Store(0) + barrier.storeCalls.Store(0) + barrier.expungeCalls.Store(0) + var faultErr error + barrier.onAppend = func() { + if fault == "message" { + _, faultErr = fixture.store.DB().Exec("ALTER TABLE message_bodies RENAME TO fault_message_bodies") + } else { + _, faultErr = fixture.store.DB().Exec("ALTER TABLE messages RENAME COLUMN reply_to_message_id TO fault_reply_to_message_id") + } + } + server := httptest.NewServer(api.NewServerWithOptions(api.ServerOptions{Config: &config.Config{HomeDir: t.TempDir()}, Store: fixture.adapter, Logger: slog.New(slog.DiscardHandler)}).Router()) + t.Cleanup(server.Close) + testCtx := configureRemoteDaemonForTest(t, server.URL) + root := &cobra.Command{Use: "msgvault"} + root.SetContext(testCtx) + root.AddCommand(newDraftEditCommand()) + silenceUsageInRunE(root) + var stdout, stderr bytes.Buffer + root.SetOut(&stdout) + root.SetErr(&stderr) + args := []string{"draft-edit", fixture.draft.DraftID, "--revision", "1", "--body", "saved candidate"} + if asJSON { + args = append(args, "--json") + } + root.SetArgs(args) + err := root.ExecuteContext(testCtx) + requirements.NoError(faultErr) + requirements.ErrorContains(err, "accepted_local_failed") + assertions.Empty(stdout.String()) + if fault == "message" { + _, err = fixture.store.DB().Exec("ALTER TABLE fault_message_bodies RENAME TO message_bodies") + } else { + _, err = fixture.store.DB().Exec("ALTER TABLE messages RENAME COLUMN fault_reply_to_message_id TO reply_to_message_id") + } + requirements.NoError(err) + if asJSON { + var output draftLifecycleOutput + requirements.NoError(json.Unmarshal(stderr.Bytes(), &output)) + assertions.Equal("accepted_local_failed", output.Status) + assertions.Equal(fixture.draft.DraftID, output.DraftID) + assertions.Contains(output.CandidateContent, "saved candidate") + requirements.NotNil(output.PendingReceipt) + assertions.Equal(uint32(2), output.PendingReceipt.UID) + assertions.True(output.ManualReconciliation) + } else { + assertions.Contains(stderr.String(), "status: accepted_local_failed") + assertions.Contains(stderr.String(), "saved candidate") + assertions.Contains(stderr.String(), "pending receipt (revision 1): Drafts uidvalidity=1 uid=2") + assertions.Contains(stderr.String(), "old draft ID remains blocked at revision 1") + } + pending, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + requirements.NotNil(pending.Pending) + requirements.NotNil(pending.Pending.ReplacementReceipt) + assertions.Equal(uint32(2), pending.Pending.ReplacementReceipt.UID) + assertions.Equal(int64(1), pending.Revision) + assertions.Contains(string(pending.Pending.Raw), "saved candidate") + assertions.Equal(int32(1), barrier.appendCalls.Load()) + assertions.Zero(barrier.storeCalls.Load()) + assertions.Zero(barrier.expungeCalls.Load()) + assertions.Equal(uint32(2), reviewDraftMailboxCount(t, addr)) + }) + } + } +} + +func TestDraftLifecyclePublicationFailure(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + created := createReviewDraft(t, fixture, adapter, "original") + conversationID, err := fixture.store.EnsureConversation(fixture.source.ID, "publication-conflict", "Publication conflict") + requirements.NoError(err) + _, err = fixture.store.PersistMessage(&store.MessagePersistData{ + Message: &store.Message{ + SourceID: fixture.source.ID, SourceMessageID: "Drafts|2", ConversationID: conversationID, + MessageType: store.MessageTypeEmail, + }, + BodyText: sql.NullString{String: "conflict", Valid: true}, + RawMIME: []byte("From: conflict@example.com\r\n\r\nconflict\r\n"), + }) + requirements.NoError(err) + + server := httptest.NewServer(api.NewServerWithOptions(api.ServerOptions{Config: &config.Config{HomeDir: t.TempDir()}, Store: adapter, Logger: slog.New(slog.DiscardHandler)}).Router()) + t.Cleanup(server.Close) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx + root := &cobra.Command{Use: "msgvault"} + root.SetContext(testCtx) + root.AddCommand(newDraftEditCommand()) + silenceUsageInRunE(root) + var stdout, stderr bytes.Buffer + root.SetOut(&stdout) + root.SetErr(&stderr) + root.SetArgs([]string{api.CLIRunDraftEditCommand, created.DraftID, "--revision", "1", "--body", "candidate", "--json"}) + err = root.ExecuteContext(testCtx) + assertions.Empty(stdout.String()) + events := []api.CLIRunEvent{{Type: cliStreamStderr, Data: stderr.String()}} + requirements.Error(err) + assertions.Equal("accepted_local_failed", err.Error()) + requirements.Len(events, 1) + assertions.Equal(cliStreamStderr, events[0].Type) + var output struct { + Status string `json:"status"` + Revision int64 `json:"revision"` + CandidateContent string `json:"candidate_content"` + PendingOperation string `json:"pending_operation"` + PendingReceipt struct { + UID uint32 `json:"uid"` + } `json:"pending_receipt"` + } + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Equal("accepted_local_failed", output.Status) + assertions.Equal(int64(1), output.Revision) + assertions.Contains(output.CandidateContent, "candidate") + assertions.Equal("edit", output.PendingOperation) + assertions.Equal(uint32(2), output.PendingReceipt.UID) +} + +func TestDraftLifecycleHumanAcknowledgedReceiptWhenRecordFails(t *testing.T) { + testutil.SkipIfPostgres(t, "receipt write fault injection uses a SQLite trigger") + requirements := require.New(t) + assertions := assert.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + created := createReviewDraft(t, fixture, adapter, "original") + _, err := fixture.store.DB().Exec(` + CREATE TRIGGER fail_draft_replacement_receipt + BEFORE UPDATE OF pending_replacement_uid ON imap_drafts + WHEN NEW.pending_replacement_uid IS NOT NULL + AND OLD.pending_replacement_uid IS NULL + BEGIN + SELECT RAISE(FAIL, 'injected replacement receipt failure'); + END + `) + requirements.NoError(err) + + server := httptest.NewServer(api.NewServerWithOptions(api.ServerOptions{ + Config: &config.Config{HomeDir: t.TempDir()}, Store: adapter, + Logger: slog.New(slog.DiscardHandler), + }).Router()) + t.Cleanup(server.Close) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx + root := &cobra.Command{Use: "msgvault"} + root.SetContext(testCtx) + root.AddCommand(newDraftEditCommand()) + silenceUsageInRunE(root) + var stdout, stderr bytes.Buffer + root.SetOut(&stdout) + root.SetErr(&stderr) + root.SetArgs([]string{api.CLIRunDraftEditCommand, created.DraftID, + "--revision", "1", "--body", "candidate"}) + err = root.ExecuteContext(testCtx) + requirements.Error(err) + assertions.Equal("accepted_local_failed", err.Error()) + assertions.Empty(stdout.String()) + assertions.Contains(stderr.String(), "status: accepted_local_failed") + assertions.Contains(stderr.String(), "pending operation: edit") + assertions.Contains(stderr.String(), "provider outcome: append_uidplus") + assertions.Contains(stderr.String(), "acknowledged replacement receipt: Drafts uidvalidity=1 uid=2") + assertions.Contains(stderr.String(), "receipt (revision 1): Drafts uidvalidity=1 uid=1") + assertions.Contains(stderr.String(), "candidate content:") + assertions.Contains(stderr.String(), "candidate") + assertions.Contains(stderr.String(), "old draft ID remains blocked at revision 1") + assertions.Contains(stderr.String(), "manual action:") + assertions.NotContains(stderr.String(), "pending receipt:") + assertions.NotContains(stderr.String(), "Error: accepted_local_failed") + + pending, err := fixture.store.GetIMAPDraftContext(testCtx, created.DraftID) + requirements.NoError(err) + requirements.Equal(int64(1), pending.Revision) + requirements.NotNil(pending.Pending) + requirements.Nil(pending.Pending.ReplacementReceipt) + requirements.Equal(uint32(1), pending.Pending.OriginalReceipt.UID) + requirements.Contains(string(pending.Pending.Raw), "candidate") + provider := imaplib.NewClient(fixture.config, testutil.IMAPTestPassword) + oldObservation, err := provider.InspectDraft(testCtx, imaplib.DraftReceipt{ + Mailbox: "Drafts", UIDValidity: 1, UID: 1, + }) + requirements.NoError(err) + newObservation, err := provider.InspectDraft(testCtx, imaplib.DraftReceipt{ + Mailbox: "Drafts", UIDValidity: 1, UID: 2, + }) + requirements.NoError(err) + assertions.True(oldObservation.Present) + assertions.True(newObservation.Present) + requirements.NoError(provider.Close()) +} + +func TestDraftLifecycleUnknownAppend(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + addr, user := startReviewDropAppendServer(t) + dbPath := filepath.Join(t.TempDir(), "lifecycle.db") + fixture := newReviewManagedLifecycleFixtureOnServerWithDBPath(t, addr, dbPath, func() { + testutil.AppendIMAPRawMessage(t, user, "Drafts", []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Original\r\n\r\noriginal\r\n")) + }) + var providerCalls atomic.Int32 + fixture.adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + providerCalls.Add(1) + return imaplib.NewClient(fixture.config, testutil.IMAPTestPassword), nil + } + + events, err := runReviewLifecycle(t, fixture.adapter, api.CLIRunDraftEditCommand, fixture.draft.DraftID, "--revision", "1", "--body", "candidate", "--json") + requirements.Error(err) + assertions.Equal("remote_unknown", err.Error()) + requirements.Len(events, 1) + assertions.Equal(cliStreamStderr, events[0].Type) + assertions.Contains(events[0].Data, `"candidate_content"`) + assertions.Contains(events[0].Data, "candidate") + assertions.Equal(uint32(2), reviewDraftMailboxCount(t, fixture.config.Addr())) + pending, err := fixture.store.GetIMAPDraftContext(context.Background(), fixture.draft.DraftID) + requirements.NoError(err) + requirements.NotNil(pending.Pending) + assertions.Equal("remote_unknown", pending.Pending.Code) + assertions.Equal(int32(1), providerCalls.Load()) + + requirements.NoError(fixture.store.Close()) + reopened, err := store.OpenForTest(dbPath) + requirements.NoError(err) + t.Cleanup(func() { _ = reopened.Close() }) + reopenedDraft, err := reopened.GetIMAPDraftContext(context.Background(), fixture.draft.DraftID) + requirements.NoError(err) + requirements.NotNil(reopenedDraft.Pending) + assertions.Equal("remote_unknown", reopenedDraft.Pending.Code) + assertions.Contains(string(reopenedDraft.Pending.Raw), "candidate") + freshAdapter := &storeAPIAdapter{ + store: reopened, + draftPolicy: []config.IMAPDraftSource{{SourceID: fixture.source.ID, Enabled: true, Mailbox: "Drafts"}}, + draftClientFactory: func(context.Context, *store.Source) (*imaplib.Client, error) { + providerCalls.Add(1) + return imaplib.NewClient(fixture.config, testutil.IMAPTestPassword), nil + }, + } + events, err = runReviewLifecycle(t, freshAdapter, api.CLIRunDraftEditCommand, fixture.draft.DraftID, "--revision", "1", "--body", "retry", "--json") + requirements.Error(err) + assertions.Equal("pending_operation", err.Error()) + assertions.Empty(events) + assertions.Equal(int32(1), providerCalls.Load()) +} + +func TestDraftLifecycleHumanUnknownAppendThroughHTTP(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + addr, user := startReviewDropAppendServer(t) + fixture := newReviewManagedLifecycleFixtureOnServer(t, addr, func() { + testutil.AppendIMAPRawMessage(t, user, "Drafts", []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Original\r\n\r\noriginal\r\n")) + }) + + server := httptest.NewServer(api.NewServerWithOptions(api.ServerOptions{ + Config: &config.Config{HomeDir: t.TempDir()}, Store: fixture.adapter, + Logger: slog.New(slog.DiscardHandler), + }).Router()) + t.Cleanup(server.Close) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx + root := &cobra.Command{Use: "msgvault"} + root.SetContext(testCtx) + root.AddCommand(newDraftEditCommand()) + silenceUsageInRunE(root) + var stdout, stderr bytes.Buffer + root.SetOut(&stdout) + root.SetErr(&stderr) + root.SetArgs([]string{api.CLIRunDraftEditCommand, fixture.draft.DraftID, + "--revision", "1", "--body", "candidate"}) + err := root.ExecuteContext(testCtx) + requirements.Error(err) + assertions.Equal("remote_unknown", err.Error()) + assertions.Empty(stdout.String()) + assertions.Contains(stderr.String(), "status: pending") + assertions.Contains(stderr.String(), "pending operation: edit") + assertions.Contains(stderr.String(), "provider outcome: remote_unknown") + assertions.Contains(stderr.String(), "candidate content:") + assertions.Contains(stderr.String(), "candidate") + assertions.Contains(stderr.String(), "receipt (revision 1): Drafts") + assertions.Contains(stderr.String(), "old draft ID remains blocked at revision 1") + assertions.Contains(stderr.String(), "manual action:") + assertions.NotContains(stderr.String(), "Error: remote_unknown") + assertions.Equal(1, strings.Count(stderr.String(), "provider outcome:")) +} + +func TestDraftLifecycleRetainedGet(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + created := createReviewDraft(t, fixture, adapter, "retained") + var providerCalls atomic.Int32 + adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + providerCalls.Add(1) + return nil, errors.New("draft-get must not connect to IMAP") + } + _, err := fixture.store.DB().Exec(fixture.store.Rebind(` + UPDATE messages SET deleted_from_source_at = CURRENT_TIMESTAMP WHERE id = ? + `), created.MessageID) + requirements.NoError(err) + events, err := runReviewLifecycle(t, adapter, api.CLIRunDraftGetCommand, created.DraftID, "--json") + requirements.NoError(err) + requirements.Len(events, 1) + assertions.Contains(events[0].Data, "retained") + assertions.Contains(events[0].Data, `"provider_observation":{"state":"not_checked","code":"not_checked"`) + assertions.Zero(providerCalls.Load()) + + _, err = fixture.store.ClaimIMAPDraftContext(context.Background(), created.DraftID, 1, store.IMAPDraftOperationEdit, []byte("candidate raw")) + requirements.NoError(err) + events, err = runReviewLifecycle(t, adapter, api.CLIRunDraftGetCommand, created.DraftID, "--json") + requirements.NoError(err) + assertions.Contains(events[0].Data, `"candidate_content":"candidate raw"`) +} + +func TestDraftLifecycleRepeatedDelete(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + created := createReviewDraft(t, fixture, adapter, "discard me") + events, err := runReviewLifecycle(t, adapter, api.CLIRunDraftDeleteCommand, created.DraftID, "--revision", "1", "--json") + requirements.NoError(err) + requirements.Len(events, 1) + var deleted struct { + Revision int64 `json:"revision"` + } + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &deleted)) + assertions.Equal(int64(2), deleted.Revision) + var providerCalls atomic.Int32 + adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + providerCalls.Add(1) + return nil, errors.New("discarded retry must be offline") + } + events, err = runReviewLifecycle(t, adapter, api.CLIRunDraftDeleteCommand, created.DraftID, "--revision", "2", "--json") + requirements.NoError(err) + requirements.Len(events, 1) + assertions.Contains(events[0].Data, `"status":"already_discarded"`) + assertions.Zero(providerCalls.Load()) + _, err = runReviewLifecycle(t, adapter, api.CLIRunDraftDeleteCommand, created.DraftID, "--revision", "1", "--json") + requirements.Error(err) + assertions.Equal("revision_mismatch", err.Error()) + _, err = runReviewLifecycle(t, adapter, api.CLIRunDraftEditCommand, created.DraftID, "--revision", "2", "--body", "new", "--json") + requirements.Error(err) + assertions.Equal("draft_discarded", err.Error()) +} + +func TestDraftLifecycleExactCopy(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + created := createReviewDraft(t, fixture, adapter, "copy") + copyClient := imaplib.NewClient(fixture.config, testutil.IMAPTestPassword) + copyReceipt, err := copyClient.AppendDraft(t.Context(), "Drafts", []byte("From: alice@example.com\r\nTo: "+testutil.IMAPTestUsername+"\r\nSubject: Question\r\n\r\ncopy\r\n")) + requirements.NoError(err) + requirements.NoError(copyClient.Close()) + requirements.NotEqual(uint32(0), copyReceipt.UID) + testutil.ExpungeIMAPMessage(t, fixture.config.Addr(), "Drafts", emersionimap.UID(1)) + events, err := runReviewLifecycle(t, adapter, api.CLIRunDraftEditCommand, created.DraftID, "--revision", "1", "--body", "must refuse", "--json") + requirements.Error(err) + assertions.Equal("absent", err.Error()) + requirements.Len(events, 1) + assertions.Contains(events[0].Data, `"code":"absent"`) + draft, err := fixture.store.GetIMAPDraftContext(context.Background(), created.DraftID) + requirements.NoError(err) + assertions.Nil(draft.Pending) + + second := createReviewDraft(t, fixture, adapter, "valid second draft") + events, err = runReviewLifecycle(t, adapter, api.CLIRunDraftEditCommand, second.DraftID, "--revision", "1", "--body", "works", "--json") + requirements.NoError(err) + requirements.Len(events, 1) + assertions.Contains(events[0].Data, `"revision":2`) + inspection := imaplib.NewClient(fixture.config, testutil.IMAPTestPassword) + observation, err := inspection.InspectDraft(t.Context(), imaplib.DraftReceipt{Mailbox: "Drafts", UIDValidity: copyReceipt.UIDValidity, UID: copyReceipt.UID}) + requirements.NoError(err) + assertions.True(observation.Present) +} + +func TestDraftLifecyclePolicy(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + created := createReviewDraft(t, fixture, adapter, "policy") + var providerCalls atomic.Int32 + adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + providerCalls.Add(1) + return nil, errors.New("policy refusal must precede provider") + } + adapter.draftPolicy[0].Mailbox = "Other" + events, err := runReviewLifecycle(t, adapter, api.CLIRunDraftEditCommand, created.DraftID, "--revision", "1", "--body", "blocked", "--json") + requirements.Error(err) + assertions.Equal("invalid_mailbox", err.Error()) + assertions.Empty(events) + assertions.Zero(providerCalls.Load()) + adapter.draftPolicy = nil + events, err = runReviewLifecycle(t, adapter, api.CLIRunDraftGetCommand, created.DraftID, "--json") + requirements.NoError(err) + requirements.Len(events, 1) + assertions.Contains(events[0].Data, "policy") + assertions.Zero(providerCalls.Load()) + + adapter.draftPolicy = []config.IMAPDraftSource{{SourceID: fixture.source.ID, Enabled: true, Mailbox: "Drafts"}} + badConfig, err := (&imaplib.Config{Host: "127.0.0.1", Port: 1, Username: testutil.IMAPTestUsername}).ToJSON() + requirements.NoError(err) + requirements.NoError(fixture.store.UpdateSourceSyncConfig(fixture.source.ID, badConfig)) + _, err = runReviewLifecycle(t, adapter, api.CLIRunDraftDeleteCommand, created.DraftID, "--revision", "1", "--json") + requirements.Error(err) + assertions.Equal("invalid_source", err.Error()) + assertions.Zero(providerCalls.Load()) +} + +func TestDraftLifecycleBoundaries(t *testing.T) { + requirements := require.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + created := createReviewDraft(t, fixture, adapter, "boundaries") + var providerCalls atomic.Int32 + adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + providerCalls.Add(1) + return nil, errors.New("local draft-get must not open provider") + } + for _, state := range []string{"active", "pending", "discarded"} { + t.Run(state, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + switch state { + case "pending": + _, err := fixture.store.ClaimIMAPDraftContext(context.Background(), created.DraftID, 1, store.IMAPDraftOperationEdit, []byte("pending")) + requirements.NoError(err) + case "discarded": + _, err := fixture.store.ClaimIMAPDraftContext(context.Background(), created.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + requirements.NoError(fixture.store.RecordIMAPDraftOutcomeContext(context.Background(), created.DraftID, 1, store.IMAPDraftCodeRemoved, nil)) + _, err = fixture.store.FinishIMAPDraftRemovalContext(context.Background(), created.DraftID, 1) + requirements.NoError(err) + } + events, err := runReviewLifecycle(t, adapter, api.CLIRunDraftGetCommand, created.DraftID, "--json") + requirements.NoError(err) + requirements.Len(events, 1) + assertions.Contains(events[0].Data, "boundaries") + assertions.Zero(providerCalls.Load()) + }) + if state == "pending" { + _, err := fixture.store.DB().Exec(fixture.store.Rebind(`UPDATE imap_drafts SET pending_operation = NULL, pending_original_message_id = NULL, pending_original_mailbox = NULL, pending_original_uidvalidity = NULL, pending_original_uid = NULL, pending_raw = NULL, pending_code = NULL WHERE draft_id = ?`), created.DraftID) + requirements.NoError(err) + } + } +} + +func TestDraftLifecycleCleanupOutcomeEvidence(t *testing.T) { + for _, operation := range []string{api.CLIRunDraftEditCommand, api.CLIRunDraftDeleteCommand} { + for _, fault := range []string{"none", "record", "read"} { + t.Run(fmt.Sprintf("%s/fault=%s", operation, fault), func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + barrier := &reviewCmdStoreBarrier{stored: make(chan struct{}), release: make(chan struct{})} + addr, user := startReviewCmdStoreBarrierServer(t, barrier) + fixture := newReviewManagedLifecycleFixtureOnServer(t, addr, func() { + testutil.AppendIMAPRawMessage(t, user, "Drafts", []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Original\r\n\r\noriginal\r\n")) + }) + barrier.armed.Store(true) + args := []string{operation, fixture.draft.DraftID, "--revision", "1", "--json"} + if operation == api.CLIRunDraftEditCommand { + args = append(args, "--body", "candidate") + } + eventsCh := make(chan []api.CLIRunEvent, 1) + errCh := make(chan error, 1) + go func() { + events, err := runReviewLifecycle(t, fixture.adapter, args...) + eventsCh <- events + errCh <- err + }() + <-barrier.stored + reviewStoreFlagsForLifecycle(t, addr, 1, emersionimap.StoreFlagsDel, emersionimap.FlagDeleted) + if fault == "record" { + _, err := fixture.store.DB().Exec(fixture.store.Rebind("UPDATE imap_drafts SET revision = revision + 1 WHERE draft_id = ?"), fixture.draft.DraftID) + requirements.NoError(err) + } + if fault == "read" { + _, err := fixture.store.DB().Exec("ALTER TABLE imap_drafts RENAME TO fault_imap_drafts") + requirements.NoError(err) + } + close(barrier.release) + events, err := <-eventsCh, <-errCh + requirements.Error(err) + requirements.Len(events, 1) + var output draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Equal("pending", output.Status) + assertions.True(output.ManualReconciliation) + requirements.NotNil(output.Observation) + assertions.Equal("store_conflict", output.Observation.Code) + assertions.Equal(uint32(1), output.Observation.UID) + assertions.True(output.Observation.Present) + if fault == "read" { + _, err := fixture.store.DB().Exec("ALTER TABLE fault_imap_drafts RENAME TO imap_drafts") + requirements.NoError(err) + } + latest, loadErr := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(loadErr) + requirements.NotNil(latest.Pending) + assertions.Equal(latest.Pending.Code, output.PendingCode) + assertions.Equal(latest.Revision, output.Revision) + if fault != "none" { + assertions.Equal("local_persistence_failed", err.Error()) + coded, ok := errors.AsType[*api.CLIRunCodedError](err) + requirements.True(ok) + if fault == "record" { + requirements.ErrorIs(coded.Err, store.ErrIMAPDraftRevision) + } else { + requirements.ErrorContains(coded.Err, "imap_drafts") + } + assertions.NotEqual("store_conflict", latest.Pending.Code) + } else { + assertions.Equal("store_conflict", err.Error()) + assertions.Equal("store_conflict", latest.Pending.Code) + } + if operation == api.CLIRunDraftEditCommand { + assertions.Contains(output.CandidateContent, "candidate") + assertions.Equal(uint32(2), output.Receipt.UID) + requirements.NotNil(output.PendingReceipt) + assertions.Equal(uint32(2), output.PendingReceipt.UID) + assertions.Equal(uint32(2), latest.CurrentReceipt.UID) + assertions.Equal(uint32(1), latest.Pending.OriginalReceipt.UID) + } else { + assertions.Nil(latest.DiscardedAt) + } + }) + } + } +} + +func TestDraftLifecycleCleanup(t *testing.T) { + testutil.SkipIfPostgres(t, "finish failure injection uses a SQLite trigger") + for _, operation := range []string{api.CLIRunDraftEditCommand, api.CLIRunDraftDeleteCommand} { + t.Run(operation, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + barrier := &reviewCmdStoreBarrier{stored: make(chan struct{}), release: make(chan struct{})} + addr, user := startReviewCmdStoreBarrierServer(t, barrier) + fixture := newReviewManagedLifecycleFixtureOnServer(t, addr, func() { + testutil.AppendIMAPRawMessage(t, user, "Drafts", []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Original\r\n\r\noriginal\r\n")) + }) + _, err := fixture.store.DB().Exec(` + CREATE TRIGGER fail_draft_finish + AFTER UPDATE OF pending_code ON imap_drafts + WHEN NEW.pending_code = 'removed' + BEGIN + UPDATE imap_drafts SET revision = NEW.revision + 1 WHERE draft_id = NEW.draft_id; + END + `) + requirements.NoError(err) + barrier.armed.Store(true) + eventsCh := make(chan []api.CLIRunEvent, 1) + errCh := make(chan error, 1) + args := []string{operation, fixture.draft.DraftID, "--revision", "1", "--json"} + if operation == api.CLIRunDraftEditCommand { + args = append(args, "--body", "candidate\nline two") + } + go func() { + events, err := runReviewLifecycle(t, fixture.adapter, args...) + eventsCh <- events + errCh <- err + }() + <-barrier.stored + close(barrier.release) + events, err := <-eventsCh, <-errCh + requirements.Error(err) + assertions.Equal("cleanup_local_failed", err.Error()) + requirements.Len(events, 1) + assertions.Equal(cliStreamStderr, events[0].Type) + assertions.Contains(events[0].Data, `"status":"pending"`) + assertions.Contains(events[0].Data, `"manual_reconciliation":true`) + assertions.Contains(events[0].Data, `"pending_operation":"`+strings.TrimPrefix(operation, "draft-")+`"`) + assertions.Contains(events[0].Data, `"pending_code":"removed"`) + assertions.Contains(events[0].Data, `"code":"removed"`) + latest, loadErr := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(loadErr) + requirements.NotNil(latest.Pending) + assertions.Equal(store.IMAPDraftCodeRemoved, latest.Pending.Code) + assertions.Equal(operation, "draft-"+latest.Pending.Operation) + + _, err = fixture.store.DB().Exec("DROP TRIGGER fail_draft_finish") + requirements.NoError(err) + fixture.adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + return nil, errors.New("local completion must not connect to IMAP") + } + retryArgs := []string{operation, latest.DraftID, "--revision", strconv.FormatInt(latest.Revision, 10), "--json"} + if operation == api.CLIRunDraftEditCommand { + _, err = runReviewLifecycle(t, fixture.adapter, append(retryArgs, "--body", "different")...) + requirements.ErrorContains(err, "pending_operation") + retryArgs = append(retryArgs, "--body", "candidate\r\nline two") + } + staleArgs := append([]string(nil), retryArgs...) + staleArgs[3] = "1" + _, err = runReviewLifecycle(t, fixture.adapter, staleArgs...) + requirements.ErrorContains(err, "revision_mismatch") + if operation == api.CLIRunDraftDeleteCommand { + _, err = runReviewLifecycle(t, fixture.adapter, api.CLIRunDraftEditCommand, latest.DraftID, "--revision", strconv.FormatInt(latest.Revision, 10), "--body", "candidate") + } else { + _, err = runReviewLifecycle(t, fixture.adapter, api.CLIRunDraftDeleteCommand, latest.DraftID, "--revision", strconv.FormatInt(latest.Revision, 10)) + } + requirements.ErrorContains(err, "pending_operation") + events, err = runReviewLifecycle(t, fixture.adapter, retryArgs...) + requirements.NoError(err) + requirements.Len(events, 1) + finished, err := fixture.store.GetIMAPDraftContext(t.Context(), latest.DraftID) + requirements.NoError(err) + assertions.Nil(finished.Pending) + if operation == api.CLIRunDraftDeleteCommand { + assertions.NotNil(finished.DiscardedAt) + assertions.Equal(latest.Revision+1, finished.Revision) + } else { + assertions.Equal(latest.Revision, finished.Revision) + assertions.Contains(events[0].Data, `"status":"edited"`) + } + }) + } +} + +func TestDraftLifecycleRemovedOutcomePersistenceFailure(t *testing.T) { + testutil.SkipIfPostgres(t, "removed outcome failure injection uses a SQLite trigger") + for _, operation := range []string{api.CLIRunDraftEditCommand, api.CLIRunDraftDeleteCommand} { + t.Run(operation, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + addr, user := startReviewCmdStoreBarrierServer(t, &reviewCmdStoreBarrier{ + stored: make(chan struct{}), release: make(chan struct{}), + }) + fixture := newReviewManagedLifecycleFixtureOnServer(t, addr, func() { + testutil.AppendIMAPRawMessage(t, user, "Drafts", []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Original\r\n\r\noriginal\r\n")) + }) + _, err := fixture.store.DB().Exec(` + CREATE TRIGGER fail_removed_outcome + BEFORE UPDATE OF pending_code ON imap_drafts + WHEN NEW.pending_code = 'removed' + BEGIN + SELECT RAISE(FAIL, 'injected removed outcome persistence failure'); + END + `) + requirements.NoError(err) + args := []string{operation, fixture.draft.DraftID, "--revision", "1", "--json"} + if operation == api.CLIRunDraftEditCommand { + args = append(args, "--body", "candidate") + } + events, err := runReviewLifecycle(t, fixture.adapter, args...) + requirements.Error(err) + assertions.Equal("local_persistence_failed", err.Error()) + requirements.Len(events, 1) + var output draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Equal("pending", output.Status) + assertions.True(output.ManualReconciliation) + requirements.NotNil(output.Observation) + assertions.Equal(store.IMAPDraftCodeRemoved, output.Observation.Code) + latest, loadErr := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(loadErr) + requirements.NotNil(latest.Pending) + assertions.Equal(operation, "draft-"+latest.Pending.Operation) + assertions.NotEqual(store.IMAPDraftCodeRemoved, latest.Pending.Code) + assertions.Nil(latest.DiscardedAt) + }) + } +} + +func TestDraftLifecycleHumanCleanupPartialThroughHTTP(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + barrier := &reviewCmdStoreBarrier{stored: make(chan struct{}), release: make(chan struct{})} + addr, user := startReviewCmdStoreBarrierServer(t, barrier) + fixture := newReviewManagedLifecycleFixtureOnServer(t, addr, func() { + testutil.AppendIMAPRawMessage(t, user, "Drafts", []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Original\r\n\r\noriginal\r\n")) + }) + barrier.armed.Store(true) + server := httptest.NewServer(api.NewServerWithOptions(api.ServerOptions{ + Config: &config.Config{HomeDir: t.TempDir()}, Store: fixture.adapter, + Logger: slog.New(slog.DiscardHandler), + }).Router()) + t.Cleanup(server.Close) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx + root := &cobra.Command{Use: "msgvault"} + root.SetContext(testCtx) + root.AddCommand(newDraftDeleteCommand()) + silenceUsageInRunE(root) + var stdout, stderr bytes.Buffer + root.SetOut(&stdout) + root.SetErr(&stderr) + root.SetArgs([]string{api.CLIRunDraftDeleteCommand, fixture.draft.DraftID, + "--revision", "1"}) + + errCh := make(chan error, 1) + go func() { errCh <- root.ExecuteContext(testCtx) }() + <-barrier.stored + _, err := fixture.store.DB().Exec(fixture.store.Rebind(` + UPDATE imap_drafts SET revision = 2 WHERE draft_id = ? + `), fixture.draft.DraftID) + requirements.NoError(err) + close(barrier.release) + err = <-errCh + requirements.Error(err) + assertions.Equal("local_persistence_failed", err.Error()) + assertions.Empty(stdout.String()) + assertions.Contains(stderr.String(), "status: pending") + assertions.Contains(stderr.String(), "pending operation: delete") + assertions.Contains(stderr.String(), "provider outcome: removed") + assertions.Contains(stderr.String(), "old provider receipt: Drafts") + assertions.Contains(stderr.String(), "receipt (revision 2): Drafts") + assertions.Contains(stderr.String(), "old draft ID remains blocked at revision 2") + assertions.Contains(stderr.String(), "manual action:") + assertions.NotContains(stderr.String(), "Error: cleanup_local_failed") +} + +func TestDraftLifecycleSyncProjection(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + created := createReviewDraft(t, fixture, adapter, "sync projection") + var analyticsDir string + if !fixture.store.IsPostgreSQL() { + cacheRoot := t.TempDir() + cacheDB := filepath.Join(cacheRoot, "cache.db") + analyticsDir = filepath.Join(cacheRoot, "analytics") + refreshCache := func() error { + if err := os.Remove(cacheDB); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + if err := fixture.store.BackupDatabase(cacheDB); err != nil { + return err + } + _, err := buildCache(cacheDB, analyticsDir, true) + return err + } + requirements.NoError(refreshCache()) + adapter.draftCacheRefresh = func(context.Context, string) error { return refreshCache() } + } + events, err := runReviewLifecycle(t, adapter, api.CLIRunDraftEditCommand, created.DraftID, "--revision", "1", "--body", "projected", "--json") + requirements.NoError(err) + requirements.Len(events, 1) + var output struct { + MessageID int64 `json:"message_id"` + Revision int64 `json:"revision"` + Receipt struct { + UIDValidity uint32 `json:"uidvalidity"` + UID uint32 `json:"uid"` + } `json:"receipt"` + } + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Equal(int64(2), output.Revision) + raw, err := fixture.store.GetMessageRawContext(context.Background(), output.MessageID) + requirements.NoError(err) + assertions.Contains(string(raw), "projected") + draft, err := fixture.store.GetIMAPDraftContext(context.Background(), created.DraftID) + requirements.NoError(err) + assertions.Equal(output.MessageID, draft.CurrentMessageID) + assertions.Equal(output.Receipt.UID, draft.CurrentReceipt.UID) + assertions.Nil(draft.Pending) + if !fixture.store.IsPostgreSQL() { + engine, err := query.NewDuckDBEngine(analyticsDir, "", nil) + requirements.NoError(err) + t.Cleanup(func() { _ = engine.Close() }) + results, err := engine.SearchFast(t.Context(), search.Parse("projected"), query.MessageFilter{}, 100, 0) + requirements.NoError(err) + requirements.Len(results, 1) + assertions.Equal(output.MessageID, results[0].ID) + assertions.Contains(results[0].Snippet, "projected") + rows, err := engine.Aggregate(t.Context(), query.ViewLabels, query.DefaultAggregateOptions()) + requirements.NoError(err) + var draftLabel query.AggregateRow + for _, row := range rows { + if row.Key == "Drafts" { + draftLabel = row + break + } + } + assertions.Equal(int64(1), draftLabel.Count) + } + adminClient, err := imapclient.DialInsecure(fixture.config.Addr(), nil) + requirements.NoError(err) + requirements.NoError(adminClient.Login(testutil.IMAPTestUsername, testutil.IMAPTestPassword).Wait()) + requirements.NoError(adminClient.Create("INBOX", nil).Wait()) + requirements.NoError(adminClient.Close()) + syncClient := imaplib.NewClient(fixture.config, testutil.IMAPTestPassword) + syncOptions := msgsync.DefaultOptions() + syncOptions.SourceType = "imap" + syncOptions.NoResume = true + summary, err := newMessageSyncer(syncClient, fixture.store, syncOptions, + testInvocationWithConfig(testConfigValue())). + WithLogger(slog.New(slog.DiscardHandler)). + FullWithFinalizer(t.Context(), fixture.source, func(summary *gmail.SyncSummary) error { + return saveIMAPFolderStates(t.Context(), fixture.store, fixture.source, syncClient, summary, syncOptions.Limit) + }) + requirements.NoError(err) + requirements.Zero(summary.Errors) + requirements.NoError(syncClient.Close()) + draft, err = fixture.store.GetIMAPDraftContext(context.Background(), created.DraftID) + requirements.NoError(err) + assertions.Equal(output.MessageID, draft.CurrentMessageID) + assertions.Equal(output.Receipt.UIDValidity, draft.CurrentReceipt.UIDValidity) + assertions.Equal(output.Receipt.UID, draft.CurrentReceipt.UID) + assertions.Nil(draft.Pending) + states, err := fixture.store.GetIMAPFolderStates(fixture.source.ID) + requirements.NoError(err) + var draftsState store.IMAPFolderState + for _, state := range states { + if state.Mailbox == "Drafts" { + draftsState = state + break + } + } + assertions.Equal("Drafts", draftsState.Mailbox) + assertions.Equal(output.Receipt.UIDValidity, draftsState.UIDValidity) + assertions.Equal(output.Receipt.UID+1, draftsState.UIDNext) + provider := imaplib.NewClient(fixture.config, testutil.IMAPTestPassword) + observation, err := provider.InspectDraft(t.Context(), imaplib.DraftReceipt{Mailbox: "Drafts", UIDValidity: output.Receipt.UIDValidity, UID: output.Receipt.UID}) + requirements.NoError(err) + assertions.True(observation.Present) +} + +func TestDraftLifecycleCancellation(t *testing.T) { + requirements := require.New(t) + barrier := &reviewCmdStoreBarrier{stored: make(chan struct{}), release: make(chan struct{})} + baseCtx, cancel := context.WithCancel(t.Context()) + ctx := &reviewLateCancelContext{Context: baseCtx, cancel: cancel} + barrier.onAppend = ctx.arm + addr, _ := startReviewCmdStoreBarrierServer(t, barrier) + fixture := newReviewDraftReplyFixtureAtAddr(t, addr) + adapter := fixture.grantedAdapter() + adapter.draftCacheRefresh = func(context.Context, string) error { return nil } + args := []string{"draft-reply", strconv.FormatInt(fixture.parentID, 10), + "--from", testutil.IMAPTestUsername, "--body", "known cancellation", "--json"} + var events []api.CLIRunEvent + err := adapter.runCLIReplyDraft(ctx, api.CLIRunRequest{Args: args}, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + requirements.NoError(err) + requirements.True(ctx.cancelled.Load()) + requirements.Len(events, 1) + var created draftReplyOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &created)) + requirements.Equal(draftReplyStatusCreated, created.Status) + requirements.Equal(int64(1), created.Revision) + draft, err := fixture.store.GetIMAPDraftContext(context.Background(), created.DraftID) + requirements.NoError(err) + requirements.Equal(created.MessageID, draft.CurrentMessageID) + requirements.Equal(created.UID, draft.CurrentReceipt.UID) + requirements.Nil(draft.Pending) + requirements.Equal(int32(1), barrier.appendCalls.Load()) + requirements.Zero(barrier.storeCalls.Load()) +} + +func TestDraftLifecycleCleanupCompletesAfterLateCancellation(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + barrier := &reviewCmdStoreBarrier{stored: make(chan struct{}), release: make(chan struct{})} + baseCtx, cancel := context.WithCancel(t.Context()) + ctx := &reviewLateCancelContext{Context: baseCtx, cancel: cancel} + barrier.onFetch = func(n int32) { + if n == 4 { + ctx.arm() + } + } + addr, user := startReviewCmdStoreBarrierServer(t, barrier) + fixture := newReviewManagedLifecycleFixtureOnServer(t, addr, func() { + testutil.AppendIMAPRawMessage(t, user, "Drafts", []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Original\r\n\r\noriginal\r\n")) + }) + var events []api.CLIRunEvent + err := fixture.adapter.runCLIDraftLifecycle(ctx, api.CLIRunRequest{ + Args: []string{api.CLIRunDraftDeleteCommand, fixture.draft.DraftID, "--revision", "1", "--json"}, + }, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + requirements.NoError(err) + requirements.True(ctx.cancelled.Load()) + requirements.Len(events, 1) + assertions.Contains(events[0].Data, `"status":"deleted"`) + draft, err := fixture.store.GetIMAPDraftContext(context.Background(), fixture.draft.DraftID) + requirements.NoError(err) + requirements.NotNil(draft.DiscardedAt) + requirements.Nil(draft.Pending) + assertions.Equal(int32(4), barrier.fetchCalls.Load()) + assertions.Equal(int32(2), barrier.storeCalls.Load()) + assertions.Equal(int32(1), barrier.expungeCalls.Load()) +} + +func TestDraftLifecycleRejectsEnvAndCwd(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + adapter := &storeAPIAdapter{} + for _, request := range []api.CLIRunRequest{ + {Args: []string{api.CLIRunDraftGetCommand, "draft-test"}, Env: map[string]string{"HOME": "blocked"}}, + {Args: []string{api.CLIRunDraftGetCommand, "draft-test"}, Cwd: "C:\\blocked"}, + } { + err := adapter.runCLIDraftLifecycle(t.Context(), request, nil) + requirements.Error(err) + assertions.Equal("invalid_args", err.Error()) + } +} + +type reviewCreatedDraft struct { + DraftID string `json:"draft_id"` + Revision int64 `json:"revision"` + MessageID int64 `json:"message_id"` +} + +type reviewLateCancelContext struct { + context.Context + + cancel context.CancelFunc + armed atomic.Bool + cancelled atomic.Bool +} + +func (c *reviewLateCancelContext) arm() { + c.armed.Store(true) +} + +func (c *reviewLateCancelContext) Err() error { + if c.armed.Load() && c.cancelled.CompareAndSwap(false, true) { + c.cancel() + } + return c.Context.Err() +} + +func newReviewDraftReplyFixtureAtAddr(t *testing.T, addr string) draftReplyFixture { + t.Helper() + host, portText, err := net.SplitHostPort(addr) + require.NoError(t, err) + port, err := strconv.Atoi(portText) + require.NoError(t, err) + imapConfig := &imaplib.Config{Host: host, Port: port, Username: testutil.IMAPTestUsername} + + st := testutil.NewTestStore(t) + source, err := st.GetOrCreateSource("imap", imapConfig.Identifier()) + require.NoError(t, err) + configJSON, err := imapConfig.ToJSON() + require.NoError(t, err) + configJSON = strings.TrimSuffix(configJSON, "}") + `,"draft_enabled":false,"drafts_mailbox":"AttackerMailbox"}` + require.NoError(t, st.UpdateSourceSyncConfig(source.ID, configJSON)) + require.NoError(t, st.AddAccountIdentity(source.ID, testutil.IMAPTestUsername, "manual")) + conversationID, err := st.EnsureConversation(source.ID, "thread-666", "Question") + require.NoError(t, err) + senderID, err := st.EnsureParticipant("sender@example.com", "Sender", "example.com") + require.NoError(t, err) + ownerID, err := st.EnsureParticipant(testutil.IMAPTestUsername, "", "example.com") + require.NoError(t, err) + parentRaw := []byte("From: Sender \r\n" + + "To: " + testutil.IMAPTestUsername + "\r\n" + + "Subject: Question\r\n" + + "Message-ID: \r\n\r\n" + + "Parent body\r\n") + parentID, err := st.PersistMessage(&store.MessagePersistData{ + Message: &store.Message{ + SourceID: source.ID, SourceMessageID: "INBOX|9", + ConversationID: conversationID, RFC822MessageID: sql.NullString{String: "parent@example.com", Valid: true}, + MessageType: store.MessageTypeEmail, SenderID: sql.NullInt64{Int64: senderID, Valid: true}, + Subject: sql.NullString{String: "Question", Valid: true}, + SentAt: sql.NullTime{Time: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), Valid: true}, + SizeEstimate: int64(len(parentRaw)), + }, + BodyText: sql.NullString{String: "Parent body", Valid: true}, RawMIME: parentRaw, + Recipients: []store.RecipientSet{ + {Type: "from", ParticipantIDs: []int64{senderID}, EmailAddresses: []string{"sender@example.com"}}, + {Type: "to", ParticipantIDs: []int64{ownerID}, EmailAddresses: []string{testutil.IMAPTestUsername}}, + }, + }) + require.NoError(t, err) + return draftReplyFixture{store: st, source: source, parentID: parentID, config: imapConfig, refreshed: new([]string)} +} + +func createReviewDraft(t *testing.T, fixture draftReplyFixture, adapter *storeAPIAdapter, body string) reviewCreatedDraft { + t.Helper() + events, err := fixture.run(t, adapter, "--body", body, "--json") + require.NoError(t, err) + require.Len(t, events, 1) + var created reviewCreatedDraft + require.NoError(t, json.Unmarshal([]byte(events[0].Data), &created)) + require.NotEmpty(t, created.DraftID) + require.Equal(t, int64(1), created.Revision) + return created +} + +func runReviewLifecycle(t *testing.T, adapter *storeAPIAdapter, args ...string) ([]api.CLIRunEvent, error) { + t.Helper() + var events []api.CLIRunEvent + err := adapter.runCLIDraftLifecycle(t.Context(), api.CLIRunRequest{Args: args}, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + return events, err +} + +type reviewManagedLifecycleFixture struct { + store *store.Store + source *store.Source + config *imaplib.Config + draft store.IMAPDraft + adapter *storeAPIAdapter +} + +func newReviewManagedLifecycleFixtureOnServer(t *testing.T, addr string, appendInitial func()) reviewManagedLifecycleFixture { + t.Helper() + imapConfig := reviewIMAPConfig(t, addr) + appendInitial() + reviewStoreFlagsForLifecycle(t, addr, 1, emersionimap.StoreFlagsAdd, emersionimap.FlagDraft) + st := testutil.NewTestStore(t) + source, err := st.GetOrCreateSource("imap", imapConfig.Identifier()) + require.NoError(t, err) + configJSON, err := imapConfig.ToJSON() + require.NoError(t, err) + require.NoError(t, st.UpdateSourceSyncConfig(source.ID, configJSON)) + conversationID, err := st.EnsureConversation(source.ID, "lifecycle-review", "Lifecycle review") + require.NoError(t, err) + receipt := store.IMAPDraftReceipt{SourceID: source.ID, Mailbox: "Drafts", UIDValidity: 1, UID: 1} + raw := []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Original\r\nContent-Type: text/plain\r\n\r\noriginal\r\n") + draft, err := st.PersistIMAPDraftContext(context.Background(), receipt, nil, func([]int64) *store.MessagePersistData { + return &store.MessagePersistData{ + Message: &store.Message{SourceID: source.ID, SourceMessageID: store.IMAPDraftSourceMessageID(receipt), MessageType: store.MessageTypeEmail, ConversationID: conversationID}, + BodyText: sql.NullString{String: "original", Valid: true}, RawMIME: raw, + } + }) + require.NoError(t, err) + adapter := &storeAPIAdapter{ + store: st, + draftPolicy: []config.IMAPDraftSource{{SourceID: source.ID, Enabled: true, Mailbox: "Drafts"}}, + draftClientFactory: func(context.Context, *store.Source) (*imaplib.Client, error) { + return imaplib.NewClient(imapConfig, testutil.IMAPTestPassword), nil + }, + } + return reviewManagedLifecycleFixture{store: st, source: source, config: imapConfig, draft: draft, adapter: adapter} +} + +func newReviewManagedLifecycleFixtureOnServerWithDBPath( + t *testing.T, addr, dbPath string, appendInitial func(), +) reviewManagedLifecycleFixture { + t.Helper() + imapConfig := reviewIMAPConfig(t, addr) + appendInitial() + reviewStoreFlagsForLifecycle(t, addr, 1, emersionimap.StoreFlagsAdd, emersionimap.FlagDraft) + st, err := store.OpenForTest(dbPath) + require.NoError(t, err) + require.NoError(t, st.InitSchema()) + source, err := st.GetOrCreateSource("imap", imapConfig.Identifier()) + require.NoError(t, err) + configJSON, err := imapConfig.ToJSON() + require.NoError(t, err) + require.NoError(t, st.UpdateSourceSyncConfig(source.ID, configJSON)) + conversationID, err := st.EnsureConversation(source.ID, "lifecycle-review", "Lifecycle review") + require.NoError(t, err) + receipt := store.IMAPDraftReceipt{SourceID: source.ID, Mailbox: "Drafts", UIDValidity: 1, UID: 1} + raw := []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Original\r\nContent-Type: text/plain\r\n\r\noriginal\r\n") + draft, err := st.PersistIMAPDraftContext(context.Background(), receipt, nil, func([]int64) *store.MessagePersistData { + return &store.MessagePersistData{ + Message: &store.Message{SourceID: source.ID, SourceMessageID: store.IMAPDraftSourceMessageID(receipt), MessageType: store.MessageTypeEmail, ConversationID: conversationID}, + BodyText: sql.NullString{String: "original", Valid: true}, RawMIME: raw, + } + }) + require.NoError(t, err) + adapter := &storeAPIAdapter{ + store: st, + draftPolicy: []config.IMAPDraftSource{{SourceID: source.ID, Enabled: true, Mailbox: "Drafts"}}, + draftClientFactory: func(context.Context, *store.Source) (*imaplib.Client, error) { + return imaplib.NewClient(imapConfig, testutil.IMAPTestPassword), nil + }, + } + return reviewManagedLifecycleFixture{store: st, source: source, config: imapConfig, draft: draft, adapter: adapter} +} + +type reviewDropAppendSession struct { + imapserver.Session + + conn *imapserver.Conn +} + +type reviewUIDValidityChangeControl struct { + selects atomic.Int32 +} + +type reviewUIDValidityChangeSession struct { + imapserver.Session + + control *reviewUIDValidityChangeControl +} + +func (s *reviewUIDValidityChangeSession) Select( + mailbox string, + options *emersionimap.SelectOptions, +) (*emersionimap.SelectData, error) { + data, err := s.Session.Select(mailbox, options) + if err != nil { + return nil, fmt.Errorf("select mailbox: %w", err) + } + if s.control.selects.Add(1) == 2 { + changed := *data + changed.UIDValidity++ + return &changed, nil + } + return data, nil +} + +type reviewExpungeFailureSession struct { + imapserver.Session +} + +func (s *reviewExpungeFailureSession) Expunge( + *imapserver.ExpungeWriter, + *emersionimap.UIDSet, +) error { + return errors.New("synthetic UID EXPUNGE failure") +} + +func startReviewCustomIMAPServer( + t *testing.T, + wrap func(imapserver.Session) imapserver.Session, +) (string, *imapmemserver.User) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + user := imapmemserver.NewUser(testutil.IMAPTestUsername, testutil.IMAPTestPassword) + require.NoError(t, user.Create("Drafts", nil)) + memServer := imapmemserver.New() + memServer.AddUser(user) + server := imapserver.New(&imapserver.Options{ + Caps: emersionimap.CapSet{emersionimap.CapIMAP4rev1: {}, emersionimap.CapUIDPlus: {}}, + InsecureAuth: true, + NewSession: func(*imapserver.Conn) (imapserver.Session, *imapserver.GreetingData, error) { + return wrap(memServer.NewSession()), nil, nil + }, + }) + go func() { _ = server.Serve(ln) }() + t.Cleanup(func() { _ = server.Close() }) + return ln.Addr().String(), user +} + +func startReviewUIDValidityChangeServer( + t *testing.T, + control *reviewUIDValidityChangeControl, +) (string, *imapmemserver.User) { + t.Helper() + return startReviewCustomIMAPServer(t, func(session imapserver.Session) imapserver.Session { + return &reviewUIDValidityChangeSession{Session: session, control: control} + }) +} + +func startReviewExpungeFailureServer(t *testing.T) (string, *imapmemserver.User) { + t.Helper() + return startReviewCustomIMAPServer(t, func(session imapserver.Session) imapserver.Session { + return &reviewExpungeFailureSession{Session: session} + }) +} + +func (s *reviewDropAppendSession) Append( + mailbox string, + r emersionimap.LiteralReader, + options *emersionimap.AppendOptions, +) (*emersionimap.AppendData, error) { + data, err := s.Session.Append(mailbox, r, options) + if err == nil { + _ = s.conn.NetConn().Close() + } + if err != nil { + return nil, fmt.Errorf("drop append session: %w", err) + } + return data, nil +} + +func startReviewDropAppendServer(t *testing.T) (string, *imapmemserver.User) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + user := imapmemserver.NewUser(testutil.IMAPTestUsername, testutil.IMAPTestPassword) + require.NoError(t, user.Create("Drafts", nil)) + memServer := imapmemserver.New() + memServer.AddUser(user) + server := imapserver.New(&imapserver.Options{ + Caps: emersionimap.CapSet{emersionimap.CapIMAP4rev1: {}, emersionimap.CapUIDPlus: {}}, + InsecureAuth: true, + NewSession: func(conn *imapserver.Conn) (imapserver.Session, *imapserver.GreetingData, error) { + return &reviewDropAppendSession{Session: memServer.NewSession(), conn: conn}, nil, nil + }, + }) + go func() { _ = server.Serve(ln) }() + t.Cleanup(func() { _ = server.Close() }) + return ln.Addr().String(), user +} + +type reviewCmdStoreBarrier struct { + stored chan struct{} + release chan struct{} + armed atomic.Bool + blocked atomic.Bool + appendCalls atomic.Int32 + fetchCalls atomic.Int32 + storeCalls atomic.Int32 + expungeCalls atomic.Int32 + appendErr error + onAppend func() + onFetch func(int32) +} + +type reviewCmdStoreBarrierSession struct { + imapserver.Session + + barrier *reviewCmdStoreBarrier +} + +func (s *reviewCmdStoreBarrierSession) Append( + mailbox string, + r emersionimap.LiteralReader, + options *emersionimap.AppendOptions, +) (*emersionimap.AppendData, error) { + if s.barrier.appendErr != nil { + s.barrier.appendCalls.Add(1) + return nil, s.barrier.appendErr + } + data, err := s.Session.Append(mailbox, r, options) + s.barrier.appendCalls.Add(1) + if s.barrier.onAppend != nil { + s.barrier.onAppend() + } + if err != nil { + return nil, fmt.Errorf("append barrier session: %w", err) + } + return data, nil +} + +func (s *reviewCmdStoreBarrierSession) Fetch( + w *imapserver.FetchWriter, + numSet emersionimap.NumSet, + options *emersionimap.FetchOptions, +) error { + err := s.Session.Fetch(w, numSet, options) + n := s.barrier.fetchCalls.Add(1) + if s.barrier.onFetch != nil { + s.barrier.onFetch(n) + } + if err != nil { + return fmt.Errorf("fetch barrier session: %w", err) + } + return nil +} + +func (s *reviewCmdStoreBarrierSession) Store( + w *imapserver.FetchWriter, + numSet emersionimap.NumSet, + flags *emersionimap.StoreFlags, + options *emersionimap.StoreOptions, +) error { + err := s.Session.Store(w, numSet, flags, options) + s.barrier.storeCalls.Add(1) + if s.barrier.armed.Load() && s.barrier.blocked.CompareAndSwap(false, true) { + close(s.barrier.stored) + <-s.barrier.release + } + if err != nil { + return fmt.Errorf("store barrier session: %w", err) + } + return nil +} + +func (s *reviewCmdStoreBarrierSession) Expunge( + w *imapserver.ExpungeWriter, + uids *emersionimap.UIDSet, +) error { + err := s.Session.Expunge(w, uids) + s.barrier.expungeCalls.Add(1) + if err != nil { + return fmt.Errorf("expunge barrier session: %w", err) + } + return nil +} + +func startReviewCmdStoreBarrierServer(t *testing.T, barrier *reviewCmdStoreBarrier) (string, *imapmemserver.User) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + user := imapmemserver.NewUser(testutil.IMAPTestUsername, testutil.IMAPTestPassword) + require.NoError(t, user.Create("Drafts", nil)) + memServer := imapmemserver.New() + memServer.AddUser(user) + server := imapserver.New(&imapserver.Options{ + Caps: emersionimap.CapSet{emersionimap.CapIMAP4rev1: {}, emersionimap.CapUIDPlus: {}}, + InsecureAuth: true, + NewSession: func(conn *imapserver.Conn) (imapserver.Session, *imapserver.GreetingData, error) { + return &reviewCmdStoreBarrierSession{Session: memServer.NewSession(), barrier: barrier}, nil, nil + }, + }) + go func() { _ = server.Serve(ln) }() + t.Cleanup(func() { _ = server.Close() }) + return ln.Addr().String(), user +} + +func reviewIMAPConfig(t *testing.T, addr string) *imaplib.Config { + t.Helper() + host, portText, err := net.SplitHostPort(addr) + require.NoError(t, err) + port, err := strconv.Atoi(portText) + require.NoError(t, err) + return &imaplib.Config{Host: host, Port: port, Username: testutil.IMAPTestUsername} +} + +func reviewStoreFlagsForLifecycle(t *testing.T, addr string, uid uint32, op emersionimap.StoreFlagsOp, flag emersionimap.Flag) { + t.Helper() + client, err := imapclient.DialInsecure(addr, nil) + require.NoError(t, err) + t.Cleanup(func() { _ = client.Close() }) + require.NoError(t, client.Login(testutil.IMAPTestUsername, testutil.IMAPTestPassword).Wait()) + _, err = client.Select("Drafts", nil).Wait() + require.NoError(t, err) + var uids emersionimap.UIDSet + uids.AddNum(emersionimap.UID(uid)) + require.NoError(t, client.Store(uids, &emersionimap.StoreFlags{Op: op, Flags: []emersionimap.Flag{flag}}, nil).Close()) +} + +func reviewDraftMailboxCount(t *testing.T, addr string) uint32 { + t.Helper() + client, err := imapclient.DialInsecure(addr, nil) + require.NoError(t, err) + t.Cleanup(func() { _ = client.Close() }) + require.NoError(t, client.Login(testutil.IMAPTestUsername, testutil.IMAPTestPassword).Wait()) + data, err := client.Status("Drafts", &emersionimap.StatusOptions{NumMessages: true}).Wait() + require.NoError(t, err) + require.NotNil(t, data.NumMessages) + return *data.NumMessages +} diff --git a/cmd/msgvault/cmd/serve_draft_lifecycle_sync_test.go b/cmd/msgvault/cmd/serve_draft_lifecycle_sync_test.go new file mode 100644 index 000000000..84d052ebc --- /dev/null +++ b/cmd/msgvault/cmd/serve_draft_lifecycle_sync_test.go @@ -0,0 +1,186 @@ +package cmd + +import ( + "context" + "database/sql" + "encoding/json" + "log/slog" + "strings" + "sync/atomic" + "testing" + + imapv2 "github.com/emersion/go-imap/v2" + "github.com/emersion/go-imap/v2/imapclient" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/agentgrant" + "go.kenn.io/msgvault/internal/api" + imaplib "go.kenn.io/msgvault/internal/imap" + "go.kenn.io/msgvault/internal/store" + msgsync "go.kenn.io/msgvault/internal/sync" + "go.kenn.io/msgvault/internal/testutil" +) + +// Schedule a real sync at the next query after the first MIME read completes. +// Only the timing is controlled; storage, sync, and authorization run normally. +type draftContentReadHandler struct { + slog.Handler + + afterRead func() + read atomic.Bool +} + +func (h *draftContentReadHandler) Enabled(context.Context, slog.Level) bool { + return true +} + +func (h *draftContentReadHandler) Handle(_ context.Context, record slog.Record) error { + var statement, requestID string + record.Attrs(func(attr slog.Attr) bool { + switch attr.Key { + case "stmt": + statement = attr.Value.String() + case "request_id": + requestID = attr.Value.String() + } + return true + }) + if requestID == "draft-sync-race" { + if h.read.Load() { + h.afterRead() + } + if strings.Contains(statement, "FROM message_raw") { + h.read.Store(true) + } + } + return nil +} + +func TestDelegatedDraftLifecycleConcurrentIMAPSync(t *testing.T) { + for _, tc := range []struct { + name, operation, sender string + allowed, pending bool + }{ + {"get excluded sender", api.CLIRunDraftGetCommand, "other@example.test", false, false}, + {"get excluded candidate sender", api.CLIRunDraftGetCommand, "other@example.test", false, true}, + {"get allowed candidate sender", api.CLIRunDraftGetCommand, "alice@example.com", true, true}, + {"get allowed sender", api.CLIRunDraftGetCommand, "alice@example.com", true, false}, + {"edit excluded sender", api.CLIRunDraftEditCommand, "other@example.test", false, false}, + {"delete excluded sender", api.CLIRunDraftDeleteCommand, "other@example.test", false, false}, + } { + t.Run(tc.name, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + addr, user := testutil.StartIMAPMemServerWithSpecialUse(t, + map[string]int{"INBOX": 0, "Drafts": 0, "Sent": 0}, + map[string][]imapv2.MailboxAttr{"Drafts": {imapv2.MailboxAttrDrafts}, "Sent": {imapv2.MailboxAttrSent}}, + ) + const original = "From: alice@example.com\r\nTo: bob@example.com\r\nMessage-ID: \r\nSubject: Draft\r\n\r\noriginal body" + fixture := newReviewManagedLifecycleFixtureOnServer(t, addr, func() { + testutil.AppendIMAPRawMessage(t, user, "Drafts", []byte(original)) + }) + // Match the fixture to this server's mailbox epoch and original MIME. + control, err := imapclient.DialInsecure(addr, nil) + require.NoError(err) + t.Cleanup(func() { _ = control.Close() }) + require.NoError(control.Login(testutil.IMAPTestUsername, testutil.IMAPTestPassword).Wait()) + selected, err := control.Select("Drafts", nil).Wait() + require.NoError(err) + _, err = fixture.store.DB().Exec(fixture.store.Rebind("UPDATE imap_drafts SET current_uidvalidity = ? WHERE draft_id = ?"), selected.UIDValidity, fixture.draft.DraftID) + require.NoError(err) + _, err = fixture.store.DB().Exec(fixture.store.Rebind("UPDATE imap_message_memberships SET uidvalidity = ? WHERE message_id = ?"), selected.UIDValidity, fixture.draft.CurrentMessageID) + require.NoError(err) + message, err := fixture.store.GetMessage(fixture.draft.CurrentMessageID) + require.NoError(err) + archivedID, err := fixture.store.PersistMessage(&store.MessagePersistData{ + Message: &store.Message{ + SourceID: fixture.source.ID, SourceMessageID: "Drafts|1", ConversationID: message.ConversationID, + MessageType: store.MessageTypeEmail, RFC822MessageID: sql.NullString{String: "", Valid: true}, + }, + BodyText: sql.NullString{String: "original body", Valid: true}, RawMIME: []byte(original), + }) + require.NoError(err) + require.Equal(fixture.draft.CurrentMessageID, archivedID) + testutil.AppendIMAPRawMessage(t, user, "Sent", []byte(strings.ReplaceAll(strings.ReplaceAll(original, + "alice@example.com", tc.sender), "original body", "synced body"))) + + client := imaplib.NewClient(fixture.config, testutil.IMAPTestPassword) + t.Cleanup(func() { _ = client.Close() }) + opts := msgsync.DefaultOptions() + opts.SourceType = "imap" + syncer := msgsync.New(client, fixture.store, opts) + var synced atomic.Bool + if tc.pending { + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationEdit, []byte(original)) + require.NoError(err) + _, err = syncer.Full(t.Context(), fixture.source.Identifier) + require.NoError(err) + synced.Store(true) + } + previous := slog.Default() + slog.SetDefault(slog.New(&draftContentReadHandler{ + Handler: slog.DiscardHandler, + afterRead: func() { + if synced.CompareAndSwap(false, true) { + _, syncErr := syncer.Full(t.Context(), fixture.source.Identifier) + require.NoError(syncErr) + } + }, + })) + t.Cleanup(func() { slog.SetDefault(previous) }) + + grant := &agentgrant.Grant{ + ID: "sender-scoped", + Permissions: []agentgrant.Permission{agentgrant.PermissionDraftCreate, agentgrant.PermissionDraftEdit, agentgrant.PermissionDraftDelete}, + Sources: []agentgrant.SourceRef{{Type: "imap", Identifier: fixture.source.Identifier, SenderKeys: []string{"alice@example.com"}}}, + } + if tc.pending { + grant.Sources[0].SenderKeys = []string{tc.sender} + } + args := []string{tc.operation, fixture.draft.DraftID, "--json"} + if tc.operation != api.CLIRunDraftGetCommand { + args = append(args, "--revision", "1") + } + if tc.operation == api.CLIRunDraftEditCommand { + args = append(args, "--body", "delegated body") + } + providerCalls := 0 + factory := fixture.adapter.draftClientFactory + fixture.adapter.draftClientFactory = func(ctx context.Context, source *store.Source) (*imaplib.Client, error) { + providerCalls++ + return factory(ctx, source) + } + var events []api.CLIRunEvent + err = fixture.adapter.runCLIDraftLifecycle(store.WithRequestID(t.Context(), "draft-sync-race"), api.CLIRunRequest{Args: args, Grant: grant}, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + assert.True(synced.Load()) + if tc.allowed { + require.NoError(err) + require.Len(events, 1) + var output draftLifecycleOutput + require.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assert.Equal("synced body", output.Content) + assert.Contains(output.RawMIME, "From: "+tc.sender+"\r\n") + assert.Contains(output.RawMIME, "synced body") + if tc.pending { + assert.Equal(original, output.CandidateContent) + } + } else { + require.Error(err) + assert.Equal("not_permitted", err.Error()) + assert.Empty(events) + } + assert.Zero(providerCalls) + latest, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + require.NoError(err) + assert.Equal(int64(1), latest.Revision) + archived, err := fixture.store.GetMessage(latest.CurrentMessageID) + require.NoError(err) + assert.Equal(tc.sender, archived.FromEmail) + assert.Equal("synced body", archived.BodyText) + assert.Equal(fixture.draft.CurrentMessageID, latest.CurrentMessageID) + }) + } +} diff --git a/cmd/msgvault/cmd/serve_draft_lifecycle_test.go b/cmd/msgvault/cmd/serve_draft_lifecycle_test.go new file mode 100644 index 000000000..63c886a4d --- /dev/null +++ b/cmd/msgvault/cmd/serve_draft_lifecycle_test.go @@ -0,0 +1,225 @@ +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "log/slog" + "net/http/httptest" + "strconv" + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/agentgrant" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/config" + imaplib "go.kenn.io/msgvault/internal/imap" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil" +) + +func TestDraftLifecycleEndToEnd(t *testing.T) { + requirements := require.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + server := httptest.NewServer(api.NewServerWithOptions(api.ServerOptions{Config: &config.Config{HomeDir: t.TempDir()}, Store: adapter, Logger: slog.New(slog.DiscardHandler)}).Router()) + t.Cleanup(server.Close) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx + run := func(args ...string) (string, error) { + root := &cobra.Command{Use: "msgvault"} + root.SetContext(testCtx) + root.AddCommand(newDraftReplyCommand(), newDraftGetCommand(), newDraftEditCommand(), newDraftDeleteCommand()) + silenceUsageInRunE(root) + var stdout, stderr bytes.Buffer + root.SetOut(&stdout) + root.SetErr(&stderr) + root.SetArgs(args) + err := root.ExecuteContext(testCtx) + requirements.Empty(stderr.String()) + return stdout.String(), err + } + createdJSON, err := run("draft-reply", strconv.FormatInt(fixture.parentID, 10), "--from", testutil.IMAPTestUsername, "--body", "initial body", "--json") + requirements.NoError(err) + var created struct { + DraftID string `json:"draft_id"` + Revision int64 `json:"revision"` + } + requirements.NoError(json.Unmarshal([]byte(createdJSON), &created)) + requirements.NotEmpty(created.DraftID) + requirements.Equal(int64(1), created.Revision) + + getEvent, err := run(api.CLIRunDraftGetCommand, created.DraftID, "--json") + requirements.NoError(err) + requirements.Contains(getEvent, "initial body") + humanGet, err := run(api.CLIRunDraftGetCommand, created.DraftID) + requirements.NoError(err) + requirements.Contains(humanGet, "content:\ninitial body") + requirements.Contains(humanGet, "receipt (revision 1): Drafts") + + editEvent, err := run(api.CLIRunDraftEditCommand, created.DraftID, "--revision", "1", "--body", "edited body", "--json") + requirements.NoError(err) + requirements.Contains(editEvent, "\"revision\":2") + + deleteEvent, err := run(api.CLIRunDraftDeleteCommand, created.DraftID, "--revision", "2", "--json") + requirements.NoError(err) + requirements.Contains(deleteEvent, "\"lifecycle\":\"discarded\"") + _, err = run(api.CLIRunDraftGetCommand, created.DraftID, "--json") + requirements.NoError(err) +} + +func TestDraftLifecycleDelegatedIMAPGetEditDelete(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + var created draftReplyOutput + err := adapter.runCLIReplyDraft(t.Context(), api.CLIRunRequest{Args: []string{ + api.CLIRunDraftReplyCommand, strconv.FormatInt(fixture.parentID, 10), + "--from", testutil.IMAPTestUsername, "--body", "initial body", "--json", + }}, func(event api.CLIRunEvent) error { + return json.Unmarshal([]byte(event.Data), &created) + }) + requirements.NoError(err) + + providerCalls := 0 + clientFactory := adapter.draftClientFactory + adapter.draftClientFactory = func(ctx context.Context, source *store.Source) (*imaplib.Client, error) { + providerCalls++ + return clientFactory(ctx, source) + } + _, senderKey, err := parseDraftSender(testutil.IMAPTestUsername) + requirements.NoError(err) + sourceRef := agentgrant.SourceRef{ID: fixture.source.ID, Type: fixture.source.SourceType, Identifier: fixture.source.Identifier} + grantFor := func(ref agentgrant.SourceRef, permissions ...agentgrant.Permission) *agentgrant.Grant { + return &agentgrant.Grant{ID: "imap-lifecycle-grant", Permissions: permissions, Sources: []agentgrant.SourceRef{ref}} + } + run := func(grant *agentgrant.Grant, args ...string) (string, error) { + var out strings.Builder + err := adapter.runCLIDraftLifecycle(t.Context(), api.CLIRunRequest{Args: args, Grant: grant}, func(event api.CLIRunEvent) error { + out.WriteString(event.Data) + return nil + }) + return out.String(), err + } + revision := strconv.FormatInt(created.Revision, 10) + + otherSource := sourceRef + otherSource.Identifier = "other@example.test" + createWithoutSender := grantFor(sourceRef, agentgrant.PermissionDraftCreate) + otherSender := sourceRef + otherSender.SenderKeys = []string{"other@example.test"} + for _, tc := range []struct { + name string + grant *agentgrant.Grant + args []string + }{ + {"edit on another source", grantFor(otherSource, agentgrant.PermissionDraftEdit), []string{api.CLIRunDraftEditCommand, created.DraftID, "--revision", revision, "--body", "delegated"}}, + {"delete without draft.delete", grantFor(sourceRef, agentgrant.PermissionDraftEdit), []string{api.CLIRunDraftDeleteCommand, created.DraftID, "--revision", revision}}, + {"get with draft.create and no sender", createWithoutSender, []string{api.CLIRunDraftGetCommand, created.DraftID, "--json"}}, + {"get with edit and another sender", grantFor(otherSender, agentgrant.PermissionDraftEdit), []string{api.CLIRunDraftGetCommand, created.DraftID, "--json"}}, + {"get with delete and another sender", grantFor(otherSender, agentgrant.PermissionDraftDelete), []string{api.CLIRunDraftGetCommand, created.DraftID, "--json"}}, + {"edit with another sender", grantFor(otherSender, agentgrant.PermissionDraftCreate, agentgrant.PermissionDraftEdit), []string{api.CLIRunDraftEditCommand, created.DraftID, "--revision", "99", "--body", "delegated"}}, + {"delete with another sender", grantFor(otherSender, agentgrant.PermissionDraftDelete), []string{api.CLIRunDraftDeleteCommand, created.DraftID, "--revision", "99"}}, + } { + t.Run(tc.name, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + out, err := run(tc.grant, tc.args...) + requirements.Error(err) + assertions.Equal("not_permitted", err.Error()) + assertions.Empty(out) + assertions.Equal(0, providerCalls) + }) + } + + senderRef := sourceRef + senderRef.SenderKeys = []string{senderKey} + for _, permission := range []agentgrant.Permission{agentgrant.PermissionDraftCreate, agentgrant.PermissionDraftEdit, agentgrant.PermissionDraftDelete} { + got, err := run(grantFor(senderRef, permission), api.CLIRunDraftGetCommand, created.DraftID, "--json") + requirements.NoError(err) + var output draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(got), &output)) + assertions.Equal(created.Revision, output.Revision) + if permission == agentgrant.PermissionDraftDelete { + assertions.Empty(output.Content) + assertions.Empty(output.RawMIME) + } else { + assertions.Contains(output.Content, "initial body") + } + } + + _, err = fixture.store.ClaimIMAPDraftContext(t.Context(), created.DraftID, created.Revision, store.IMAPDraftOperationEdit, []byte("pending candidate")) + requirements.NoError(err) + for _, asJSON := range []bool{false, true} { + args := []string{api.CLIRunDraftGetCommand, created.DraftID} + if asJSON { + args = append(args, "--json") + } + got, err := run(grantFor(senderRef, agentgrant.PermissionDraftDelete), args...) + requirements.NoError(err) + assertions.NotContains(got, "initial body") + assertions.NotContains(got, "pending candidate") + assertions.NotContains(got, `"raw_mime"`) + assertions.Contains(got, "edit") + } + _, err = fixture.store.AbortIMAPDraftContext(t.Context(), created.DraftID, created.Revision, "cancelled") + requirements.NoError(err) + + edited, err := run(grantFor(senderRef, agentgrant.PermissionDraftEdit), api.CLIRunDraftEditCommand, created.DraftID, "--revision", revision, "--body", "delegated body", "--json") + requirements.NoError(err) + assertions.Contains(edited, "\"revision\":2") + + deleted, err := run(grantFor(senderRef, agentgrant.PermissionDraftDelete), api.CLIRunDraftDeleteCommand, created.DraftID, "--revision", "2", "--json") + requirements.NoError(err) + assertions.Contains(deleted, "\"lifecycle\":\"discarded\"") + assertions.NotContains(deleted, "delegated body") + assertions.NotContains(deleted, `"raw_mime"`) + assertions.Equal(2, providerCalls) +} + +func TestDraftLifecycleReplacementIndexesCc(t *testing.T) { + assertions := assert.New(t) + requirements := require.New(t) + fixture := newDraftReplyFixture(t) + parentRaw, err := fixture.store.GetMessageRaw(fixture.parentID) + requirements.NoError(err) + parentRaw = bytes.Replace(parentRaw, []byte("Subject: Question\r\n"), []byte("Cc: copy@example.test\r\nSubject: Question\r\n"), 1) + requirements.NoError(fixture.store.UpsertMessageRaw(fixture.parentID, parentRaw)) + adapter := fixture.grantedAdapter() + + var created draftReplyOutput + err = adapter.runCLIReplyDraft(t.Context(), api.CLIRunRequest{Args: []string{ + "draft-reply", strconv.FormatInt(fixture.parentID, 10), "--from", testutil.IMAPTestUsername, + "--all", "--body", "initial body", "--json", + }}, func(event api.CLIRunEvent) error { + return json.Unmarshal([]byte(event.Data), &created) + }) + requirements.NoError(err) + requirements.Equal(draftReplyStatusCreated, created.Status) + requirements.Equal(int64(1), created.Revision) + + var events []api.CLIRunEvent + err = adapter.runCLIDraftLifecycle(t.Context(), api.CLIRunRequest{Args: []string{ + api.CLIRunDraftEditCommand, created.DraftID, "--revision", "1", "--body", "edited body", "--json", + }}, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + requirements.NoError(err) + requirements.Len(events, 1) + + draft, err := fixture.store.GetIMAPDraftContext(t.Context(), created.DraftID) + requirements.NoError(err) + requirements.Equal(int64(2), draft.Revision) + requirements.NotEqual(created.MessageID, draft.CurrentMessageID) + // The backends tokenize punctuation in full email queries differently. + matches, total, err := fixture.store.SearchMessages("copy", 0, 10) + requirements.NoError(err) + requirements.Equal(int64(1), total) + requirements.Len(matches, 1) + assertions.Equal(draft.CurrentMessageID, matches[0].ID) +} diff --git a/cmd/msgvault/cmd/serve_draft_recover_test.go b/cmd/msgvault/cmd/serve_draft_recover_test.go new file mode 100644 index 000000000..59b5aab87 --- /dev/null +++ b/cmd/msgvault/cmd/serve_draft_recover_test.go @@ -0,0 +1,1147 @@ +package cmd + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "log/slog" + "net/http" + "net/http/httptest" + "strconv" + "testing" + + emersionimap "github.com/emersion/go-imap/v2" + "github.com/emersion/go-imap/v2/imapclient" + "github.com/emersion/go-imap/v2/imapserver/imapmemserver" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/agentgrant" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/config" + imaplib "go.kenn.io/msgvault/internal/imap" + msgmime "go.kenn.io/msgvault/internal/mime" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil" +) + +const draftRecoverCommand = "draft-recover" + +func newDraftRecoveryFixture(t *testing.T) (reviewManagedLifecycleFixture, *imapmemserver.User) { + t.Helper() + addr, user := testutil.StartIMAPMemServerForDrafts(t, testutil.IMAPDraftServerOptions{ + MessagesPerMailbox: map[string]int{"Drafts": 0}, + Caps: emersionimap.CapSet{emersionimap.CapIMAP4rev1: {}, emersionimap.CapUIDPlus: {}}, + }) + fixture := newReviewManagedLifecycleFixtureOnServer(t, addr, func() { + testutil.AppendIMAPRawMessage(t, user, "Drafts", []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Original\r\nContent-Type: text/plain\r\n\r\noriginal\r\n")) + }) + return fixture, user +} + +func removeRecoveryOriginal(t *testing.T, fixture reviewManagedLifecycleFixture) { + t.Helper() + client := imaplib.NewClient(fixture.config, testutil.IMAPTestPassword) + observation, err := client.RemoveDraft(t.Context(), recoveryTestReceipt(fixture.draft.CurrentReceipt)) + require.NoError(t, err) + require.True(t, observation.Complete) + require.NoError(t, client.Close()) +} + +func recoveryTestReceipt(receipt store.IMAPDraftReceipt) imaplib.DraftReceipt { + return imaplib.DraftReceipt{Mailbox: receipt.Mailbox, UIDValidity: receipt.UIDValidity, UID: receipt.UID} +} + +func recoveryUIDNext(t *testing.T, fixture reviewManagedLifecycleFixture) uint32 { + t.Helper() + client, err := imapclient.DialInsecure(fixture.config.Addr(), nil) + require.NoError(t, err) + t.Cleanup(func() { _ = client.Close() }) + require.NoError(t, client.Login(testutil.IMAPTestUsername, testutil.IMAPTestPassword).Wait()) + status, err := client.Status("Drafts", &emersionimap.StatusOptions{UIDNext: true}).Wait() + require.NoError(t, err) + return uint32(status.UIDNext) +} + +func newDraftRecoveryHTTPServer(t *testing.T, fixture reviewManagedLifecycleFixture) *httptest.Server { + t.Helper() + return httptest.NewServer(api.NewServerWithOptions(api.ServerOptions{ + Config: &config.Config{ + HomeDir: t.TempDir(), + Server: config.ServerConfig{APIKey: "owner-key", AgentAccess: true}, + }, + Store: fixture.adapter, + Logger: slog.New(slog.DiscardHandler), + }).Router()) +} + +func issueDraftRecoveryToken(t *testing.T, server *httptest.Server, sourceID int64, permission string) string { + t.Helper() + body, err := json.Marshal(map[string]any{ + "label": "recovery-agent", + "permissions": []string{permission}, + "source_ids": []int64{sourceID}, + }) + require.NoError(t, err) + req, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/agent-tokens", bytes.NewReader(body)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Api-Key", "owner-key") + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer func() { _ = resp.Body.Close() }() + require.Equal(t, http.StatusCreated, resp.StatusCode) + var issued struct { + Secret string `json:"secret"` + } + require.NoError(t, json.NewDecoder(resp.Body).Decode(&issued)) + return issued.Secret +} + +func runDraftRecoveryHTTP(t *testing.T, server *httptest.Server, secret, draftID string, revision int64) []api.CLIRunEvent { + t.Helper() + body, err := json.Marshal(api.CLIRunRequest{Args: []string{ + draftRecoverCommand, draftID, "--revision", strconv.FormatInt(revision, 10), "--json", + }}) + require.NoError(t, err) + req, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/cli/run", bytes.NewReader(body)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Msgvault-Agent-Token", secret) + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer func() { _ = resp.Body.Close() }() + require.Equal(t, http.StatusOK, resp.StatusCode) + var events []api.CLIRunEvent + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + var event api.CLIRunEvent + require.NoError(t, json.Unmarshal(scanner.Bytes(), &event)) + events = append(events, event) + } + require.NoError(t, scanner.Err()) + return events +} + +func TestDraftRecoverReconcilesManualCleanup(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + removeRecoveryOriginal(t, fixture) + + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.NoError(err) + requirements.Len(events, 1) + var output draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Equal("deleted", output.Status) + assertions.Equal(int64(2), output.Revision) + assertions.Equal("discarded", output.Lifecycle) + assertions.Equal(uint32(0), reviewDraftMailboxCount(t, fixture.config.Addr())) + latest, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + requirements.Nil(latest.Pending) + requirements.NotNil(latest.DiscardedAt) +} + +func TestDraftRecoverThroughHTTP(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + server := httptest.NewServer(api.NewServerWithOptions(api.ServerOptions{ + Config: &config.Config{ + HomeDir: t.TempDir(), + Server: config.ServerConfig{APIKey: "owner-key", AgentAccess: true}, + }, + Store: fixture.adapter, + Logger: slog.New(slog.DiscardHandler), + }).Router()) + t.Cleanup(server.Close) + + issue := func(permission string) string { + body, err := json.Marshal(map[string]any{ + "label": "recovery-agent", + "permissions": []string{permission}, + "source_ids": []int64{fixture.source.ID}, + }) + requirements.NoError(err) + req, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/agent-tokens", bytes.NewReader(body)) + requirements.NoError(err) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Api-Key", "owner-key") + resp, err := http.DefaultClient.Do(req) + requirements.NoError(err) + defer func() { _ = resp.Body.Close() }() + requirements.Equal(http.StatusCreated, resp.StatusCode) + var issued struct { + Secret string `json:"secret"` + } + requirements.NoError(json.NewDecoder(resp.Body).Decode(&issued)) + return issued.Secret + } + + run := func(secret, draftID string, revision int64) []api.CLIRunEvent { + body, err := json.Marshal(api.CLIRunRequest{Args: []string{draftRecoverCommand, draftID, "--revision", strconv.FormatInt(revision, 10), "--json"}}) + requirements.NoError(err) + req, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/cli/run", bytes.NewReader(body)) + requirements.NoError(err) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Msgvault-Agent-Token", secret) + resp, err := http.DefaultClient.Do(req) + requirements.NoError(err) + defer func() { _ = resp.Body.Close() }() + requirements.Equal(http.StatusOK, resp.StatusCode) + var events []api.CLIRunEvent + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + var event api.CLIRunEvent + requirements.NoError(json.Unmarshal(scanner.Bytes(), &event)) + events = append(events, event) + } + requirements.NoError(scanner.Err()) + return events + } + + allowed := run(issue("draft.edit"), fixture.draft.DraftID, 1) + requirements.Len(allowed, 2) + assertions.Equal(cliStreamStdout, allowed[0].Type) + assertions.Contains(allowed[0].Data, `"status":"active"`) + var delegatedOutput draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(allowed[0].Data), &delegatedOutput)) + assertions.Empty(delegatedOutput.Content) + assertions.Empty(delegatedOutput.RawMIME) + assertions.Empty(delegatedOutput.CandidateContent) + assertions.Equal("complete", allowed[1].Type) + + denied := run(issue("draft.delete"), fixture.draft.DraftID, 1) + requirements.Len(denied, 1) + assertions.Empty(denied[0].Data) + assertions.Equal("not_permitted", denied[0].Error) + + otherSource, err := fixture.store.GetOrCreateSource("imap", "imap://other@example.com:143") + requirements.NoError(err) + conversationID, err := fixture.store.EnsureConversation(otherSource.ID, "other-draft", "Other draft") + requirements.NoError(err) + otherReceipt := store.IMAPDraftReceipt{SourceID: otherSource.ID, Mailbox: "Drafts", UIDValidity: 1, UID: 1} + otherDraft, err := fixture.store.PersistIMAPDraftContext(t.Context(), otherReceipt, nil, func([]int64) *store.MessagePersistData { + return &store.MessagePersistData{ + Message: &store.Message{ + SourceID: otherSource.ID, SourceMessageID: store.IMAPDraftSourceMessageID(otherReceipt), + MessageType: store.MessageTypeEmail, ConversationID: conversationID, + }, + RawMIME: []byte("From: alice@example.com\r\n\r\nother\r\n"), + } + }) + requirements.NoError(err) + for _, tc := range []struct { + name string + draftID string + revision int64 + }{ + {name: "missing", draftID: "missing-draft", revision: 1}, + {name: "out of scope current", draftID: otherDraft.DraftID, revision: 1}, + {name: "out of scope stale", draftID: otherDraft.DraftID, revision: 2}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Helper() + events := run(issue("draft.edit"), tc.draftID, tc.revision) + requirements.Len(events, 1) + assertions.Empty(events[0].Data) + assertions.Equal("not_permitted", events[0].Error) + }) + } +} + +func TestDraftRecoverDelegatedUnknownReplacementRedactsCandidate(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + server := newDraftRecoveryHTTPServer(t, fixture) + defer server.Close() + secret := issueDraftRecoveryToken(t, server, fixture.source.ID, string(agentgrant.PermissionDraftEdit)) + candidate := []byte("From: alice@example.com\r\nTo: bob@example.com\r\nContent-Type: text/plain\r\n\r\ncandidate\r\n") + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationEdit, candidate) + requirements.NoError(err) + + events := runDraftRecoveryHTTP(t, server, secret, fixture.draft.DraftID, 1) + requirements.Len(events, 2) + assertions.Equal(cliStreamStderr, events[0].Type) + var output draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Empty(output.PendingCode) + assertions.Equal("unknown_replacement", output.Status) + assertions.Equal("unknown_replacement", output.RefusalCode) + assertions.Empty(output.Content) + assertions.Empty(output.RawMIME) + assertions.Empty(output.CandidateContent) + assertions.Equal("unknown_replacement", events[1].Error) +} + +func TestDraftRecoverPolicyPrecedesSettledOutputThroughHTTP(t *testing.T) { + for _, tc := range []struct { + name string + permission string + prepare func(*testing.T, reviewManagedLifecycleFixture) (string, int64) + }{ + { + name: "active", + permission: string(agentgrant.PermissionDraftEdit), + prepare: func(_ *testing.T, fixture reviewManagedLifecycleFixture) (string, int64) { + return fixture.draft.DraftID, fixture.draft.Revision + }, + }, + { + name: "discarded", + permission: string(agentgrant.PermissionDraftDelete), + prepare: func(t *testing.T, fixture reviewManagedLifecycleFixture) (string, int64) { + t.Helper() + requirements := require.New(t) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + removeRecoveryOriginal(t, fixture) + _, err = runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.NoError(err) + latest, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + return latest.DraftID, latest.Revision + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + draftID, revision := tc.prepare(t, fixture) + providerCalls := 0 + fixture.adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + providerCalls++ + return nil, errors.New("policy refusal must not connect") + } + fixture.adapter.draftPolicy = nil + execution, err := fixture.store.AcquireSyncExecutionContext(t.Context(), fixture.source.ID) + requirements.NoError(err) + t.Cleanup(func() { _ = execution.Release() }) + server := newDraftRecoveryHTTPServer(t, fixture) + t.Cleanup(server.Close) + secret := issueDraftRecoveryToken(t, server, fixture.source.ID, tc.permission) + + events := runDraftRecoveryHTTP(t, server, secret, draftID, revision) + requirements.Len(events, 1) + assertions.Empty(events[0].Data) + assertions.Equal("draft_disabled", events[0].Error) + assertions.Zero(providerCalls) + }) + } +} + +func TestDraftRecoverPublishesKnownReplacement(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, user := newDraftRecoveryFixture(t) + candidate := []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Replacement\r\nContent-Type: text/plain\r\n\r\ncandidate\r\n") + testutil.AppendIMAPRawMessage(t, user, "Drafts", candidate) + reviewStoreFlagsForLifecycle(t, fixture.config.Addr(), 2, emersionimap.StoreFlagsAdd, emersionimap.FlagDraft) + uidNextBefore := recoveryUIDNext(t, fixture) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationEdit, candidate) + requirements.NoError(err) + replacement := store.IMAPDraftReceipt{SourceID: fixture.source.ID, Mailbox: "Drafts", UIDValidity: 1, UID: 2} + requirements.NoError(fixture.store.RecordIMAPDraftOutcomeContext(t.Context(), fixture.draft.DraftID, 1, "append_uidplus", &replacement)) + + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.NoError(err) + requirements.Len(events, 1) + var output draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Equal("edited", output.Status) + assertions.Equal(int64(2), output.Revision) + assertions.Equal(uint32(2), output.Receipt.UID) + assertions.Equal(uint32(1), reviewDraftMailboxCount(t, fixture.config.Addr())) + assertions.Equal(uidNextBefore, recoveryUIDNext(t, fixture)) + latest, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + requirements.Nil(latest.Pending) + requirements.Equal(replacement, latest.CurrentReceipt) +} + +func TestDraftRecoverCleansUpPublishedEdit(t *testing.T) { + for _, replacementState := range []string{"live", "absent", "deleted", "not draft"} { + t.Run(replacementState, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, user := newDraftRecoveryFixture(t) + candidate := []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Replacement\r\nContent-Type: text/plain\r\n\r\ncandidate\r\n") + testutil.AppendIMAPRawMessage(t, user, "Drafts", candidate) + reviewStoreFlagsForLifecycle(t, fixture.config.Addr(), 2, emersionimap.StoreFlagsAdd, emersionimap.FlagDraft) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationEdit, candidate) + requirements.NoError(err) + replacement := store.IMAPDraftReceipt{SourceID: fixture.source.ID, Mailbox: "Drafts", UIDValidity: 1, UID: 2} + requirements.NoError(fixture.store.RecordIMAPDraftOutcomeContext(t.Context(), fixture.draft.DraftID, 1, "append_uidplus", &replacement)) + published := publishRecoveryReplacement(t, fixture, candidate) + requirements.Equal(int64(2), published.Revision) + client := imaplib.NewClient(fixture.config, testutil.IMAPTestPassword) + defer func() { _ = client.Close() }() + switch replacementState { + case "absent": + _, err = client.RemoveDraft(t.Context(), recoveryTestReceipt(replacement)) + requirements.NoError(err) + case "deleted": + reviewStoreFlagsForLifecycle(t, fixture.config.Addr(), 2, emersionimap.StoreFlagsAdd, emersionimap.FlagDeleted) + case "not draft": + reviewStoreFlagsForLifecycle(t, fixture.config.Addr(), 2, emersionimap.StoreFlagsDel, emersionimap.FlagDraft) + } + + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "2", "--json") + requirements.NoError(err) + requirements.Len(events, 1) + var output draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Equal("edited", output.Status) + assertions.Equal(int64(2), output.Revision) + latest, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + assertions.Nil(latest.Pending) + assertions.Equal(replacement, latest.CurrentReceipt) + original, err := client.InspectDraft(t.Context(), recoveryTestReceipt(fixture.draft.CurrentReceipt)) + requirements.NoError(err) + assertions.False(original.Present) + }) + } +} + +func TestDraftRecoverRefusesUnavailableReplacement(t *testing.T) { + for _, tc := range []struct { + name string + code string + }{ + {name: "absent", code: "absent"}, + {name: "deleted", code: "already_deleted"}, + {name: "not draft", code: "not_draft"}, + } { + t.Run(tc.name, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, user := newDraftRecoveryFixture(t) + candidate := []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Replacement\r\nContent-Type: text/plain\r\n\r\ncandidate\r\n") + if tc.name == "deleted" || tc.name == "not draft" { + testutil.AppendIMAPRawMessage(t, user, "Drafts", candidate) + if tc.name == "deleted" { + reviewStoreFlagsForLifecycle(t, fixture.config.Addr(), 2, emersionimap.StoreFlagsAdd, emersionimap.FlagDraft) + reviewStoreFlagsForLifecycle(t, fixture.config.Addr(), 2, emersionimap.StoreFlagsAdd, emersionimap.FlagDeleted) + } + } + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationEdit, candidate) + requirements.NoError(err) + replacement := store.IMAPDraftReceipt{SourceID: fixture.source.ID, Mailbox: "Drafts", UIDValidity: 1, UID: 2} + requirements.NoError(fixture.store.RecordIMAPDraftOutcomeContext(t.Context(), fixture.draft.DraftID, 1, "append_uidplus", &replacement)) + + uidNextBefore := recoveryUIDNext(t, fixture) + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.Error(err) + assertions.Equal(tc.code, err.Error()) + requirements.Len(events, 1) + assertions.Equal(cliStreamStderr, events[0].Type) + assertions.Contains(events[0].Data, `"status":"refused"`) + assertions.Equal(uidNextBefore, recoveryUIDNext(t, fixture)) + + latest, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + assertions.Equal(int64(1), latest.Revision) + assertions.Equal(fixture.draft.CurrentReceipt, latest.CurrentReceipt) + requirements.NotNil(latest.Pending) + assertions.Equal(store.IMAPDraftOperationEdit, latest.Pending.Operation) + assertions.Equal("append_uidplus", latest.Pending.Code) + assertions.Equal(replacement, *latest.Pending.ReplacementReceipt) + }) + } +} + +func TestDraftRecoverRemovesPresentOriginal(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + uidNextBefore := recoveryUIDNext(t, fixture) + + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.NoError(err) + requirements.Len(events, 1) + var output draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Equal("deleted", output.Status) + assertions.Equal(int64(2), output.Revision) + assertions.Equal(uidNextBefore, recoveryUIDNext(t, fixture)) + assertions.Equal(uint32(0), reviewDraftMailboxCount(t, fixture.config.Addr())) + latest, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + assertions.Nil(latest.Pending) + assertions.NotNil(latest.DiscardedAt) +} + +func TestDraftRecoverArchivedReplacementReportsAcceptedLocalFailure(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, user := newDraftRecoveryFixture(t) + candidate := []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Replacement\r\nContent-Type: text/plain\r\n\r\ncandidate\r\n") + testutil.AppendIMAPRawMessage(t, user, "Drafts", candidate) + reviewStoreFlagsForLifecycle(t, fixture.config.Addr(), 2, emersionimap.StoreFlagsAdd, emersionimap.FlagDraft) + message, err := fixture.store.GetMessageContext(t.Context(), fixture.draft.CurrentMessageID) + requirements.NoError(err) + archivedReceipt := store.IMAPDraftReceipt{SourceID: fixture.source.ID, Mailbox: "Drafts", UIDValidity: 1, UID: 2} + _, err = fixture.store.PersistIMAPDraftContext(t.Context(), archivedReceipt, nil, func([]int64) *store.MessagePersistData { + return &store.MessagePersistData{ + Message: &store.Message{ + SourceID: fixture.source.ID, SourceMessageID: store.IMAPDraftSourceMessageID(archivedReceipt), + MessageType: store.MessageTypeEmail, ConversationID: message.ConversationID, + }, + RawMIME: candidate, + } + }) + requirements.NoError(err) + _, err = fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationEdit, candidate) + requirements.NoError(err) + requirements.NoError(fixture.store.RecordIMAPDraftOutcomeContext(t.Context(), fixture.draft.DraftID, 1, "append_uidplus", &archivedReceipt)) + + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.Error(err) + assertions.Equal("accepted_local_failed", err.Error()) + requirements.Len(events, 1) + assertions.Equal(cliStreamStderr, events[0].Type) + assertions.Contains(events[0].Data, `"status":"accepted_local_failed"`) + latest, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + assertions.Equal(int64(1), latest.Revision) + assertions.Equal(fixture.draft.CurrentReceipt, latest.CurrentReceipt) + requirements.NotNil(latest.Pending) + assertions.Equal("append_uidplus", latest.Pending.Code) + assertions.Equal(archivedReceipt, *latest.Pending.ReplacementReceipt) +} + +func TestDraftRecoverRefusesDifferingRecordedGenerations(t *testing.T) { + requirements := require.New(t) + fixture, _ := newDraftRecoveryFixture(t) + candidate := []byte("From: alice@example.com\r\nTo: bob@example.com\r\nContent-Type: text/plain\r\n\r\ncandidate\r\n") + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationEdit, candidate) + requirements.NoError(err) + replacement := store.IMAPDraftReceipt{SourceID: fixture.source.ID, Mailbox: "Drafts", UIDValidity: 2, UID: 2} + requirements.NoError(fixture.store.RecordIMAPDraftOutcomeContext(t.Context(), fixture.draft.DraftID, 1, "append_uidplus", &replacement)) + + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.Error(err) + assert.Equal(t, "uidvalidity_mismatch", err.Error()) + requirements.Len(events, 1) + latest, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + requirements.NotNil(latest.Pending) + assert.Equal(t, int64(1), latest.Revision) +} + +func TestDraftRecoverKeepsRowUnchangedOnRemovalGenerationRefusal(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + control := &reviewUIDValidityChangeControl{} + addr, user := startReviewUIDValidityChangeServer(t, control) + fixture := newReviewManagedLifecycleFixtureOnServer(t, addr, func() { + testutil.AppendIMAPRawMessage(t, user, "Drafts", []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Original\r\nContent-Type: text/plain\r\n\r\noriginal\r\n")) + }) + control.selects.Store(0) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + before, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + var beforeUpdatedAt string + requirements.NoError(fixture.store.DB().QueryRow(fixture.store.Rebind(` + SELECT CAST(updated_at AS TEXT) FROM imap_drafts WHERE draft_id = ? + `), fixture.draft.DraftID).Scan(&beforeUpdatedAt)) + + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.Error(err) + assertions.Equal("uidvalidity_mismatch", err.Error()) + requirements.Len(events, 1) + assertions.Equal(cliStreamStderr, events[0].Type) + assertions.Contains(events[0].Data, `"code":"uidvalidity_mismatch"`) + var output draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Equal(before.Pending.Code, output.PendingCode) + assertions.Equal("uidvalidity_mismatch", output.RefusalCode) + assertions.Equal(int32(2), control.selects.Load()) + after, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + var afterUpdatedAt string + requirements.NoError(fixture.store.DB().QueryRow(fixture.store.Rebind(` + SELECT CAST(updated_at AS TEXT) FROM imap_drafts WHERE draft_id = ? + `), fixture.draft.DraftID).Scan(&afterUpdatedAt)) + assertions.Equal(before.Revision, after.Revision) + assertions.Equal(before.CurrentReceipt, after.CurrentReceipt) + requirements.NotNil(after.Pending) + assertions.Equal(before.Pending.Code, after.Pending.Code) + assertions.Equal(before.Pending.OriginalReceipt, after.Pending.OriginalReceipt) + assertions.Equal(before.Pending.ReplacementReceipt, after.Pending.ReplacementReceipt) + assertions.Equal(beforeUpdatedAt, afterUpdatedAt) +} + +func TestDraftRecoverRecordsWriteAttemptedRemovalFailure(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + addr, user := startReviewExpungeFailureServer(t) + fixture := newReviewManagedLifecycleFixtureOnServer(t, addr, func() { + testutil.AppendIMAPRawMessage(t, user, "Drafts", []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Original\r\nContent-Type: text/plain\r\n\r\noriginal\r\n")) + }) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.Error(err) + assertions.Equal("expunge_failed", err.Error()) + requirements.Len(events, 1) + assertions.Equal(cliStreamStderr, events[0].Type) + assertions.Contains(events[0].Data, `"pending_code":"expunge_failed"`) + latest, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + requirements.NotNil(latest.Pending) + assertions.Equal("expunge_failed", latest.Pending.Code) + assertions.Equal(int64(1), latest.Revision) +} + +func TestDraftRecoverReportsProviderRefusal(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + fixture.adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + return imaplib.NewClient(&imaplib.Config{ + Host: "127.0.0.1", Port: 1, Username: testutil.IMAPTestUsername, + }, testutil.IMAPTestPassword), nil + } + + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.Error(err) + assertions.Equal("provider_refused", err.Error()) + requirements.Len(events, 1) + assertions.Equal(cliStreamStderr, events[0].Type) + assertions.Contains(events[0].Data, `"status":"refused"`) + latest, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + requirements.NotNil(latest.Pending) + assertions.Empty(latest.Pending.Code) + assertions.Equal(int64(1), latest.Revision) +} + +func TestDraftRecoverRefusesUnknownReplacement(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationEdit, []byte("candidate")) + requirements.NoError(err) + providerCalls := 0 + fixture.adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + providerCalls++ + return nil, errors.New("unknown replacement must not connect") + } + + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.Error(err) + assertions.Equal("unknown_replacement", err.Error()) + assertions.Equal(0, providerCalls) + requirements.Len(events, 1) + assertions.Equal(cliStreamStderr, events[0].Type) + latest, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + requirements.NotNil(latest.Pending) + assertions.Equal(int64(1), latest.Revision) +} + +func TestDraftRecoverPolicy(t *testing.T) { + for _, tc := range []struct { + name string + setup func(*testing.T, reviewManagedLifecycleFixture) + code string + }{ + {name: "draft disabled", setup: func(t *testing.T, fixture reviewManagedLifecycleFixture) { + t.Helper() + requirements := require.New(t) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + fixture.adapter.draftPolicy = nil + }, code: "draft_disabled"}, + {name: "invalid mailbox", setup: func(t *testing.T, fixture reviewManagedLifecycleFixture) { + t.Helper() + requirements := require.New(t) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + fixture.adapter.draftPolicy[0].Mailbox = "\n" + }, code: "invalid_mailbox"}, + } { + t.Run(tc.name, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + tc.setup(t, fixture) + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.Error(err) + assertions.Equal(tc.code, err.Error()) + assertions.Empty(events) + assertions.Equal(uint32(1), reviewDraftMailboxCount(t, fixture.config.Addr())) + }) + } +} + +func TestDraftRecoverParser(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + for _, args := range [][]string{ + {draftRecoverCommand, "draft-1"}, + {draftRecoverCommand, "draft-1", "--revision", "1", "--body", "blocked"}, + {draftRecoverCommand, "draft-1", "--revision", "0"}, + } { + _, err := parseDraftLifecycleArgs(args) + requirements.Error(err) + assertions.Equal("invalid_args", err.Error()) + } + intent, err := parseDraftLifecycleArgs([]string{draftRecoverCommand, "draft-1", "--revision", "1", "--json"}) + requirements.NoError(err) + assertions.Equal(draftRecoverCommand, intent.Operation) +} + +func TestDraftRecoverSavedRemovedDeleteFinishesWithoutProvider(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + requirements.NoError(fixture.store.RecordIMAPDraftOutcomeContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftCodeRemoved, nil)) + fixture.adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + return nil, errors.New("saved removal must finish locally") + } + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.NoError(err) + requirements.Len(events, 1) + assertions.Contains(events[0].Data, `"status":"deleted"`) + assertions.Equal(uint32(1), reviewDraftMailboxCount(t, fixture.config.Addr())) +} + +func TestDraftRecoverSavedRemovedEditFinishesWithoutProvider(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, user := newDraftRecoveryFixture(t) + candidate := []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Replacement\r\nContent-Type: text/plain\r\n\r\ncandidate\r\n") + testutil.AppendIMAPRawMessage(t, user, "Drafts", candidate) + reviewStoreFlagsForLifecycle(t, fixture.config.Addr(), 2, emersionimap.StoreFlagsAdd, emersionimap.FlagDraft) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationEdit, candidate) + requirements.NoError(err) + replacement := store.IMAPDraftReceipt{SourceID: fixture.source.ID, Mailbox: "Drafts", UIDValidity: 1, UID: 2} + requirements.NoError(fixture.store.RecordIMAPDraftOutcomeContext(t.Context(), fixture.draft.DraftID, 1, "append_uidplus", &replacement)) + published := publishRecoveryReplacement(t, fixture, candidate) + requirements.Equal(int64(2), published.Revision) + requirements.NoError(fixture.store.RecordIMAPDraftOutcomeContext(t.Context(), fixture.draft.DraftID, published.Revision, store.IMAPDraftCodeRemoved, nil)) + fixture.adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + return nil, errors.New("saved removal must finish locally") + } + + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "2", "--json") + requirements.NoError(err) + requirements.Len(events, 1) + assertions.Contains(events[0].Data, `"status":"edited"`) + latest, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + assertions.Nil(latest.Pending) + assertions.Equal(int64(2), latest.Revision) + assertions.Equal(replacement, latest.CurrentReceipt) +} + +func TestDraftRecoverReloadsSettledStateAfterSourceLock(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, user := newDraftRecoveryFixture(t) + candidate := []byte("From: alice@example.com\r\nTo: bob@example.com\r\nSubject: Replacement\r\nContent-Type: text/plain\r\n\r\ncandidate\r\n") + testutil.AppendIMAPRawMessage(t, user, "Drafts", candidate) + reviewStoreFlagsForLifecycle(t, fixture.config.Addr(), 2, emersionimap.StoreFlagsAdd, emersionimap.FlagDraft) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationEdit, candidate) + requirements.NoError(err) + replacement := store.IMAPDraftReceipt{SourceID: fixture.source.ID, Mailbox: "Drafts", UIDValidity: 1, UID: 2} + requirements.NoError(fixture.store.RecordIMAPDraftOutcomeContext(t.Context(), fixture.draft.DraftID, 1, "append_uidplus", &replacement)) + published := publishRecoveryReplacement(t, fixture, candidate) + requirements.NoError(fixture.store.RecordIMAPDraftOutcomeContext(t.Context(), fixture.draft.DraftID, published.Revision, store.IMAPDraftCodeRemoved, nil)) + providerCalls := 0 + fixture.adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + providerCalls++ + return nil, errors.New("settled recovery must not connect") + } + fired := false + previous := slog.Default() + slog.SetDefault(slog.New(reviewDraftCommitHandler{ + Handler: slog.NewTextHandler(&bytes.Buffer{}, &slog.HandlerOptions{Level: slog.LevelDebug}), + onCommit: func() { + if fired { + return + } + fired = true + _, finishErr := fixture.store.FinishIMAPDraftRemovalContext(t.Context(), fixture.draft.DraftID, published.Revision) + requirements.NoError(finishErr) + }, + })) + defer slog.SetDefault(previous) + + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "2", "--json") + requirements.NoError(err) + requirements.True(fired) + requirements.Len(events, 1) + assertions.Contains(events[0].Data, `"status":"active"`) + assertions.Zero(providerCalls) +} + +func TestDraftRecoverReloadsActionAndSourceAfterSourceLock(t *testing.T) { + for _, tc := range []struct { + name string + mutate func(*testing.T, reviewManagedLifecycleFixture) + }{ + { + name: "action", + mutate: func(t *testing.T, fixture reviewManagedLifecycleFixture) { + t.Helper() + _, err := fixture.store.DB().Exec(fixture.store.Rebind(` + UPDATE imap_drafts + SET pending_operation = 'delete', pending_raw = NULL, + pending_replacement_mailbox = NULL, + pending_replacement_uidvalidity = NULL, + pending_replacement_uid = NULL + WHERE draft_id = ? + `), fixture.draft.DraftID) + require.NoError(t, err) + }, + }, + { + name: "draft removed", + mutate: func(t *testing.T, fixture reviewManagedLifecycleFixture) { + t.Helper() + _, err := fixture.store.DB().Exec(fixture.store.Rebind(`DELETE FROM imap_drafts WHERE draft_id = ?`), fixture.draft.DraftID) + require.NoError(t, err) + }, + }, + { + name: "source type", + mutate: func(t *testing.T, fixture reviewManagedLifecycleFixture) { + t.Helper() + _, err := fixture.store.DB().Exec(fixture.store.Rebind(` + UPDATE sources SET source_type = 'gmail' WHERE id = ? + `), fixture.source.ID) + require.NoError(t, err) + }, + }, + { + name: "source identifier", + mutate: func(t *testing.T, fixture reviewManagedLifecycleFixture) { + t.Helper() + _, err := fixture.store.DB().Exec(fixture.store.Rebind(` + UPDATE sources SET identifier = 'imap://changed@example.test:143' WHERE id = ? + `), fixture.source.ID) + require.NoError(t, err) + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + candidate := []byte("From: alice@example.com\r\nTo: bob@example.com\r\nContent-Type: text/plain\r\n\r\ncandidate\r\n") + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationEdit, candidate) + requirements.NoError(err) + replacement := store.IMAPDraftReceipt{SourceID: fixture.source.ID, Mailbox: "Drafts", UIDValidity: 1, UID: 2} + requirements.NoError(fixture.store.RecordIMAPDraftOutcomeContext(t.Context(), fixture.draft.DraftID, 1, "append_uidplus", &replacement)) + grant := &agentgrant.Grant{ + ID: "reload-grant", + Permissions: []agentgrant.Permission{agentgrant.PermissionDraftEdit}, + Sources: []agentgrant.SourceRef{{ID: fixture.source.ID, Type: fixture.source.SourceType, Identifier: fixture.source.Identifier}}, + } + providerCalls := 0 + fixture.adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + providerCalls++ + return nil, errors.New("reload denial must not connect") + } + fired := false + previous := slog.Default() + slog.SetDefault(slog.New(reviewDraftCommitHandler{ + Handler: slog.NewTextHandler(&bytes.Buffer{}, &slog.HandlerOptions{Level: slog.LevelDebug}), + onCommit: func() { + if fired { + return + } + fired = true + tc.mutate(t, fixture) + }, + })) + defer slog.SetDefault(previous) + + var events []api.CLIRunEvent + err = fixture.adapter.runCLIDraftLifecycle(t.Context(), api.CLIRunRequest{ + Args: []string{draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json"}, + Grant: grant, + }, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + requirements.Error(err) + assertions.Equal("not_permitted", err.Error()) + assertions.True(fired) + assertions.Empty(events) + assertions.Zero(providerCalls) + }) + } +} + +func TestDraftRecoverRejectsInvalidPendingOperation(t *testing.T) { + testutil.SkipIfPostgres(t, "invalid pending operation injection uses SQLite check-constraint bypass") + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + requirements.NoError(func() error { + _, err := fixture.store.DB().Exec("PRAGMA ignore_check_constraints = ON") + return err + }()) + _, err = fixture.store.DB().Exec(fixture.store.Rebind(` + UPDATE imap_drafts SET pending_operation = 'invalid' WHERE draft_id = ? + `), fixture.draft.DraftID) + requirements.NoError(err) + requirements.NoError(func() error { + _, err := fixture.store.DB().Exec("PRAGMA ignore_check_constraints = OFF") + return err + }()) + providerCalls := 0 + fixture.adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + providerCalls++ + return nil, errors.New("invalid operation must not connect") + } + + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.Error(err) + assertions.Equal("invalid_state", err.Error()) + assertions.Empty(events) + assertions.Zero(providerCalls) +} + +func TestDraftRecoverRejectsWrongSourceAndAction(t *testing.T) { + for _, tc := range []struct { + name string + permission agentgrant.Permission + source func(reviewManagedLifecycleFixture) agentgrant.SourceRef + }{ + { + name: "wrong action", + permission: agentgrant.PermissionDraftDelete, + source: func(fixture reviewManagedLifecycleFixture) agentgrant.SourceRef { + return agentgrant.SourceRef{ID: fixture.source.ID, Type: fixture.source.SourceType, Identifier: fixture.source.Identifier} + }, + }, + { + name: "wrong source type", + permission: agentgrant.PermissionDraftEdit, + source: func(fixture reviewManagedLifecycleFixture) agentgrant.SourceRef { + return agentgrant.SourceRef{ID: fixture.source.ID, Type: "gmail", Identifier: fixture.source.Identifier} + }, + }, + { + name: "wrong source identifier", + permission: agentgrant.PermissionDraftEdit, + source: func(fixture reviewManagedLifecycleFixture) agentgrant.SourceRef { + return agentgrant.SourceRef{ID: fixture.source.ID, Type: fixture.source.SourceType, Identifier: "imap://changed@example.test:143"} + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + candidate := []byte("From: alice@example.com\r\nTo: bob@example.com\r\nContent-Type: text/plain\r\n\r\ncandidate\r\n") + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationEdit, candidate) + requirements.NoError(err) + grant := &agentgrant.Grant{ + ID: "denial-grant", + Permissions: []agentgrant.Permission{tc.permission}, + Sources: []agentgrant.SourceRef{tc.source(fixture)}, + } + providerCalls := 0 + fixture.adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + providerCalls++ + return nil, errors.New("denial must not connect") + } + var events []api.CLIRunEvent + err = fixture.adapter.runCLIDraftLifecycle(t.Context(), api.CLIRunRequest{ + Args: []string{draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json"}, + Grant: grant, + }, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + requirements.Error(err) + assertions.Equal("not_permitted", err.Error()) + assertions.Empty(events) + assertions.Zero(providerCalls) + }) + } +} + +func TestDraftRecoverSettledRepeatNoProvider(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + normalFactory := fixture.adapter.draftClientFactory + providerCalls := 0 + fixture.adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + providerCalls++ + return nil, errors.New("settled recovery must not connect") + } + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.NoError(err) + requirements.Len(events, 1) + assertions.Contains(events[0].Data, `"status":"active"`) + assertions.Equal(0, providerCalls) + + fixture.adapter.draftClientFactory = normalFactory + _, err = fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + removeRecoveryOriginal(t, fixture) + _, err = runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.NoError(err) + + providerCalls = 0 + fixture.adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + providerCalls++ + return nil, errors.New("discarded recovery must not connect") + } + events, err = runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "2", "--json") + requirements.NoError(err) + requirements.Len(events, 1) + assertions.Contains(events[0].Data, `"status":"already_discarded"`) + assertions.Equal(0, providerCalls) +} + +func TestDraftRecoverRejectsEnvAndCwd(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + adapter := &storeAPIAdapter{} + for _, request := range []api.CLIRunRequest{ + {Args: []string{draftRecoverCommand, "draft-1", "--revision", "1"}, Env: map[string]string{"HOME": "blocked"}}, + {Args: []string{draftRecoverCommand, "draft-1", "--revision", "1"}, Cwd: `C:\blocked`}, + } { + err := adapter.runCLIDraftLifecycle(t.Context(), request, nil) + requirements.Error(err) + assertions.Equal("invalid_args", err.Error()) + } +} + +func publishRecoveryReplacement(t *testing.T, fixture reviewManagedLifecycleFixture, candidate []byte) store.IMAPDraft { + t.Helper() + draft, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + require.NoError(t, err) + parsed, err := msgmime.Parse(candidate) + require.NoError(t, err) + message, err := fixture.store.GetMessageContext(t.Context(), draft.CurrentMessageID) + require.NoError(t, err) + replyTo, err := fixture.store.GetMessageReplyToMessageIDContext(t.Context(), draft.CurrentMessageID) + require.NoError(t, err) + replacement := *draft.Pending.ReplacementReceipt + participants, build := draftLifecyclePersistData(message.ConversationID, replyTo, imaplib.ReplyDraft{Raw: candidate, Parsed: parsed}, replacement, nil) + published, err := fixture.store.PublishIMAPDraftReplacementContext(t.Context(), draft.DraftID, draft.Revision, participants, build) + require.NoError(t, err) + return published +} + +func TestDraftRecoverRefusalAfterCancellation(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + fixture.adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + cancel() + return imaplib.NewClient(fixture.config, testutil.IMAPTestPassword), nil + } + var events []api.CLIRunEvent + err = fixture.adapter.runCLIDraftLifecycle(ctx, api.CLIRunRequest{ + Args: []string{draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json"}, + }, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + requirements.EqualError(err, "cancelled") + requirements.Len(events, 1) + assertions.Equal(cliStreamStderr, events[0].Type) + var output draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Equal("refused", output.Status) + assertions.Equal("cancelled", output.RefusalCode) + assertions.Empty(output.PendingCode) + requirements.NotNil(output.ProviderObservation) + assertions.Equal("cancelled", output.ProviderObservation.Code) +} + +func TestDraftRecoverDelegatedCompletionWithoutContentReads(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + _, err = fixture.store.DB().Exec(fixture.store.Rebind(` + UPDATE message_raw SET raw_data = ?, compression = 'zlib' WHERE message_id = ? + `), []byte("invalid compressed content"), fixture.draft.CurrentMessageID) + requirements.NoError(err) + server := newDraftRecoveryHTTPServer(t, fixture) + defer server.Close() + secret := issueDraftRecoveryToken(t, server, fixture.source.ID, string(agentgrant.PermissionDraftDelete)) + + events := runDraftRecoveryHTTP(t, server, secret, fixture.draft.DraftID, 1) + requirements.Len(events, 2) + assertions.Equal(cliStreamStdout, events[0].Type) + assertions.Equal("complete", events[1].Type) + assertions.Empty(events[1].Error) + var output draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Equal("deleted", output.Status) + assertions.Empty(output.Content) + assertions.Empty(output.RawMIME) + assertions.Empty(output.CandidateContent) +} + +func TestDraftRecoverAlreadyDeletedNeedsManualExpunge(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture, _ := newDraftRecoveryFixture(t) + _, err := fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, 1, store.IMAPDraftOperationDelete, nil) + requirements.NoError(err) + requirements.NoError(fixture.store.RecordIMAPDraftOutcomeContext(t.Context(), fixture.draft.DraftID, 1, "expunge_failed", nil)) + reviewStoreFlagsForLifecycle(t, fixture.config.Addr(), 1, emersionimap.StoreFlagsAdd, emersionimap.FlagDeleted) + + events, err := runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.EqualError(err, "already_deleted") + requirements.Len(events, 1) + var output draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Equal("expunge_failed", output.PendingCode) + assertions.Equal("already_deleted", output.RefusalCode) + assertions.Equal(uint32(1), reviewDraftMailboxCount(t, fixture.config.Addr())) + + client, err := imapclient.DialInsecure(fixture.config.Addr(), nil) + requirements.NoError(err) + defer func() { _ = client.Close() }() + requirements.NoError(client.Login(testutil.IMAPTestUsername, testutil.IMAPTestPassword).Wait()) + _, err = client.Select("Drafts", nil).Wait() + requirements.NoError(err) + requirements.NoError(client.UIDExpunge(emersionimap.UIDSetNum(1)).Close()) + events, err = runReviewLifecycle(t, fixture.adapter, draftRecoverCommand, fixture.draft.DraftID, "--revision", "1", "--json") + requirements.NoError(err) + requirements.Len(events, 1) + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assertions.Equal("deleted", output.Status) + latest, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + assertions.Nil(latest.Pending) + assertions.NotNil(latest.DiscardedAt) +} diff --git a/cmd/msgvault/cmd/serve_environment_test.go b/cmd/msgvault/cmd/serve_environment_test.go new file mode 100644 index 000000000..fee9d587c --- /dev/null +++ b/cmd/msgvault/cmd/serve_environment_test.go @@ -0,0 +1,231 @@ +package cmd + +import ( + "bytes" + "context" + "fmt" + "io" + "net" + "net/http" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" + "time" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/providercredentials" +) + +func TestRunServeFailedRestartKeepsRunningDaemonKey(t *testing.T) { //nolint:paralleltest // process environment and daemon lifecycle + if runtime.GOOS == "windows" || os.Geteuid() == 0 { + t.Skip("requires Unix file permissions enforced for the current user") + } + require := require.New(t) + clearServerKeyEnvironment(t) + t.Setenv("MSGVAULT_REMOTE_URL", "") + t.Setenv("MSGVAULT_ALLOW_INSECURE", "false") + home := t.TempDir() + port := freeTCPPort(t) + path := filepath.Join(home, "config.toml") + require.NoError(os.WriteFile(path, []byte(fmt.Sprintf("[server]\nbind_addr = '127.0.0.1'\napi_port = %d\ndaemon_auto_start = false\ndaemon_auto_restart = 'never'\n[analytics]\nengine = 'sql'\n", port)), 0o600)) + running, err := config.Load(path, home) + require.NoError(err) + ctx, cancel := context.WithCancel(t.Context()) + command := &cobra.Command{Use: "serve"} + command.SetContext(testInvocationContext(ctx, running, invocationOptions{})) + done := make(chan error, 1) + go func() { done <- runServe(command, nil) }() + t.Cleanup(func() { + cancel() + select { + case err := <-done: + require.NoError(err) + case <-time.After(serveLifecycleTestTimeout): + require.Fail("daemon did not stop") + } + }) + waitForServeHealthBounded(t, port, done) + initial, _, err := OpenHTTPStore(withStoreResolverConfig(t, running)) + require.NoError(err) + require.NoError(initial.Close()) + + replacement, err := config.LoadWithOverrides(path, home, config.RuntimeOverrides{BindAddr: new("0.0.0.0")}) + require.NoError(err) + lockPath := daemonOwnerLockPath(home) + require.NoError(os.Chmod(lockPath, 0o400)) + t.Cleanup(func() { require.NoError(os.Chmod(lockPath, 0o600)) }) + restartCommand, _, _ := lifecycleTestCommand() + err = runServeRestart(restartCommand, replacement) + require.ErrorIs(err, os.ErrPermission) + require.NoError(os.Chmod(lockPath, 0o600)) + _, err = os.Stat(replacement.ServerKeyFilePath()) + require.ErrorIs(err, os.ErrNotExist, "a failed restart must not change the running daemon's credential") + + fresh, err := config.Load(path, home) + require.NoError(err) + client, _, err := OpenHTTPStore(withStoreResolverConfig(t, fresh)) + require.NoError(err, "fresh clients must still connect after the failed restart") + require.NoError(client.Close()) +} + +func TestRunServeRejectedContenderKeepsRunningDaemonKey(t *testing.T) { //nolint:paralleltest // process environment and daemon lifecycle + require := require.New(t) + clearServerKeyEnvironment(t) + t.Setenv("MSGVAULT_REMOTE_URL", "") + t.Setenv("MSGVAULT_ALLOW_INSECURE", "false") + home := t.TempDir() + port := freeTCPPort(t) + path := filepath.Join(home, "config.toml") + require.NoError(os.WriteFile(path, []byte(fmt.Sprintf("[server]\nbind_addr = '127.0.0.1'\napi_port = %d\ndaemon_auto_start = false\ndaemon_auto_restart = 'never'\n[analytics]\nengine = 'sql'\n", port)), 0o600)) + running, err := config.Load(path, home) + require.NoError(err) + ctx, cancel := context.WithCancel(t.Context()) + command := &cobra.Command{Use: "serve"} + command.SetContext(testInvocationContext(ctx, running, invocationOptions{})) + done := make(chan error, 1) + go func() { done <- runServe(command, nil) }() + t.Cleanup(func() { + cancel() + select { + case err := <-done: + require.NoError(err) + case <-time.After(serveLifecycleTestTimeout): + require.Fail("daemon did not stop") + } + }) + waitForServeHealthBounded(t, port, done) + initial, _, err := OpenHTTPStore(withStoreResolverConfig(t, running)) + require.NoError(err) + require.NoError(initial.Close()) + + contender, err := config.LoadWithOverrides(path, home, config.RuntimeOverrides{BindAddr: new("0.0.0.0"), APIPort: new(0)}) + require.NoError(err) + contenderCommand := &cobra.Command{Use: "serve"} + contenderCommand.SetContext(testInvocationContext(t.Context(), contender, invocationOptions{})) + err = runServe(contenderCommand, nil) + require.ErrorAs(err, &daemonOwnerLockHeldError{}) + _, err = os.Stat(contender.ServerKeyFilePath()) + require.ErrorIs(err, os.ErrNotExist, "a rejected contender must not create a new daemon credential") + + fresh, err := config.Load(path, home) + require.NoError(err) + client, _, err := OpenHTTPStore(withStoreResolverConfig(t, fresh)) + require.NoError(err, "the running daemon must remain usable by fresh local clients") + require.NoError(client.Close()) +} + +func TestRunServeCredentialFailureReleasesStartupResources(t *testing.T) { + require := require.New(t) + cfg := config.NewDefaultConfig() + cfg.HomeDir = t.TempDir() + cfg.Data.DataDir = cfg.HomeDir + cfg.Server.APIPort = freeTCPPort(t) + cfg.Server.APIKeyFile = filepath.Join(cfg.HomeDir, "missing-key") + command := &cobra.Command{Use: "serve"} + command.SetContext(testInvocationContext(t.Context(), cfg, invocationOptions{})) + require.ErrorIs(runServe(command, nil), os.ErrNotExist) + + daemonLock, err := tryAcquireDaemonOwnerLock(cfg.Data.DataDir) + require.NoError(err, "credential failure must release daemon ownership") + t.Cleanup(func() { require.NoError(daemonLock.Close()) }) + writeLock, err := tryAcquireWriteOwnerLock(cfg.Data.DataDir) + require.NoError(err, "credential failure must release archive ownership") + t.Cleanup(func() { require.NoError(writeLock.Close()) }) + listener, err := net.Listen("tcp", net.JoinHostPort(cfg.Server.BindAddr, strconv.Itoa(cfg.Server.APIPort))) + require.NoError(err, "credential failure must release the reserved listener") + require.NoError(listener.Close()) + records, err := daemonRuntimeStore(cfg.Data.DataDir).List() + require.NoError(err) + require.Empty(records, "failed credential preparation must not publish a runtime record") +} + +func TestRunServeEnvironmentOnlyAuthenticatedRestart(t *testing.T) { //nolint:paralleltest // process environment and daemon lifecycle + assert := assert.New(t) + require := require.New(t) + home := t.TempDir() + port := freeTCPPort(t) + t.Setenv("MSGVAULT_HOME", home) + t.Setenv("MSGVAULT_BIND_ADDR", "0.0.0.0") + t.Setenv("MSGVAULT_API_PORT", strconv.Itoa(port)) + var firstKey string + for start := range 2 { + cfg, err := config.Load("", home) + require.NoError(err) + ctx, cancel := context.WithCancel(t.Context()) + command := &cobra.Command{Use: "serve"} + command.SetContext(testInvocationContext(ctx, cfg, invocationOptions{})) + done := make(chan error, 1) + stopped := false + stop := func() { + if stopped { + return + } + cancel() + select { + case err := <-done: + require.NoError(err) + stopped = true + case <-time.After(serveLifecycleTestTimeout): + assert.Fail("daemon did not stop") + } + } + t.Cleanup(stop) + go func() { done <- runServe(command, nil) }() + waitForServeHealthBounded(t, port, done) + key, err := providercredentials.ReadSecretFile(cfg.ServerKeyFilePath()) + require.NoError(err) + if start == 0 { + firstKey = key + } else { + assert.Equal(firstKey, key) + } + _, err = os.Stat(filepath.Join(home, "config.toml")) + require.ErrorIs(err, os.ErrNotExist) + client := &http.Client{Timeout: serveLifecycleTestTimeout} + var status int + require.Eventually(func() bool { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/api/v1/health", port), nil) + if err != nil { + return false + } + request.Header.Set("X-Api-Key", key) + response, err := client.Do(request) + if err != nil { + return false + } + status = response.StatusCode + _ = response.Body.Close() + return status == http.StatusOK + }, serveLifecycleTestTimeout, 20*time.Millisecond) + // Startup polling can consume the public request burst. Wait for the + // unauthenticated request to reach auth after the limiter replenishes. + require.Eventually(func() bool { + response, err := client.Get(fmt.Sprintf("http://127.0.0.1:%d/api/v1/health", port)) + if err != nil { + return false + } + status = response.StatusCode + _ = response.Body.Close() + return status == http.StatusUnauthorized + }, serveLifecycleTestTimeout, 200*time.Millisecond) + var output bytes.Buffer + statusConfig, err := config.Load("", home) + require.NoError(err) + statusCommand := newLifecycleCommand("status", false) + statusCommand.SetContext(testInvocationContext(ctx, statusConfig, invocationOptions{})) + statusCommand.SetOut(&output) + statusCommand.SetErr(io.Discard) + require.Eventually(func() bool { + output.Reset() + return statusCommand.RunE(statusCommand, nil) == nil && strings.Contains(output.String(), strconv.Itoa(port)) + }, serveLifecycleTestTimeout, 200*time.Millisecond) + assert.NotContains(output.String(), key) + stop() + } +} diff --git a/cmd/msgvault/cmd/serve_forward_draft.go b/cmd/msgvault/cmd/serve_forward_draft.go new file mode 100644 index 000000000..e9c521e18 --- /dev/null +++ b/cmd/msgvault/cmd/serve_forward_draft.go @@ -0,0 +1,305 @@ +package cmd + +import ( + "context" + "encoding/json/v2" + "errors" + "fmt" + "io" + "strconv" + "strings" + "time" + + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/attachmentpolicy" + imaplib "go.kenn.io/msgvault/internal/imap" + msgmime "go.kenn.io/msgvault/internal/mime" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/textutil" +) + +type draftForwardIntent struct { + draftComposeIntent + + MessageID int64 +} + +type draftForwardProblem struct { + Filename string `json:"filename,omitempty"` + PartKey string `json:"part_key,omitempty"` + Reason string `json:"reason"` + Detail string `json:"detail,omitempty"` +} + +type draftForwardPreflightOutput struct { + Status string `json:"status"` + Problems []draftForwardProblem `json:"problems"` +} + +// parseDraftForwardArgs takes one message ID plus draft-compose's flags, +// except --subject, which the forward derives from the parent. +func parseDraftForwardArgs(args []string) (draftForwardIntent, error) { + invalid := func(format string, args ...any) (draftForwardIntent, error) { + return draftForwardIntent{}, draftReplyError("invalid_args", fmt.Errorf(format, args...)) + } + if !api.IsCLIRunDraftForward(args) { + return invalid("expected %s as the first argument", api.CLIRunDraftForwardCommand) + } + composeArgs := []string{api.CLIRunDraftComposeCommand} + var positional []string + for i := 1; i < len(args); i++ { + nameValue, isFlag := strings.CutPrefix(args[i], "--") + name, _, hasValue := strings.Cut(nameValue, "=") + switch { + case !isFlag: + positional = append(positional, args[i]) + case name == "subject": + return invalid("unknown flag --subject") + default: + composeArgs = append(composeArgs, args[i]) + switch name { + case draftFromFlag, "account", "source-id", "body", "to", "cc", "bcc", "log-level", "log-sql-slow-ms": + if !hasValue && i+1 < len(args) { + i++ + composeArgs = append(composeArgs, args[i]) + } + } + } + } + if len(positional) != 1 { + return invalid("expected exactly one message ID") + } + messageID, err := strconv.ParseInt(strings.TrimSpace(positional[0]), 10, 64) + if err != nil || messageID <= 0 { + return invalid("message ID must be a positive integer") + } + compose, err := parseDraftComposeArgs(composeArgs) + if err != nil { + return draftForwardIntent{}, err + } + return draftForwardIntent{draftComposeIntent: compose, MessageID: messageID}, nil +} + +func (a *storeAPIAdapter) runCLIForwardDraft( + ctx context.Context, + req api.CLIRunRequest, + emit func(api.CLIRunEvent) error, +) error { + if req.Grant != nil { + return draftReplyError("not_permitted", errors.New("draft-forward requires owner access")) + } + if len(req.Env) != 0 || req.Cwd != "" { + return draftReplyError("invalid_args", errors.New("draft-forward accepts no environment or working directory")) + } + intent, err := parseDraftForwardArgs(req.Args) + if err != nil { + return err + } + target, from, _, err := a.resolveDraftTarget( + ctx, &intent.MessageID, intent.Account, intent.SourceID, intent.SourceIDSet, + intent.From, nil, + ) + if err != nil { + return err + } + if target.source.SourceType != "imap" { + return draftReplyError("draft_disabled", errors.New("draft-forward requires an IMAP source")) + } + parent, err := msgmime.Parse(target.raw) + if err != nil { + return draftReplyError("invalid_parent", fmt.Errorf("parse parent MIME: %w", err)) + } + // Sync stores UTF-8-normalized filenames; match against the same form. + for i := range parent.Attachments { + parent.Attachments[i].Filename = textutil.EnsureUTF8(parent.Attachments[i].Filename) + } + maintenance := a.attachmentMaintenance + if maintenance != nil && maintenance.blob == nil { + maintenance = nil + } + return runWithAttachmentMutation(ctx, maintenance, func(ctx context.Context) error { + refs, err := a.store.MessageMIMEAttachmentsContext(ctx, target.parent.ID) + if err != nil { + return draftReplyError("attachment_preflight_failed", err) + } + if maintenance == nil && (len(refs) > 0 || len(parent.Attachments) > 0) { + return a.emitDraftForwardPreflight(emit, intent.JSON, []draftForwardProblem{{Reason: "attachment_reader_unavailable"}}) + } + attachments, problems := a.readForwardAttachments(ctx, parent, refs) + if len(problems) != 0 { + return a.emitDraftForwardPreflight(emit, intent.JSON, problems) + } + draft, err := imaplib.BuildForward(imaplib.ForwardOptions{ + From: from, To: intent.To, Cc: intent.Cc, Bcc: intent.Bcc, + Subject: parent.Subject, Body: intent.Body, QuotedHeader: forwardHeaderSummary(parent), + QuotedText: textutil.EnsureUTF8(parent.GetBodyText()), + Attachments: attachments, + }, time.Now(), "") + if err != nil { + return draftReplyError("invalid_forward_metadata", err) + } + writes, err := prepareIMAPDraftAttachmentWrites(ctx, draft.Parsed, refs) + if err != nil { + return a.emitDraftForwardPreflight(emit, intent.JSON, []draftForwardProblem{{ + Reason: "unrepresentable_attachment", Detail: err.Error(), + }}) + } + target.forward = true + target.attachmentWrites = &writes + return a.createDraft(ctx, target, draft, intent.JSON, emit) + }) +} + +func (a *storeAPIAdapter) readForwardAttachments( + ctx context.Context, + parsed *msgmime.Message, + refs []store.AttachmentRef, +) ([]imaplib.ForwardAttachment, []draftForwardProblem) { + attachments := make([]imaplib.ForwardAttachment, 0, len(parsed.Attachments)) + problems := make([]draftForwardProblem, 0) + used := make([]bool, len(refs)) + boundHashes := make(map[string]bool, len(refs)) + for _, part := range msgmime.DistinctAttachments(parsed.Attachments) { + attachment := imaplib.ForwardAttachment{ + Filename: part.Filename, ContentType: part.ContentType, ContentID: part.ContentID, + Disposition: part.Disposition, IsInline: part.IsInline, Content: part.Content, + } + index := matchForwardAttachmentRef(part, refs, used) + if part.Size != 0 && index >= 0 { + used[index] = true + boundHashes[strings.ToLower(refs[index].ContentHash)] = true + } + if err := imaplib.ValidateForwardAttachment(attachment); err != nil { + problems = append(problems, draftForwardProblem{ + Filename: part.Filename, PartKey: part.PartKey, Reason: "unrepresentable_attachment", Detail: err.Error(), + }) + continue + } + if part.Size == 0 { + // Sync stores no file or row for an empty part; forward it as is. + attachments = append(attachments, attachment) + continue + } + if index < 0 { + problems = append(problems, draftForwardProblem{Filename: part.Filename, PartKey: part.PartKey, Reason: "missing_catalog_reference"}) + continue + } + ref := refs[index] + if (ref.State != "" && ref.State != attachmentpolicy.StateStored) || + (ref.State == "" && ref.SkipReason != "") { + state := string(ref.State) + if state == "" { + state = string(attachmentpolicy.StateSkipped) + } + problem := draftForwardProblem{Filename: ref.Filename, PartKey: part.PartKey, Reason: "attachment_" + state} + if ref.SkipReason != "" { + problem.Detail = string(ref.SkipReason) + } + problems = append(problems, problem) + continue + } + reader, _, openErr := a.attachmentMaintenance.blob.OpenStream(ctx, ref.ContentHash) + if openErr != nil { + problems = append(problems, draftForwardProblem{Filename: ref.Filename, PartKey: part.PartKey, Reason: "unreadable_file", Detail: openErr.Error()}) + continue + } + // Consume through EOF to verify the blob without retaining a second copy. + size, readErr := io.Copy(io.Discard, reader) + closeErr := reader.Close() + if readErr != nil || closeErr != nil { + detail := "read attachment failed" + if readErr != nil { + detail = readErr.Error() + } else if closeErr != nil { + detail = closeErr.Error() + } + problems = append(problems, draftForwardProblem{Filename: ref.Filename, PartKey: part.PartKey, Reason: "unreadable_file", Detail: detail}) + continue + } + if size != int64(part.Size) { + problems = append(problems, draftForwardProblem{Filename: ref.Filename, PartKey: part.PartKey, Reason: "catalog_size_mismatch", Detail: fmt.Sprintf("parent part has %d bytes, read %d", part.Size, size)}) + continue + } + attachments = append(attachments, attachment) + } + for i, ref := range refs { + // A keyless legacy row shadowed by a bound copy of its bytes is not a separate occurrence. + if used[i] || (ref.SourcePartKey == "" && boundHashes[strings.ToLower(ref.ContentHash)]) { + continue + } + problems = append(problems, draftForwardProblem{Filename: ref.Filename, PartKey: ref.SourcePartKey, Reason: "unrepresented_attachment"}) + } + return attachments, problems +} + +func (a *storeAPIAdapter) emitDraftForwardPreflight( + emit func(api.CLIRunEvent) error, + asJSON bool, + problems []draftForwardProblem, +) error { + output := draftForwardPreflightOutput{Status: "attachment_preflight_failed", Problems: problems} + if emit != nil { + if asJSON { + data, err := json.Marshal(output, json.Deterministic(true)) + if err != nil { + return draftReplyError("output_failed", err) + } + if err := emit(api.CLIRunEvent{Type: cliStreamStderr, Data: string(data) + "\n"}); err != nil { + return draftReplyError("output_failed", err) + } + } else { + var text strings.Builder + text.WriteString("draft-forward refused before APPEND:\n") + for _, problem := range problems { + if problem.Filename != "" || problem.PartKey != "" { + text.WriteString("attachment") + } + if problem.Filename != "" { + fmt.Fprintf(&text, " %q", problem.Filename) + } + if problem.PartKey != "" { + fmt.Fprintf(&text, " (%s)", problem.PartKey) + } + if problem.Filename != "" || problem.PartKey != "" { + text.WriteString(": ") + } + text.WriteString(problem.Reason) + if problem.Detail != "" { + fmt.Fprintf(&text, " (%s)", problem.Detail) + } + text.WriteByte('\n') + } + if err := emit(api.CLIRunEvent{Type: cliStreamStderr, Data: text.String()}); err != nil { + return draftReplyError("output_failed", err) + } + } + } + return draftReplyError("attachment_preflight_failed", errors.New("one or more archived attachments are unavailable")) +} + +func forwardHeaderSummary(parent *msgmime.Message) string { + lines := make([]string, 0, 5) + add := func(name, value string) { + if value = strings.TrimSpace(textutil.EnsureUTF8(value)); value != "" { + lines = append(lines, name+": "+value) + } + } + add("From", forwardAddressList(parent.From)) + add("Date", parent.RawDateHeader) + add("Subject", parent.Subject) + add("To", forwardAddressList(parent.To)) + add("Cc", forwardAddressList(parent.Cc)) + return strings.Join(lines, "\r\n") +} + +func forwardAddressList(addresses []msgmime.Address) string { + values := make([]string, len(addresses)) + for i, address := range addresses { + // Summary is body text, so write names unencoded. + values[i] = address.Email + if address.Name != "" { + values[i] = address.Name + " <" + address.Email + ">" + } + } + return strings.Join(values, ", ") +} diff --git a/cmd/msgvault/cmd/serve_forward_draft_test.go b/cmd/msgvault/cmd/serve_forward_draft_test.go new file mode 100644 index 000000000..ebf80d050 --- /dev/null +++ b/cmd/msgvault/cmd/serve_forward_draft_test.go @@ -0,0 +1,581 @@ +package cmd + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "database/sql" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + emersionimap "github.com/emersion/go-imap/v2" + "github.com/emersion/go-imap/v2/imapclient" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/agentgrant" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/attachmentpolicy" + "go.kenn.io/msgvault/internal/config" + imaplib "go.kenn.io/msgvault/internal/imap" + msgmime "go.kenn.io/msgvault/internal/mime" + "go.kenn.io/msgvault/internal/query" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil" +) + +func TestDraftForwardAuthorizationPrecedesContentIO(t *testing.T) { + requirements := require.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + grant := &agentgrant.Grant{ + ID: "forward-create-grant", + Permissions: []agentgrant.Permission{agentgrant.PermissionDraftCreate}, + Sources: []agentgrant.SourceRef{{ + ID: fixture.source.ID, Type: fixture.source.SourceType, Identifier: fixture.source.Identifier, + SenderKeys: []string{testutil.IMAPTestUsername}, + }}, + } + err := adapter.runCLIForwardDraft(t.Context(), api.CLIRunRequest{ + Args: []string{ + "draft-forward", strconv.FormatInt(fixture.parentID, 10), + "--source-id", strconv.FormatInt(fixture.source.ID, 10), + "--from", testutil.IMAPTestUsername, "--to", "to@example.test", + }, Grant: grant, + }, nil) + requirements.Error(err) + requirements.Equal("not_permitted", err.Error()) + _, ok := errors.AsType[*api.CLIRunCodedError](err) + requirements.True(ok) +} + +func TestDraftForwardRejectsUnsupportedDestinationBeforeParentRead(t *testing.T) { + for _, scenario := range []struct { + name string + destination string + from string + policy bool + want string + }{ + {name: "Gmail destination", destination: "gmail", from: "gmail@example.test", policy: true, want: "draft_disabled"}, + {name: "disabled IMAP source", destination: "imap", from: testutil.IMAPTestUsername, policy: false, want: "draft_disabled"}, + {name: "disallowed sender", destination: "imap", from: "other@example.test", policy: true, want: "invalid_from"}, + } { + t.Run(scenario.name, func(t *testing.T) { + requirements := require.New(t) + fixture := newDraftReplyFixture(t) + adapter := fixture.grantedAdapter() + var sourceID int64 + if scenario.destination == "gmail" { + source, err := fixture.store.GetOrCreateSource("gmail", "gmail@example.test") + requirements.NoError(err) + requirements.NoError(fixture.store.AddAccountIdentity(source.ID, source.Identifier, "manual")) + sourceID = source.ID + } else { + sourceID = fixture.source.ID + adapter.draftPolicy = []config.IMAPDraftSource{{SourceID: sourceID, Enabled: scenario.policy, Mailbox: "Drafts"}} + } + err := adapter.runCLIForwardDraft(t.Context(), api.CLIRunRequest{Args: []string{ + "draft-forward", strconv.FormatInt(fixture.parentID, 10), + "--source-id", strconv.FormatInt(sourceID, 10), "--from", scenario.from, "--to", "recipient@example.test", + }}, nil) + requirements.Error(err) + requirements.Equal(scenario.want, err.Error()) + }) + } +} + +func TestDraftForwardHTTPPublishesManagedDraft(t *testing.T) { + assertions := assert.New(t) + requirements := require.New(t) + fixture := newDraftReplyFixture(t) + attachmentContent := []byte("provider copied attachment bytes") + parentRaw := []byte("From: Sender \r\n" + + "To: " + testutil.IMAPTestUsername + "\r\n" + + "Subject: Question\r\n" + + "Message-ID: \r\n" + + "MIME-Version: 1.0\r\n" + + "Content-Type: multipart/mixed; boundary=source-boundary\r\n\r\n" + + "--source-boundary\r\n" + + "Content-Type: text/plain; charset=utf-8\r\n\r\n" + + "Parent body\r\n" + + "--source-boundary\r\n" + + "Content-Type: text/plain; name=source.txt\r\n" + + "Content-Disposition: attachment; filename=source.txt\r\n" + + "Content-Transfer-Encoding: base64\r\n\r\n" + + base64.StdEncoding.EncodeToString(attachmentContent) + "\r\n" + + "--source-boundary--\r\n") + parsedParent, err := msgmime.Parse(parentRaw) + requirements.NoError(err) + requirements.Len(parsedParent.Attachments, 1) + requirements.NoError(fixture.store.UpsertMessageRaw(fixture.parentID, parentRaw)) + digest := sha256.Sum256(attachmentContent) + hash := hex.EncodeToString(digest[:]) + dataDir := t.TempDir() + attachmentDir := filepath.Join(dataDir, "attachments") + relativePath := filepath.Join(hash[:2], hash) + requirements.NoError(os.MkdirAll(filepath.Dir(filepath.Join(attachmentDir, relativePath)), 0o700)) + requirements.NoError(os.WriteFile(filepath.Join(attachmentDir, relativePath), attachmentContent, 0o600)) + role, roleSource := store.AttachmentRoleFromMIME( + parsedParent.Attachments[0].Disposition, parsedParent.Attachments[0].IsInline, parsedParent.Attachments[0].ContentID, + ) + requirements.NoError(fixture.store.UpsertAttachmentRecord(t.Context(), fixture.parentID, store.AttachmentWrite{ + Filename: parsedParent.Attachments[0].Filename, MIMEType: parsedParent.Attachments[0].ContentType, + StoragePath: filepath.ToSlash(relativePath), ContentHash: hash, Size: int64(len(attachmentContent)), + Role: role, RoleSource: roleSource, SourcePartKey: parsedParent.Attachments[0].PartKey, + State: attachmentpolicy.StateStored, + })) + remoteKey := "remote-image:" + hash + requirements.NoError(fixture.store.UpsertRemoteImageAttachment(t.Context(), fixture.parentID, store.AttachmentWrite{ + Filename: "cached-image.png", MIMEType: "image/png", StoragePath: filepath.ToSlash(relativePath), + ContentHash: hash, Size: int64(len(attachmentContent)), SourceAttachmentID: remoteKey, + SourcePartKey: remoteKey, ContentID: remoteKey, MediaType: "image", Role: store.AttachmentRoleInline, + RoleSource: store.AttachmentRoleSourceImporterSemantics, State: attachmentpolicy.StateStored, + })) + parentRefs, err := fixture.store.MessageMIMEAttachmentsContext(t.Context(), fixture.parentID) + requirements.NoError(err) + requirements.Len(parentRefs, 1) + maintenance, err := newAttachmentMaintenance(fixture.store, attachmentDir, slog.New(slog.DiscardHandler), true) + requirements.NoError(err) + t.Cleanup(func() { _ = maintenance.close() }) + adapter := fixture.grantedAdapter() + adapter.attachmentMaintenance = maintenance + server := httptest.NewServer(api.NewServerWithOptions(api.ServerOptions{ + Config: &config.Config{ + HomeDir: t.TempDir(), Data: config.DataConfig{DataDir: dataDir}, + Server: config.ServerConfig{APIKey: "owner-test-key"}, + }, + Store: adapter, Engine: query.NewEngine(fixture.store.DB(), fixture.store.IsPostgreSQL()), BlobStore: maintenance.blob, + Logger: slog.New(slog.DiscardHandler), + }).Router()) + t.Cleanup(server.Close) + + args := []string{ + "draft-forward", strconv.FormatInt(fixture.parentID, 10), + "--source-id", strconv.FormatInt(fixture.source.ID, 10), + "--from", testutil.IMAPTestUsername, "--to", "to@example.test", + "--cc", "copy@example.test", "--bcc", "hidden@example.test", + "--body", "forward note", "--json", + } + body, err := json.Marshal(map[string]any{"args": args}) + requirements.NoError(err) + request, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/cli/run", bytes.NewReader(body)) + requirements.NoError(err) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("X-Api-Key", "owner-test-key") + response, err := http.DefaultClient.Do(request) + requirements.NoError(err) + defer func() { _ = response.Body.Close() }() + requirements.Equal(http.StatusOK, response.StatusCode) + + var events []api.CLIRunEvent + scanner := bufio.NewScanner(response.Body) + for scanner.Scan() { + var event api.CLIRunEvent + requirements.NoError(json.Unmarshal(scanner.Bytes(), &event)) + events = append(events, event) + } + requirements.NoError(scanner.Err()) + requirements.Len(events, 2) + requirements.Equal(cliStreamStdout, events[0].Type) + requirements.Equal("complete", events[1].Type) + + var result draftReplyOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &result)) + assertions.Equal(draftReplyStatusCreated, result.Status) + assertions.Equal(int64(1), result.Revision) + + draft, err := fixture.store.GetIMAPDraft(result.DraftID) + requirements.NoError(err) + assertions.Equal(result.MessageID, draft.CurrentMessageID) + assertions.Equal(result.UID, draft.CurrentReceipt.UID) + message, err := fixture.store.GetMessage(result.MessageID) + requirements.NoError(err) + assertions.Equal([]string{"to@example.test"}, message.To) + assertions.Equal([]string{"copy@example.test"}, message.Cc) + assertions.Equal([]string{"hidden@example.test"}, message.Bcc) + assertions.Len(message.Attachments, 1) + assertions.Equal(hash, message.Attachments[0].ContentHash) + draftRefs, err := fixture.store.MessageMIMEAttachmentsContext(t.Context(), result.MessageID) + requirements.NoError(err) + requirements.Len(draftRefs, 1) + assertions.Empty(draftRefs[0].SourceAttachmentID) + storedRaw, err := fixture.store.GetMessageRaw(result.MessageID) + requirements.NoError(err) + assertions.Contains(string(storedRaw), "X-Msgvault-Forward: 1") + assertions.Contains(string(storedRaw), "forward note") + assertions.Contains(string(storedRaw), "Parent body") + assertions.Contains(string(storedRaw), "Bcc:") + assertions.Contains(*fixture.refreshed, fixture.source.Identifier) + + _, fetchedRaw := fetchDraftMailboxMessage(t, fixture.config, draft.CurrentReceipt) + assertions.Equal(storedRaw, fetchedRaw) + fetched, err := msgmime.Parse(fetchedRaw) + requirements.NoError(err) + requirements.Len(fetched.Attachments, 1) + assertions.Equal(attachmentContent, fetched.Attachments[0].Content) + + getBody, err := json.Marshal(map[string]any{"args": []string{"draft-get", result.DraftID, "--json"}}) + requirements.NoError(err) + getRequest, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/cli/run", bytes.NewReader(getBody)) + requirements.NoError(err) + getRequest.Header.Set("Content-Type", "application/json") + getRequest.Header.Set("X-Api-Key", "owner-test-key") + getResponse, err := http.DefaultClient.Do(getRequest) + requirements.NoError(err) + defer func() { _ = getResponse.Body.Close() }() + requirements.Equal(http.StatusOK, getResponse.StatusCode) + var getEvents []api.CLIRunEvent + getScanner := bufio.NewScanner(getResponse.Body) + for getScanner.Scan() { + var event api.CLIRunEvent + requirements.NoError(json.Unmarshal(getScanner.Bytes(), &event)) + getEvents = append(getEvents, event) + } + requirements.NoError(getScanner.Err()) + requirements.Len(getEvents, 2) + var localGet draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(getEvents[0].Data), &localGet)) + assertions.Equal(result.DraftID, localGet.DraftID) + assertions.Equal(string(storedRaw), localGet.RawMIME) + + editBody, err := json.Marshal(map[string]any{"args": []string{ + api.CLIRunDraftEditCommand, result.DraftID, "--revision", "1", "--body", "edited forward note", "--json", + }}) + requirements.NoError(err) + editRequest, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/cli/run", bytes.NewReader(editBody)) + requirements.NoError(err) + editRequest.Header.Set("Content-Type", "application/json") + editRequest.Header.Set("X-Api-Key", "owner-test-key") + editResponse, err := http.DefaultClient.Do(editRequest) + requirements.NoError(err) + defer func() { _ = editResponse.Body.Close() }() + requirements.Equal(http.StatusOK, editResponse.StatusCode) + var editEvents []api.CLIRunEvent + editScanner := bufio.NewScanner(editResponse.Body) + for editScanner.Scan() { + var event api.CLIRunEvent + requirements.NoError(json.Unmarshal(editScanner.Bytes(), &event)) + editEvents = append(editEvents, event) + } + requirements.NoError(editScanner.Err()) + requirements.Len(editEvents, 2) + var editOutput draftLifecycleOutput + requirements.NoError(json.Unmarshal([]byte(editEvents[0].Data), &editOutput)) + assertions.Equal(int64(2), editOutput.Revision) + editedDraft, err := fixture.store.GetIMAPDraftContext(t.Context(), result.DraftID) + requirements.NoError(err) + assertions.Equal(int64(2), editedDraft.Revision) + editedRaw, err := fixture.store.GetMessageRawContext(t.Context(), editedDraft.CurrentMessageID) + requirements.NoError(err) + edited, err := msgmime.Parse(editedRaw) + requirements.NoError(err) + assertions.Contains(edited.BodyText, "edited forward note") + requirements.Len(edited.Attachments, 1) + assertions.Equal(attachmentContent, edited.Attachments[0].Content) + editedRefs, err := fixture.store.MessageMIMEAttachmentsContext(t.Context(), editedDraft.CurrentMessageID) + requirements.NoError(err) + requirements.Len(editedRefs, 1) + assertions.Equal(hash, editedRefs[0].ContentHash) +} + +func TestDraftForwardRefusesCorruptStoredAttachmentBeforeAppend(t *testing.T) { + assertions := assert.New(t) + requirements := require.New(t) + fixture := newDraftReplyFixture(t) + content := []byte("archived attachment") + corruptContent := bytes.Repeat([]byte("x"), len(content)) + raw := []byte("From: Sender \r\n" + + "To: " + testutil.IMAPTestUsername + "\r\n" + + "Subject: Question\r\n" + + "MIME-Version: 1.0\r\n" + + "Content-Type: multipart/mixed; boundary=corrupt-boundary\r\n\r\n" + + "--corrupt-boundary\r\n" + + "Content-Type: text/plain; charset=utf-8\r\n\r\n" + + "Parent body\r\n" + + "--corrupt-boundary\r\n" + + "Content-Type: text/plain; name=report.txt\r\n" + + "Content-Disposition: attachment; filename=report.txt\r\n" + + "Content-Transfer-Encoding: base64\r\n\r\n" + + base64.StdEncoding.EncodeToString(content) + "\r\n" + + "--corrupt-boundary--\r\n") + parsed, err := msgmime.Parse(raw) + requirements.NoError(err) + requirements.Len(parsed.Attachments, 1) + requirements.NoError(fixture.store.UpsertMessageRaw(fixture.parentID, raw)) + digest := sha256.Sum256(content) + hash := hex.EncodeToString(digest[:]) + attachmentDir := t.TempDir() + relativePath := filepath.Join(hash[:2], hash) + requirements.NoError(os.MkdirAll(filepath.Dir(filepath.Join(attachmentDir, relativePath)), 0o700)) + requirements.NoError(os.WriteFile(filepath.Join(attachmentDir, relativePath), corruptContent, 0o600)) + role, roleSource := store.AttachmentRoleFromMIME( + parsed.Attachments[0].Disposition, parsed.Attachments[0].IsInline, parsed.Attachments[0].ContentID, + ) + requirements.NoError(fixture.store.UpsertAttachmentRecord(t.Context(), fixture.parentID, store.AttachmentWrite{ + Filename: parsed.Attachments[0].Filename, MIMEType: parsed.Attachments[0].ContentType, + StoragePath: filepath.ToSlash(relativePath), ContentHash: hash, Size: int64(len(content)), + Role: role, RoleSource: roleSource, SourcePartKey: parsed.Attachments[0].PartKey, + State: attachmentpolicy.StateStored, + })) + maintenance, err := newAttachmentMaintenance(fixture.store, attachmentDir, nil, true) + requirements.NoError(err) + t.Cleanup(func() { _ = maintenance.close() }) + adapter := fixture.grantedAdapter() + adapter.attachmentMaintenance = maintenance + var events []api.CLIRunEvent + err = adapter.runCLIForwardDraft(t.Context(), api.CLIRunRequest{Args: []string{ + "draft-forward", strconv.FormatInt(fixture.parentID, 10), + "--source-id", strconv.FormatInt(fixture.source.ID, 10), + "--from", testutil.IMAPTestUsername, "--to", "to@example.test", "--json", + }}, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + requirements.Error(err) + assertions.Equal("attachment_preflight_failed", err.Error()) + requirements.Len(events, 1) + assertions.Equal(cliStreamStderr, events[0].Type) + var output draftForwardPreflightOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + requirements.Len(output.Problems, 1) + assertions.Equal("report.txt", output.Problems[0].Filename) + assertions.Equal("unreadable_file", output.Problems[0].Reason) + assertions.NotEmpty(output.Problems[0].Detail) + var drafts int + requirements.NoError(fixture.store.DB().QueryRow(fixture.store.Rebind( + "SELECT COUNT(*) FROM imap_drafts", + )).Scan(&drafts)) + assertions.Zero(drafts) + client, err := imapclient.DialInsecure(fixture.config.Addr(), nil) + requirements.NoError(err) + t.Cleanup(func() { _ = client.Close() }) + requirements.NoError(client.Login(testutil.IMAPTestUsername, testutil.IMAPTestPassword).Wait()) + status, err := client.Status("Drafts", &emersionimap.StatusOptions{NumMessages: true}).Wait() + requirements.NoError(err) + requirements.NotNil(status.NumMessages) + assertions.Zero(*status.NumMessages) +} + +func TestDraftForwardLooseAndPackedAttachments(t *testing.T) { + assertions := assert.New(t) + requirements := require.New(t) + fixture := newAttachmentMaintenanceFixture(t) + content := []byte("same archived bytes") + firstHash := fixture.addLoose(content) + + draft, err := imaplib.BuildForward(imaplib.ForwardOptions{ + From: "sender@example.test", To: []string{"recipient@example.test"}, Subject: "Archived", + Attachments: []imaplib.ForwardAttachment{ + {Filename: "attachment-1.bin", ContentType: "application/octet-stream", Content: content}, + }, + }, time.Now(), "forward@example.test") + requirements.NoError(err) + refs, err := fixture.store.MessageMIMEAttachmentsContext(t.Context(), fixture.messageID) + requirements.NoError(err) + adapter := &storeAPIAdapter{attachmentMaintenance: fixture.maintenance} + read := func() { + attachments, problems := adapter.readForwardAttachments(t.Context(), draft.Parsed, refs) + requirements.Empty(problems) + requirements.Len(attachments, 1) + assertions.Equal(content, attachments[0].Content) + } + read() + _, err = fixture.maintenance.pack(t.Context(), 0) + requirements.NoError(err) + requirements.NotNil(fixture.packedEntry(firstHash)) + read() +} + +func TestDraftForwardAttachmentAvailability(t *testing.T) { + assertions := assert.New(t) + requirements := require.New(t) + content := []byte("archived state") + draft, err := imaplib.BuildForward(imaplib.ForwardOptions{ + From: "sender@example.test", To: []string{"recipient@example.test"}, Subject: "Archived", + Attachments: []imaplib.ForwardAttachment{ + {Filename: "pending.bin", ContentType: "application/octet-stream", Content: content}, + {Filename: "skipped.bin", ContentType: "application/octet-stream", Content: content}, + {Filename: "failed.bin", ContentType: "application/octet-stream", Content: content}, + }, + }, time.Now(), "forward@example.test") + requirements.NoError(err) + refs := make([]store.AttachmentRef, 0, len(draft.Parsed.Attachments)) + states := []attachmentpolicy.DownloadState{ + attachmentpolicy.StatePending, attachmentpolicy.StateSkipped, attachmentpolicy.StateFailed, + } + for i, part := range draft.Parsed.Attachments { + refs = append(refs, store.AttachmentRef{ + Filename: part.Filename, ContentHash: part.ContentHash, Size: part.Size, + State: states[i], SkipReason: attachmentpolicy.SkipReason("policy-test"), + }) + } + adapter := &storeAPIAdapter{} + attachments, problems := adapter.readForwardAttachments(t.Context(), draft.Parsed, refs) + requirements.Empty(attachments) + requirements.Len(problems, 3) + assertions.Equal([]string{"attachment_pending", "attachment_skipped", "attachment_failed"}, []string{ + problems[0].Reason, problems[1].Reason, problems[2].Reason, + }) + assertions.Equal("policy-test", problems[1].Detail) +} + +func TestDraftForwardReportsUnrepresentableAttachment(t *testing.T) { + assertions := assert.New(t) + requirements := require.New(t) + fixture := newAttachmentMaintenanceFixture(t) + content := []byte("Subject: Original\r\n\r\nzero\x00byte") + fixture.addLoose(content) + raw := []byte("From: sender@example.test\r\nMIME-Version: 1.0\r\n" + + "Content-Type: multipart/mixed; boundary=original\r\n\r\n" + + "--original\r\nContent-Type: text/plain\r\n\r\nbody\r\n" + + "--original\r\nContent-Type: message/rfc822\r\n" + + "Content-Disposition: attachment; filename=original.eml\r\n\r\n" + + string(content) + "\r\n--original--\r\n") + parent, err := msgmime.Parse(raw) + requirements.NoError(err) + refs, err := fixture.store.MessageMIMEAttachmentsContext(t.Context(), fixture.messageID) + requirements.NoError(err) + adapter := &storeAPIAdapter{attachmentMaintenance: fixture.maintenance} + attachments, problems := adapter.readForwardAttachments(t.Context(), parent, refs) + assertions.Empty(attachments) + requirements.Len(problems, 1) + assertions.Equal("original.eml", problems[0].Filename) + assertions.NotEmpty(problems[0].PartKey) + assertions.Equal("unrepresentable_attachment", problems[0].Reason) + assertions.Contains(problems[0].Detail, "binary transport") +} + +func TestDraftForwardPreflightWithoutAttachmentIdentity(t *testing.T) { + adapter := &storeAPIAdapter{} + var output strings.Builder + err := adapter.emitDraftForwardPreflight(func(event api.CLIRunEvent) error { + output.WriteString(event.Data) + return nil + }, false, []draftForwardProblem{{Reason: "attachment_reader_unavailable"}}) + require.ErrorContains(t, err, "attachment_preflight_failed") + assert.Equal(t, "draft-forward refused before APPEND:\nattachment_reader_unavailable\n", output.String()) +} + +func TestDraftForwardEditChecksAttachmentsBeforeConnecting(t *testing.T) { + assertions := assert.New(t) + requirements := require.New(t) + fixture, _ := newDraftRecoveryFixture(t) + forward, err := imaplib.BuildForward(imaplib.ForwardOptions{ + From: "sender@example.test", To: []string{"recipient@example.test"}, + Attachments: []imaplib.ForwardAttachment{{Filename: "missing.bin", ContentType: "application/octet-stream", Content: []byte("missing catalog row")}}, + }, time.Now(), "forward@example.test") + requirements.NoError(err) + requirements.NoError(fixture.store.UpsertMessageRaw(fixture.draft.CurrentMessageID, forward.Raw)) + connections := 0 + fixture.adapter.draftClientFactory = func(context.Context, *store.Source) (*imaplib.Client, error) { + connections++ + return nil, errors.New("provider offline") + } + _, err = runReviewLifecycle(t, fixture.adapter, "draft-edit", fixture.draft.DraftID, "--revision", "1", "--body", "updated") + requirements.ErrorContains(err, "invalid_draft") + assertions.Zero(connections) + current, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + assertions.Nil(current.Pending) +} + +func TestDraftForwardEditRecoveryRetainsAttachments(t *testing.T) { + assertions := assert.New(t) + requirements := require.New(t) + fixture, _ := newDraftRecoveryFixture(t) + content := []byte("recovered attachment") + digest := sha256.Sum256(content) + hash := hex.EncodeToString(digest[:]) + requirements.NoError(fixture.store.UpsertAttachmentRecord(t.Context(), fixture.draft.CurrentMessageID, store.AttachmentWrite{ + Filename: "recovered.txt", MIMEType: "text/plain", ContentHash: hash, Size: int64(len(content)), ContentID: "recovered@example.com", + Role: store.AttachmentRoleInline, RoleSource: store.AttachmentRoleSourceMIMEDisposition, + SourcePartKey: "mime:original", State: attachmentpolicy.StateStored, + })) + const note = "distinctive recovered note" + forward, err := imaplib.BuildForward(imaplib.ForwardOptions{ + From: "alice@example.com", To: []string{"bob@example.com"}, Subject: "Original", Body: note, + QuotedHeader: "From: Alice \r\nSubject: Original", + QuotedText: "distinctive recovered quote", + Attachments: []imaplib.ForwardAttachment{{ + Filename: "recovered.txt", ContentType: "text/plain", ContentID: "recovered@example.com", IsInline: true, Content: content, + }}, + }, time.Now(), "forward@example.com") + requirements.NoError(err) + _, err = fixture.store.ClaimIMAPDraftContext(t.Context(), fixture.draft.DraftID, fixture.draft.Revision, store.IMAPDraftOperationEdit, forward.Raw) + requirements.NoError(err) + replacement := store.IMAPDraftReceipt{SourceID: fixture.source.ID, Mailbox: "Drafts", UIDValidity: 1, UID: 2} + requirements.NoError(fixture.store.RecordIMAPDraftOutcomeContext(t.Context(), fixture.draft.DraftID, fixture.draft.Revision, "append_uidplus", &replacement)) + pending, err := fixture.store.GetIMAPDraftContext(t.Context(), fixture.draft.DraftID) + requirements.NoError(err) + published, err := fixture.adapter.publishRecoveredDraftReplacement(t.Context(), pending) + requirements.NoError(err) + refs, err := fixture.store.MessageMIMEAttachmentsContext(t.Context(), published.CurrentMessageID) + requirements.NoError(err) + requirements.Len(refs, 1) + assertions.Equal(hash, refs[0].ContentHash) + assertions.Equal("recovered.txt", refs[0].Filename) + assertions.Equal("recovered@example.com", refs[0].ContentID) + recoveredRaw, err := fixture.store.GetMessageRawContext(t.Context(), published.CurrentMessageID) + requirements.NoError(err) + assertions.Equal(forward.Raw, recoveredRaw) + recovered, err := msgmime.Parse(recoveredRaw) + requirements.NoError(err) + assertions.Equal(1, strings.Count(recovered.BodyText, note)) + assertions.Contains(recovered.BodyText, "distinctive recovered quote") +} + +func TestDraftForwardAttachmentsSurviveParentRemoval(t *testing.T) { + assertions := assert.New(t) + requirements := require.New(t) + fixture := newAttachmentMaintenanceFixture(t) + content := []byte("retained draft bytes") + hash := fixture.addLoose(content) + source, err := fixture.store.GetOrCreateSource("imap", "retained-draft@example.test") + requirements.NoError(err) + conversationID, err := fixture.store.EnsureConversation(source.ID, "retained-draft", "Retained draft") + requirements.NoError(err) + receipt := store.IMAPDraftReceipt{SourceID: source.ID, Mailbox: "Drafts", UIDValidity: 1, UID: 1} + draft, err := fixture.store.PersistIMAPDraftContext(t.Context(), receipt, nil, func(_ []int64) *store.MessagePersistData { + return &store.MessagePersistData{ + Message: &store.Message{ + SourceID: source.ID, SourceMessageID: store.IMAPDraftSourceMessageID(receipt), + MessageType: store.MessageTypeEmail, ConversationID: conversationID, + }, + BodyText: sql.NullString{String: "forward", Valid: true}, + RawMIME: []byte("From: test@example.com\r\nTo: recipient@example.com\r\n\r\nforward\r\n"), + MIMEAttachmentReplacement: &[]store.AttachmentWrite{{ + Filename: "retained.txt", MIMEType: "text/plain", StoragePath: hash[:2] + "/" + hash, + ContentHash: hash, Size: int64(len(content)), Role: store.AttachmentRoleStandalone, + RoleSource: store.AttachmentRoleSourceMIMEDisposition, SourcePartKey: "mime:retained", + State: attachmentpolicy.StateStored, + }}, + } + }) + requirements.NoError(err) + requirements.NotEmpty(draft.DraftID) + _, err = fixture.maintenance.pack(t.Context(), 0) + requirements.NoError(err) + _, err = fixture.store.DB().Exec(fixture.store.Rebind("DELETE FROM messages WHERE id = ?"), fixture.messageID) + requirements.NoError(err) + _, err = fixture.maintenance.repack(t.Context(), 0) + requirements.NoError(err) + reader, _, err := fixture.maintenance.blob.OpenStream(t.Context(), hash) + requirements.NoError(err) + got, err := io.ReadAll(reader) + requirements.NoError(err) + requirements.NoError(reader.Close()) + assertions.Equal(content, got) +} diff --git a/cmd/msgvault/cmd/serve_gmail_draft.go b/cmd/msgvault/cmd/serve_gmail_draft.go new file mode 100644 index 000000000..aad9c7ce6 --- /dev/null +++ b/cmd/msgvault/cmd/serve_gmail_draft.go @@ -0,0 +1,1262 @@ +package cmd + +import ( + "context" + "database/sql" + jsonv2 "encoding/json/v2" + "errors" + "fmt" + "strings" + "time" + "unicode/utf8" + + "go.kenn.io/msgvault/internal/agentgrant" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/gmail" + imaplib "go.kenn.io/msgvault/internal/imap" + msgmime "go.kenn.io/msgvault/internal/mime" + "go.kenn.io/msgvault/internal/oauth" + "go.kenn.io/msgvault/internal/sourceops" + "go.kenn.io/msgvault/internal/store" + msgsync "go.kenn.io/msgvault/internal/sync" + "go.kenn.io/msgvault/internal/textutil" +) + +const ( + gmailDraftStatusCreated = "created" + gmailDraftStatusPending = "pending" + gmailDraftStatusLocalFailed = "remote_accepted_local_failed" + gmailDraftDeleteOutcome = "deleted" + gmailDraftAbsentOutcome = "already_absent" +) + +// authorizeGmailDraft applies the daemon-start policy snapshot. Reads from +// the archive and owner-only send-as listing do not use this write grant. +func authorizeGmailDraft(policy []config.GmailDraftSource, sourceID int64, sourceType string) error { + return authorizeDraftPolicy("gmail", policy, sourceID, sourceType) +} + +// authorizeDraftPolicy checks a [[{provider}.drafts]] opt-in for the source. +func authorizeDraftPolicy(provider string, policy []config.GmailDraftSource, sourceID int64, sourceType string) error { + if sourceType != provider { + return draftReplyError("draft_disabled", fmt.Errorf("source %d is a %q source, not %s", sourceID, sourceType, provider)) + } + for _, grant := range policy { + if grant.SourceID == sourceID && grant.Enabled { + return nil + } + } + return draftReplyError("draft_disabled", fmt.Errorf("source %d has no enabled [[%s.drafts]] grant", sourceID, provider)) +} + +func gmailDraftScopeGate(ctx context.Context, cfg *config.Config, source *store.Source, accepted []string) error { + if source == nil { + return draftReplyError("invalid_source", errors.New("missing Gmail source")) + } + if cfg != nil && cfg.OAuth.ServiceAccountKeyFor(sourceOAuthApp(source)) != "" { + return nil + } + manager, err := oauthManagerCache(invocationFromContext(ctx))(sourceOAuthApp(source)) + if err != nil { + return draftReplyError("invalid_source", err) + } + if !manager.HasScopeMetadata(source.Identifier) { + return nil + } + if oauth.GrantCoversAnyScope(manager.GrantedScopes(source.Identifier), accepted) { + return nil + } + return draftReplyError("insufficient_scope", fmt.Errorf( + "saved Gmail grant for %s does not cover this operation; re-authorize with add-account %s --force", + source.Identifier, source.Identifier, + )) +} + +func defaultGmailDraftClientFactory(ctx context.Context, source *store.Source) (gmail.DraftAPI, error) { + client, err := buildAPIClient(ctx, source, oauthManagerCache(invocationFromContext(ctx)), nil) + if err != nil { + return nil, err + } + draftClient, ok := client.(gmail.DraftAPI) + if !ok { + _ = client.Close() + return nil, fmt.Errorf("source %d did not return a Gmail draft client", source.ID) + } + return draftClient, nil +} + +func validateGmailSendAs(entries []gmail.SendAs, from string) error { + for _, entry := range entries { + if store.EqualIdentifier(entry.Email, from) && + (entry.Primary || strings.EqualFold(entry.VerificationStatus, "accepted")) { + return nil + } + } + return draftReplyError("invalid_from", errors.New("--from is not a primary or accepted Gmail send-as identity")) +} + +type gmailDraftReplyOutput struct { + Status string `json:"status"` + DraftID string `json:"draft_id,omitempty"` + Revision int64 `json:"revision,omitzero"` + MessageID int64 `json:"message_id,omitzero"` + OperationRef string `json:"operation_ref"` + RFC822MessageID string `json:"rfc822_message_id"` + SourceID int64 `json:"source_id"` + GmailDraftID string `json:"gmail_draft_id,omitempty"` + GmailMessageID string `json:"gmail_message_id,omitempty"` + ThreadID string `json:"thread_id,omitempty"` +} + +func (a *storeAPIAdapter) runGmailReplyDraft( + ctx context.Context, + intent draftReplyIntent, + target draftReplyTarget, + reply imaplib.ReplyDraft, + messageIDValue string, + emit func(api.CLIRunEvent) error, +) error { + logger := a.logger + if logger == nil { + logger = loggerFromContext(ctx) + } + if err := gmailDraftScopeGate(ctx, a.config, target.source, oauth.ScopesGmailDraftWrite); err != nil { + return err + } + if err := gmailDraftScopeGate(ctx, a.config, target.source, oauth.ScopesGmailSendAsList); err != nil { + return err + } + execution, err := a.store.AcquireSyncExecutionContext(ctx, target.source.ID) + if err != nil { + if errors.Is(err, store.ErrSyncAlreadyActive) { + return draftReplyError("sync_active", fmt.Errorf("source %d: %w", target.source.ID, err)) + } + return draftReplyError("sync_lock_failed", fmt.Errorf("source %d: %w", target.source.ID, err)) + } + released := false + defer func() { + if !released { + _ = execution.Release() + } + }() + finish := func(refresh bool) { + if released { + return + } + released = true + if err := execution.Release(); err != nil { + logger.Error("release source after Gmail draft write", "source_id", target.source.ID, "error", err) + } + if refresh { + evidenceCtx, cancelEvidence := localDraftEvidenceContext(ctx) + defer cancelEvidence() + a.refreshDraftCache(evidenceCtx, target.source) + } + } + + clientFactory := a.gmailDraftClientFactory + if clientFactory == nil { + clientFactory = defaultGmailDraftClientFactory + } + client, err := clientFactory(ctx, target.source) + if err != nil { + return draftReplyError("invalid_source", fmt.Errorf("build Gmail client for source %d: %w", target.source.ID, err)) + } + defer func() { _ = client.Close() }() + sendAs, err := client.ListSendAs(ctx) + if err != nil { + return draftReplyError(gmailReadErrorCode(err), err) + } + if err := validateGmailSendAs(sendAs, reply.Parsed.From[0].Email); err != nil { + return err + } + draft, err := client.CreateDraft(ctx, reply.Raw, target.parent.SourceConversationID) + if err != nil { + return emitGmailDraftReplyFailure(emit, intent.JSON, target, messageIDValue, err) + } + if draft == nil { + return emitGmailDraftReplyFailure(emit, intent.JSON, target, messageIDValue, + &gmail.DraftWriteError{State: gmail.DraftStateRemoteUnknown, Code: "remote_unknown", Err: errors.New("gmail create returned no draft")}) + } + receipt := store.GmailDraftReceipt{ + SourceID: target.source.ID, GmailDraftID: draft.ID, + GmailMessageID: draft.Message.ID, ThreadID: draft.Message.ThreadID, + } + result := gmailDraftReplyOutput{ + Status: gmailDraftStatusCreated, + OperationRef: gmailDraftOperationRef(receipt), + RFC822MessageID: messageIDValue, SourceID: target.source.ID, + GmailDraftID: receipt.GmailDraftID, GmailMessageID: receipt.GmailMessageID, + ThreadID: receipt.ThreadID, + } + evidenceCtx, cancel := localDraftEvidenceContext(ctx) + defer cancel() + localDraft, err := a.store.PersistGmailDraftContext( + evidenceCtx, receipt, gmailDraftParticipants(reply.Parsed), + gmailDraftReplyPersistData(target, reply, receipt, messageIDValue), + ) + if err != nil { + result.Status = gmailDraftStatusLocalFailed + _ = emitGmailDraftReplyOutput(emit, cliStreamStderr, intent.JSON, result) + finish(true) + return draftReplyError(gmailDraftStatusLocalFailed, err) + } + result.DraftID = localDraft.DraftID + result.Revision = localDraft.Revision + result.MessageID = localDraft.CurrentMessageID + outputErr := emitGmailDraftReplyOutput(emit, cliStreamStdout, intent.JSON, result) + finish(true) + if outputErr != nil { + return draftReplyError("output_failed", outputErr) + } + return nil +} + +func emitGmailDraftReplyFailure( + emit func(api.CLIRunEvent) error, + asJSON bool, + target draftReplyTarget, + rfc822 string, + err error, +) error { + code := gmailWriteErrorCode(err) + result := gmailDraftReplyOutput{ + Status: code, OperationRef: fmt.Sprintf("%d:gmail:unknown", target.source.ID), + RFC822MessageID: rfc822, SourceID: target.source.ID, + } + _ = emitGmailDraftReplyOutput(emit, cliStreamStderr, asJSON, result) + return draftReplyError(code, gmailWriteCause(err)) +} + +func emitGmailDraftReplyOutput( + emit func(api.CLIRunEvent) error, + stream string, + asJSON bool, + output gmailDraftReplyOutput, +) error { + if emit == nil { + return nil + } + if asJSON { + data, err := jsonv2.Marshal(output) + if err != nil { + return err + } + return emit(api.CLIRunEvent{Type: stream, Data: string(data) + "\n"}) + } + if output.Status == gmailDraftStatusCreated { + return emit(api.CLIRunEvent{Type: stream, Data: fmt.Sprintf( + "created Gmail draft message %d (%s), operation %s, draft %s revision %d\n", + output.MessageID, textutil.SanitizeTerminal(output.GmailMessageID), + textutil.SanitizeTerminal(output.OperationRef), textutil.SanitizeTerminal(output.DraftID), output.Revision, + )}) + } + if output.RFC822MessageID != "" { + if err := emit(api.CLIRunEvent{Type: stream, Data: fmt.Sprintf( + "Gmail draft operation %s, RFC822 Message-ID: %s, inspect operation %s\n", + textutil.SanitizeTerminal(output.Status), + textutil.SanitizeTerminal(output.RFC822MessageID), + textutil.SanitizeTerminal(output.OperationRef), + )}); err != nil { + return err + } + return nil + } + return emit(api.CLIRunEvent{Type: stream, Data: fmt.Sprintf( + "Gmail draft operation %s, inspect operation %s\n", + textutil.SanitizeTerminal(output.Status), textutil.SanitizeTerminal(output.OperationRef), + )}) +} + +func gmailWriteErrorCode(err error) string { + var writeErr *gmail.DraftWriteError + if errors.As(err, &writeErr) && writeErr.Code != "" { + return writeErr.Code + } + return "provider_rejected" +} + +func gmailWriteCause(err error) error { + var writeErr *gmail.DraftWriteError + if errors.As(err, &writeErr) && writeErr.Err != nil { + return writeErr.Err + } + return err +} + +func gmailReadErrorCode(err error) string { + if _, ok := errors.AsType[*gmail.NotFoundError](err); ok { + return "provider_absent" + } + if gmail.IsInsufficientScopeError(err.Error()) { + return "insufficient_scope" + } + return "provider_refused" +} + +func gmailDraftOperationRef(receipt store.GmailDraftReceipt) string { + return fmt.Sprintf("%d:gmail:%s", receipt.SourceID, receipt.GmailDraftID) +} + +func gmailDraftParticipants(parsed *msgmime.Message) []store.ParticipantPersistData { + addresses := append([]msgmime.Address(nil), parsed.From...) + addresses = append(addresses, parsed.To...) + addresses = append(addresses, parsed.Cc...) + addresses = append(addresses, parsed.Bcc...) + participants := make([]store.ParticipantPersistData, len(addresses)) + for i, address := range addresses { + participants[i] = store.ParticipantPersistData{ + EmailAddress: address.Email, DisplayName: address.Name, Domain: address.Domain, + } + } + return participants +} + +func gmailDraftReplyPersistData( + target draftReplyTarget, + reply imaplib.ReplyDraft, + receipt store.GmailDraftReceipt, + messageIDValue string, +) func([]int64) *store.MessagePersistData { + return gmailDraftMessagePersistData(target.source.ID, target.parent.ID, reply.Parsed, reply.Raw, receipt, messageIDValue) +} + +func gmailDraftMessagePersistData( + sourceID int64, + replyToMessageID int64, + parsed *msgmime.Message, + raw []byte, + receipt store.GmailDraftReceipt, + rfc822 string, +) func([]int64) *store.MessagePersistData { + return gmailDraftMessagePersistDataWithAttachments( + sourceID, replyToMessageID, parsed, raw, receipt, rfc822, nil, + ) +} + +func gmailDraftMessagePersistDataWithAttachments( + sourceID int64, + replyToMessageID int64, + parsed *msgmime.Message, + raw []byte, + receipt store.GmailDraftReceipt, + rfc822 string, + attachmentWrites *[]store.AttachmentWrite, +) func([]int64) *store.MessagePersistData { + return func(ids []int64) *store.MessagePersistData { + fromCount := len(parsed.From) + if fromCount == 0 { + return nil + } + at := 0 + fromIDs := ids[at : at+fromCount] + at += fromCount + toIDs := ids[at : at+len(parsed.To)] + at += len(parsed.To) + ccIDs := ids[at : at+len(parsed.Cc)] + at += len(parsed.Cc) + bccIDs := ids[at : at+len(parsed.Bcc)] + toAddresses := gmailAddressStrings(parsed.To) + ccAddresses := gmailAddressStrings(parsed.Cc) + bccAddresses := gmailAddressStrings(parsed.Bcc) + fromAddresses := gmailAddressStrings(parsed.From) + return &store.MessagePersistData{ + Message: &store.Message{ + SourceID: sourceID, SourceMessageID: receipt.GmailMessageID, + RFC822MessageID: sql.NullString{String: rfc822, Valid: rfc822 != ""}, + MessageType: store.MessageTypeEmail, IsFromMe: true, IdentityDerivedIsFromMe: true, + SenderID: sql.NullInt64{Int64: fromIDs[0], Valid: true}, + ReplyToMessageID: sql.NullInt64{Int64: replyToMessageID, Valid: replyToMessageID > 0}, + Subject: sql.NullString{String: parsed.Subject, Valid: parsed.Subject != ""}, + Snippet: sql.NullString{String: strings.TrimSpace(parsed.BodyText), Valid: parsed.BodyText != ""}, + SentAt: sql.NullTime{Time: parsed.Date, Valid: !parsed.Date.IsZero()}, + InternalDate: sql.NullTime{Time: parsed.Date, Valid: !parsed.Date.IsZero()}, + SizeEstimate: int64(len(raw)), ArchivedAt: time.Now(), + }, + Conversation: &store.ConversationPersistData{ + SourceConversationID: receipt.ThreadID, + ConversationType: "email_thread", Title: parsed.Subject, + }, + BodyText: sql.NullString{String: parsed.BodyText, Valid: true}, + BodyHTML: sql.NullString{String: parsed.BodyHTML, Valid: parsed.BodyHTML != ""}, + RawMIME: raw, RawFormat: "mime", + Recipients: []store.RecipientSet{ + {Type: "from", ParticipantIDs: fromIDs, EmailAddresses: fromAddresses}, + {Type: "to", ParticipantIDs: toIDs, EmailAddresses: toAddresses}, + {Type: "cc", ParticipantIDs: ccIDs, EmailAddresses: ccAddresses}, + {Type: "bcc", ParticipantIDs: bccIDs, EmailAddresses: bccAddresses}, + }, + LabelRefs: []store.MessageLabelRef{{SourceLabelID: "DRAFT", Info: store.LabelInfo{Name: "DRAFT", Type: "system"}}}, + MIMEAttachmentReplacement: attachmentWrites, + FTS: &store.FTSDoc{ + Subject: parsed.Subject, Body: parsed.BodyText, FromAddr: firstGmailAddress(parsed.From), + ToAddrs: strings.Join(toAddresses, " "), CcAddrs: strings.Join(ccAddresses, " "), + }, + } + } +} + +func gmailDraftAttachmentWrites(cfg *config.Config, attachments []msgmime.Attachment) ([]store.AttachmentWrite, error) { + attachmentsDir := "" + if cfg != nil { + attachmentsDir = cfg.AttachmentsDir() + } + writes := make([]store.AttachmentWrite, 0, len(attachments)) + for i := range attachments { + write, err := msgsync.StoreMIMEAttachment(attachmentsDir, &attachments[i]) + if err != nil { + return nil, fmt.Errorf("store Gmail draft attachment %q: %w", attachments[i].Filename, err) + } + if write.StoragePath != "" { + writes = append(writes, write) + } + } + return writes, nil +} + +func gmailAddressStrings(addresses []msgmime.Address) []string { + result := make([]string, len(addresses)) + for i, address := range addresses { + result[i] = address.Email + } + return result +} + +func firstGmailAddress(addresses []msgmime.Address) string { + if len(addresses) == 0 { + return "" + } + return addresses[0].Email +} + +type gmailDraftLifecycleReceipt struct { + GmailDraftID string `json:"gmail_draft_id"` + GmailMessageID string `json:"gmail_message_id"` + ThreadID string `json:"thread_id"` +} + +type gmailDraftLifecycleObservation struct { + State string `json:"state"` + Code string `json:"code,omitempty"` + GmailDraftID string `json:"gmail_draft_id,omitempty"` + GmailMessageID string `json:"gmail_message_id,omitempty"` + ThreadID string `json:"thread_id,omitempty"` + Present bool `json:"present"` +} + +type gmailDraftLifecycleOutput struct { + Status string `json:"status"` + Provider string `json:"provider"` + DraftID string `json:"draft_id"` + Revision int64 `json:"revision"` + Lifecycle string `json:"lifecycle"` + MessageID int64 `json:"message_id"` + SourceID int64 `json:"source_id"` + Receipt gmailDraftLifecycleReceipt `json:"receipt"` + Content string `json:"content,omitempty"` + RawMIME string `json:"raw_mime,omitempty"` + CandidateContent string `json:"candidate_content,omitempty"` + PendingOperation string `json:"pending_operation,omitempty"` + PendingCode string `json:"pending_code,omitempty"` + PendingReplacementGmailMessageID string `json:"pending_replacement_gmail_message_id,omitempty"` + ProviderObservation *gmailDraftLifecycleObservation `json:"provider_observation,omitempty"` + Observation *gmailDraftLifecycleObservation `json:"observation,omitempty"` +} + +func (a *storeAPIAdapter) gmailDraftLifecycleOutput( + ctx context.Context, + draft store.GmailDraft, + status string, + providerObservation *gmailDraftLifecycleObservation, + observation *gmailDraftLifecycleObservation, +) (gmailDraftLifecycleOutput, error) { + body, raw, err := a.store.GetMessageBodyAndRawContext(ctx, draft.CurrentMessageID) + if err != nil { + return gmailDraftLifecycleOutput{}, fmt.Errorf("load managed Gmail draft MIME: %w", err) + } + output := gmailDraftLifecycleOutputWithoutMessage( + draft, status, providerObservation, observation, + ) + output.Content = body + output.RawMIME = string(raw) + return output, nil +} + +func gmailDraftLifecycleOutputWithoutMessage( + draft store.GmailDraft, + status string, + providerObservation *gmailDraftLifecycleObservation, + observation *gmailDraftLifecycleObservation, +) gmailDraftLifecycleOutput { + lifecycle := draftLifecycleActive + if draft.DiscardedAt != nil { + lifecycle = "discarded" + } + output := gmailDraftLifecycleOutput{ + Status: status, Provider: "gmail", DraftID: draft.DraftID, + Revision: draft.Revision, Lifecycle: lifecycle, MessageID: draft.CurrentMessageID, + SourceID: draft.SourceID, + Receipt: gmailDraftLifecycleReceipt{ + GmailDraftID: draft.CurrentReceipt.GmailDraftID, + GmailMessageID: draft.CurrentReceipt.GmailMessageID, + ThreadID: draft.CurrentReceipt.ThreadID, + }, + ProviderObservation: providerObservation, Observation: observation, + } + if draft.Pending != nil { + output.PendingOperation = draft.Pending.Operation + output.PendingCode = draft.Pending.Code + output.CandidateContent = string(draft.Pending.Raw) + output.PendingReplacementGmailMessageID = draft.Pending.ReplacementGmailMessageID + } + return output +} + +func (a *storeAPIAdapter) reportGmailDraftAcceptedLocalFailure( + emit func(api.CLIRunEvent) error, + intent draftLifecycleIntent, + claimed store.GmailDraft, + replacement store.GmailDraftReceipt, + replacementRaw []byte, + cause error, +) error { + output := gmailDraftLifecycleOutputWithoutMessage( + claimed, + "accepted_local_failed", + &gmailDraftLifecycleObservation{ + State: "present", + Code: "accepted_local_failed", + GmailDraftID: replacement.GmailDraftID, + GmailMessageID: replacement.GmailMessageID, + ThreadID: replacement.ThreadID, + Present: true, + }, + nil, + ) + output.PendingOperation = store.GmailDraftOperationEdit + output.PendingCode = "accepted_local_failed" + output.CandidateContent = string(replacementRaw) + output.PendingReplacementGmailMessageID = replacement.GmailMessageID + if claimed.Pending != nil { + output.PendingOperation = claimed.Pending.Operation + if len(claimed.Pending.Raw) != 0 { + output.CandidateContent = string(claimed.Pending.Raw) + } + } + if err := emitGmailDraftLifecycleOutput(emit, cliStreamStderr, intent, output); err != nil { + return draftReplyError("accepted_local_failed", errors.Join(cause, err)) + } + return draftReplyError("accepted_local_failed", cause) +} + +func emitGmailDraftLifecycleOutput( + emit func(api.CLIRunEvent) error, + stream string, + intent draftLifecycleIntent, + output gmailDraftLifecycleOutput, +) error { + if emit == nil { + return nil + } + if intent.MetadataOnly { + output.Content = "" + output.RawMIME = "" + output.CandidateContent = "" + } + if intent.JSON { + data, err := jsonv2.Marshal(output) + if err != nil { + return err + } + return emit(api.CLIRunEvent{Type: stream, Data: string(data) + "\n"}) + } + var data strings.Builder + fmt.Fprintf(&data, "draft %s revision %d %s\n", + textutil.SanitizeTerminal(output.DraftID), output.Revision, + textutil.SanitizeTerminal(output.Lifecycle)) + fmt.Fprintf(&data, "status: %s\n", textutil.SanitizeTerminal(output.Status)) + fmt.Fprintf(&data, "receipt (revision %d): %s\n", output.Revision, + textutil.SanitizeTerminal(formatGmailDraftLifecycleReceipt(output.Receipt))) + fmt.Fprintf(&data, "content:\n%s\n", + strings.TrimRight(textutil.SanitizeTerminalMultiline(output.Content), "\n")) + if output.PendingOperation != "" { + fmt.Fprintf(&data, "pending operation: %s\n", textutil.SanitizeTerminal(output.PendingOperation)) + } + if output.CandidateContent != "" { + fmt.Fprintf(&data, "candidate content:\n%s\n", + strings.TrimRight(textutil.SanitizeTerminalMultiline(output.CandidateContent), "\n")) + } + if output.PendingReplacementGmailMessageID != "" { + fmt.Fprintf(&data, "pending replacement Gmail message ID: %s\n", + textutil.SanitizeTerminal(output.PendingReplacementGmailMessageID)) + } + if output.Status == "accepted_local_failed" && output.ProviderObservation != nil && + output.ProviderObservation.State == "present" && output.ProviderObservation.Present { + fmt.Fprintf(&data, "acknowledged replacement receipt: %s\n", + textutil.SanitizeTerminal(formatGmailDraftLifecycleObservation(*output.ProviderObservation))) + } + if output.Observation != nil && output.Status == gmailDraftStatusPending { + fmt.Fprintf(&data, "old provider receipt: %s\n", + textutil.SanitizeTerminal(formatGmailDraftLifecycleObservation(*output.Observation))) + } + providerOutcome := output.PendingCode + if providerOutcome == "" { + observations := []*gmailDraftLifecycleObservation{output.ProviderObservation, output.Observation} + if output.Status == gmailDraftStatusPending { + observations = []*gmailDraftLifecycleObservation{output.Observation, output.ProviderObservation} + } + for _, observation := range observations { + if observation == nil { + continue + } + providerOutcome = observation.Code + if providerOutcome == "" { + providerOutcome = observation.State + } + if providerOutcome != "" { + break + } + } + } + if providerOutcome != "" { + fmt.Fprintf(&data, "provider outcome: %s\n", textutil.SanitizeTerminal(providerOutcome)) + } + if output.Status == gmailDraftStatusPending || output.Status == "accepted_local_failed" { + fmt.Fprintf(&data, "retry draft-%s with --revision %d to reconcile with Gmail\n", output.PendingOperation, output.Revision) + } + return emit(api.CLIRunEvent{Type: stream, Data: data.String()}) +} + +func formatGmailDraftLifecycleReceipt(receipt gmailDraftLifecycleReceipt) string { + return fmt.Sprintf("gmail_draft_id=%s gmail_message_id=%s thread_id=%s", + receipt.GmailDraftID, receipt.GmailMessageID, receipt.ThreadID) +} + +func formatGmailDraftLifecycleObservation(observation gmailDraftLifecycleObservation) string { + return fmt.Sprintf("state=%s code=%s gmail_draft_id=%s gmail_message_id=%s thread_id=%s present=%t", + observation.State, observation.Code, observation.GmailDraftID, + observation.GmailMessageID, observation.ThreadID, observation.Present) +} + +func (a *storeAPIAdapter) loadManagedGmailDraftSource(ctx context.Context, draft store.GmailDraft) (*store.Source, error) { + source, err := a.store.GetSourceByIDContext(ctx, draft.SourceID) + if err != nil { + return nil, draftReplyError("invalid_source", err) + } + if err := authorizeGmailDraft(a.gmailDraftPolicy, source.ID, source.SourceType); err != nil { + return nil, err + } + return source, nil +} + +func gmailDraftDeleteOutcomeIsConfirmed(code string) bool { + return code == gmailDraftDeleteOutcome || code == gmailDraftAbsentOutcome +} + +func gmailDraftDeleteObservation(draft store.GmailDraft, code string) *gmailDraftLifecycleObservation { + return &gmailDraftLifecycleObservation{ + State: "absent", + Code: code, + GmailDraftID: draft.CurrentReceipt.GmailDraftID, + GmailMessageID: draft.CurrentReceipt.GmailMessageID, + ThreadID: draft.CurrentReceipt.ThreadID, + Present: false, + } +} + +func (a *storeAPIAdapter) emitGmailDraftPending( + ctx context.Context, + intent draftLifecycleIntent, + claimed store.GmailDraft, + providerObservation *gmailDraftLifecycleObservation, + code string, + emit func(api.CLIRunEvent) error, +) { + latest := claimed + if loaded, err := a.store.GetGmailDraftContext(ctx, claimed.DraftID); err == nil { + latest = loaded + } + output, err := a.gmailDraftLifecycleOutput(ctx, latest, gmailDraftStatusPending, providerObservation, nil) + if err != nil { + output = gmailDraftLifecycleOutputWithoutMessage(latest, gmailDraftStatusPending, providerObservation, nil) + } + if output.PendingCode == "" { + output.PendingCode = code + } + _ = emitGmailDraftLifecycleOutput(emit, cliStreamStderr, intent, output) +} + +func (a *storeAPIAdapter) recordGmailDraftUncertainOutcome( + ctx context.Context, + intent draftLifecycleIntent, + claimed store.GmailDraft, + code string, + providerErr error, + emit func(api.CLIRunEvent) error, +) error { + evidenceCtx, cancel := localDraftEvidenceContext(ctx) + defer cancel() + recordErr := a.store.RecordGmailDraftOutcomeContext(evidenceCtx, intent.DraftID, intent.Revision, code, "") + a.emitGmailDraftPending(evidenceCtx, intent, claimed, nil, code, emit) + if recordErr != nil { + return draftReplyError("local_persistence_failed", errors.Join(gmailWriteCause(providerErr), recordErr)) + } + return draftReplyError(code, gmailWriteCause(providerErr)) +} + +func (a *storeAPIAdapter) completeGmailDraftDelete( + ctx context.Context, + intent draftLifecycleIntent, + claimed store.GmailDraft, + outcomeCode string, + finish func(), + refresh func(), + emit func(api.CLIRunEvent) error, +) error { + evidenceCtx, cancel := localDraftEvidenceContext(ctx) + defer cancel() + observation := gmailDraftDeleteObservation(claimed, outcomeCode) + if err := a.store.RecordGmailDraftOutcomeContext( + evidenceCtx, intent.DraftID, intent.Revision, outcomeCode, "", + ); err != nil { + a.emitGmailDraftPending(evidenceCtx, intent, claimed, observation, outcomeCode, emit) + return draftReplyError("local_persistence_failed", err) + } + finished, err := a.store.FinishGmailDraftDeleteContext(evidenceCtx, intent.DraftID, intent.Revision) + if err != nil { + a.emitGmailDraftPending(evidenceCtx, intent, claimed, observation, outcomeCode, emit) + return draftReplyError("cleanup_local_failed", err) + } + finish() + defer refresh() + status := "deleted" + if outcomeCode == gmailDraftAbsentOutcome { + status = gmailDraftAbsentOutcome + } + output, err := a.gmailDraftLifecycleOutput(evidenceCtx, finished, status, nil, observation) + if err != nil { + return draftReplyError("draft_read_failed", err) + } + return emitGmailDraftLifecycleOutput(emit, cliStreamStdout, intent, output) +} + +func (a *storeAPIAdapter) retryConfirmedGmailDraftDelete( + ctx context.Context, + intent draftLifecycleIntent, + draft store.GmailDraft, + emit func(api.CLIRunEvent) error, +) error { + logger := a.logger + if logger == nil { + logger = loggerFromContext(ctx) + } + source, err := a.loadManagedGmailDraftSource(ctx, draft) + if err != nil { + return err + } + evidenceCtx, cancelEvidence := localDraftEvidenceContext(ctx) + defer cancelEvidence() + execution, err := a.store.AcquireSyncExecutionContext(evidenceCtx, source.ID) + if err != nil { + if errors.Is(err, store.ErrSyncAlreadyActive) { + return draftReplyError("sync_active", err) + } + return draftReplyError("sync_lock_failed", err) + } + released := false + defer func() { + if !released { + _ = execution.Release() + } + }() + finish := func() { + if released { + return + } + released = true + if err := execution.Release(); err != nil { + logger.Error("release source after Gmail draft delete retry", "source_id", source.ID, "error", err) + } + } + refresh := func() { + evidenceCtx, cancelEvidence := localDraftEvidenceContext(ctx) + defer cancelEvidence() + a.refreshDraftCache(evidenceCtx, source) + } + finished, err := a.store.FinishGmailDraftDeleteContext(evidenceCtx, intent.DraftID, intent.Revision) + if err != nil { + a.emitGmailDraftPending(evidenceCtx, intent, draft, gmailDraftDeleteObservation(draft, draft.Pending.Code), draft.Pending.Code, emit) + return draftReplyError("cleanup_local_failed", err) + } + finish() + defer refresh() + status := "deleted" + if draft.Pending.Code == gmailDraftAbsentOutcome { + status = gmailDraftAbsentOutcome + } + output, err := a.gmailDraftLifecycleOutput( + evidenceCtx, finished, status, nil, gmailDraftDeleteObservation(draft, draft.Pending.Code), + ) + if err != nil { + return draftReplyError("draft_read_failed", err) + } + return emitGmailDraftLifecycleOutput(emit, cliStreamStdout, intent, output) +} + +// runDelegatedGmailDraftLifecycle authorizes a delegated get, edit, or delete +// of a managed Gmail draft before any revision, policy, lock, or provider work. +// imapErr is the failed IMAP lookup for the same draft ID. +func (a *storeAPIAdapter) runDelegatedGmailDraftLifecycle( + ctx context.Context, + intent draftLifecycleIntent, + grant *agentgrant.Grant, + imapErr error, + emit func(api.CLIRunEvent) error, +) error { + if intent.Operation == api.CLIRunDraftRecoverCommand || !errors.Is(imapErr, store.ErrIMAPDraftNotFound) { + return draftReplyNotPermitted(imapErr) + } + draft, err := a.store.GetGmailDraftContext(ctx, intent.DraftID) + if err != nil { + return draftReplyNotPermitted(err) + } + canReadContent, err := a.authorizeDelegatedDraftLifecycle(ctx, intent, grant, draft.SourceID, draft.CurrentMessageID) + if err != nil { + return err + } + intent.MetadataOnly = !canReadContent + return a.runCLIGmailDraftLifecycle(ctx, intent, draft, grant, emit) +} + +func (a *storeAPIAdapter) runCLIGmailDraftLifecycle( + ctx context.Context, + intent draftLifecycleIntent, + draft store.GmailDraft, + grant *agentgrant.Grant, + emit func(api.CLIRunEvent) error, +) error { + if intent.Operation == api.CLIRunDraftGetCommand { + output, err := a.gmailDraftLifecycleOutput(ctx, draft, "ok", &gmailDraftLifecycleObservation{State: "not_checked", Code: "not_checked"}, nil) + if err != nil { + return draftReplyError("draft_read_failed", err) + } + if grant != nil { + if err := a.authorizeDelegatedDraftOutput(ctx, intent, grant, draft.SourceID, output.RawMIME, output.CandidateContent); err != nil { + return err + } + } + return emitGmailDraftLifecycleOutput(emit, cliStreamStdout, intent, output) + } + if draft.Revision != intent.Revision { + return draftReplyError("revision_mismatch", fmt.Errorf("expected revision %d, found %d", intent.Revision, draft.Revision)) + } + if draft.DiscardedAt != nil { + if intent.Operation == api.CLIRunDraftDeleteCommand { + output, err := a.gmailDraftLifecycleOutput(ctx, draft, "already_discarded", nil, nil) + if err != nil { + return draftReplyError("draft_read_failed", err) + } + if grant != nil { + if err := a.authorizeDelegatedDraftOutput(ctx, intent, grant, draft.SourceID, output.RawMIME, output.CandidateContent); err != nil { + return err + } + } + return emitGmailDraftLifecycleOutput(emit, cliStreamStdout, intent, output) + } + return draftReplyError("draft_discarded", errors.New("discarded drafts cannot be edited")) + } + if draft.Pending != nil && draft.Pending.Operation == store.GmailDraftOperationDelete { + if intent.Operation != api.CLIRunDraftDeleteCommand { + return draftReplyError("pending_operation", store.ErrGmailDraftPending) + } + if gmailDraftDeleteOutcomeIsConfirmed(draft.Pending.Code) { + return a.retryConfirmedGmailDraftDelete(ctx, intent, draft, emit) + } + } + source, err := a.loadManagedGmailDraftSource(ctx, draft) + if err != nil { + return err + } + logger := a.logger + if logger == nil { + logger = loggerFromContext(ctx) + } + if err := gmailDraftScopeGate(ctx, a.config, source, oauth.ScopesGmailDraftWrite); err != nil { + return err + } + execution, err := a.store.AcquireSyncExecutionContext(ctx, source.ID) + if err != nil { + if errors.Is(err, store.ErrSyncAlreadyActive) { + return draftReplyError("sync_active", err) + } + return draftReplyError("sync_lock_failed", err) + } + released := false + defer func() { + if !released { + _ = execution.Release() + } + }() + finish := func() { + if released { + return + } + released = true + if err := execution.Release(); err != nil { + logger.Error("release source after Gmail draft lifecycle", "source_id", source.ID, "error", err) + } + } + refresh := func() { + evidenceCtx, cancelEvidence := localDraftEvidenceContext(ctx) + defer cancelEvidence() + a.refreshDraftCache(evidenceCtx, source) + } + draft, err = a.store.GetGmailDraftContext(ctx, intent.DraftID) + if err != nil { + return draftReplyError("draft_not_found", err) + } + if draft.Revision != intent.Revision { + return draftReplyError("revision_mismatch", errors.New("draft changed while acquiring source ownership")) + } + clientFactory := a.gmailDraftClientFactory + if clientFactory == nil { + clientFactory = defaultGmailDraftClientFactory + } + client, err := clientFactory(ctx, source) + if err != nil { + return draftReplyError("invalid_source", err) + } + defer func() { _ = client.Close() }() + observed, err := client.GetDraft(ctx, draft.CurrentReceipt.GmailDraftID) + evidenceCtx, cancelEvidence := localDraftEvidenceContext(ctx) + defer cancelEvidence() + if err != nil { + if _, ok := errors.AsType[*gmail.NotFoundError](err); ok && draft.Pending != nil && intent.Operation == api.CLIRunDraftDeleteCommand { + _, abortErr := a.store.AbortGmailDraftContext(evidenceCtx, intent.DraftID, intent.Revision) + if abortErr != nil { + return draftReplyError("local_persistence_failed", abortErr) + } + } + if _, ok := errors.AsType[*gmail.NotFoundError](err); ok && intent.Operation == api.CLIRunDraftDeleteCommand { + claimed, claimErr := a.store.ClaimGmailDraftContext(evidenceCtx, intent.DraftID, intent.Revision, store.GmailDraftOperationDelete, nil) + if claimErr != nil { + return draftReplyError("claim_failed", claimErr) + } + return a.completeGmailDraftDelete(ctx, intent, claimed, gmailDraftAbsentOutcome, finish, refresh, emit) + } + code := gmailReadErrorCode(err) + return draftReplyError(code, err) + } + recovered := draft.Pending != nil && draft.Pending.Operation == store.GmailDraftOperationEdit && + observed.Message.ID == draft.Pending.ReplacementGmailMessageID + if draft.Pending != nil { + // Release the resolved claim before retrying or adopting the observation. + // Adoption also reuses a replacement already archived by sync. + draft, err = a.store.AbortGmailDraftContext(evidenceCtx, intent.DraftID, intent.Revision) + if err != nil { + return draftReplyError("local_persistence_failed", err) + } + } + if observed.Message.ID != draft.CurrentReceipt.GmailMessageID { + parsed, parseErr := msgmime.Parse(observed.Message.Raw) + if parseErr != nil { + return draftReplyError("changed_externally", parseErr) + } + if _, messageErr := a.store.GetMessageContext(evidenceCtx, draft.CurrentMessageID); messageErr != nil { + return draftReplyError("draft_read_failed", messageErr) + } + replyTo, replyToErr := a.store.GetMessageReplyToMessageIDContext(evidenceCtx, draft.CurrentMessageID) + if replyToErr != nil { + return draftReplyError("draft_read_failed", replyToErr) + } + observedReceipt := store.GmailDraftReceipt{ + SourceID: source.ID, GmailDraftID: observed.ID, + GmailMessageID: observed.Message.ID, ThreadID: observed.Message.ThreadID, + } + participants := gmailDraftParticipants(parsed) + var attachmentWrites *[]store.AttachmentWrite + if len(parsed.Attachments) > 0 { + writes, attachmentErr := gmailDraftAttachmentWrites(a.config, parsed.Attachments) + if attachmentErr != nil { + return draftReplyError("local_persistence_failed", attachmentErr) + } + attachmentWrites = &writes + } + adopted, adoptErr := a.store.AdoptGmailDraftObservationContext( + evidenceCtx, intent.DraftID, intent.Revision, observedReceipt, participants, + gmailDraftMessagePersistDataWithAttachments(source.ID, replyTo.Int64, parsed, observed.Message.Raw, observedReceipt, + messageRFC822ID(parsed), attachmentWrites), + ) + if adoptErr != nil { + return draftReplyError("local_persistence_failed", adoptErr) + } + finish() + defer refresh() + status := "changed_externally" + if recovered { + status = "recovered" + } + output, outputErr := a.gmailDraftLifecycleOutput(evidenceCtx, adopted, status, &gmailDraftLifecycleObservation{ + State: "changed", Code: status, GmailDraftID: observed.ID, + GmailMessageID: observed.Message.ID, ThreadID: observed.Message.ThreadID, Present: true, + }, nil) + if outputErr != nil { + return draftReplyError("draft_read_failed", outputErr) + } + // A provider edit can change From. Check the exact snapshot returned. + if grant != nil { + if err := a.authorizeDelegatedDraftOutput(evidenceCtx, intent, grant, adopted.SourceID, output.RawMIME, output.CandidateContent); err != nil { + return err + } + } + if err := emitGmailDraftLifecycleOutput(emit, cliStreamStderr, intent, output); err != nil { + return draftReplyError("output_failed", err) + } + if recovered { + return draftReplyError("revision_mismatch", errors.New("pending edit recovered; review the new revision before retrying")) + } + return draftReplyError("changed_externally", errors.New("gmail draft changed outside msgvault")) + } + if intent.Operation == api.CLIRunDraftEditCommand { + currentRaw, readErr := a.store.GetMessageRawContext(ctx, draft.CurrentMessageID) + if readErr != nil { + return draftReplyError("draft_read_failed", readErr) + } + replacement, buildErr := imaplib.BuildDraftReplacement(currentRaw, intent.Body, time.Now(), "") + if buildErr != nil { + return draftReplyError("invalid_draft", buildErr) + } + return a.runGmailDraftEdit(ctx, intent, draft, source, client, replacement, finish, refresh, emit) + } + return a.runGmailDraftDelete(ctx, intent, draft, client, finish, refresh, emit) +} + +func (a *storeAPIAdapter) runGmailDraftEdit( + ctx context.Context, + intent draftLifecycleIntent, + draft store.GmailDraft, + source *store.Source, + client gmail.DraftAPI, + replacement imaplib.ReplyDraft, + finish func(), + refresh func(), + emit func(api.CLIRunEvent) error, +) error { + claimed, err := a.store.ClaimGmailDraftContext(ctx, intent.DraftID, intent.Revision, store.GmailDraftOperationEdit, replacement.Raw) + if err != nil { + return draftReplyError("claim_failed", err) + } + updated, err := client.UpdateDraft(ctx, draft.CurrentReceipt.GmailDraftID, replacement.Raw, draft.CurrentReceipt.ThreadID) + if err != nil { + evidenceCtx, cancelEvidence := localDraftEvidenceContext(ctx) + defer cancelEvidence() + code := gmailWriteErrorCode(err) + var writeErr *gmail.DraftWriteError + if errors.As(err, &writeErr) && writeErr.State == gmail.DraftStateRejected || + errors.As(err, &writeErr) && writeErr.State == gmail.DraftStateCancelled { + active, abortErr := a.store.AbortGmailDraftContext(evidenceCtx, intent.DraftID, intent.Revision) + if abortErr != nil { + return draftReplyError("local_persistence_failed", abortErr) + } + output, outputErr := a.gmailDraftLifecycleOutput(evidenceCtx, active, code, nil, nil) + if outputErr == nil { + _ = emitGmailDraftLifecycleOutput(emit, cliStreamStderr, intent, output) + } + return draftReplyError(code, gmailWriteCause(err)) + } + return a.recordGmailDraftUncertainOutcome(ctx, intent, claimed, code, err, emit) + } + if updated == nil { + return a.recordGmailDraftUncertainOutcome( + ctx, intent, claimed, "remote_unknown", errors.New("gmail update returned no draft"), emit, + ) + } + evidenceCtx, cancelEvidence := localDraftEvidenceContext(ctx) + defer cancelEvidence() + replacementReceipt := store.GmailDraftReceipt{ + SourceID: source.ID, GmailDraftID: updated.ID, + GmailMessageID: updated.Message.ID, ThreadID: updated.Message.ThreadID, + } + if err := a.store.RecordGmailDraftOutcomeContext(evidenceCtx, intent.DraftID, intent.Revision, "accepted_local_failed", updated.Message.ID); err != nil { + return a.reportGmailDraftAcceptedLocalFailure( + emit, intent, claimed, replacementReceipt, replacement.Raw, err, + ) + } + replyTo, err := a.store.GetMessageReplyToMessageIDContext(evidenceCtx, draft.CurrentMessageID) + if err != nil { + return a.reportGmailDraftAcceptedLocalFailure( + emit, intent, claimed, replacementReceipt, replacement.Raw, err, + ) + } + published, err := a.store.PublishGmailDraftReplacementContext( + evidenceCtx, intent.DraftID, intent.Revision, updated.Message.ID, + gmailDraftParticipants(replacement.Parsed), + gmailDraftMessagePersistData(source.ID, replyTo.Int64, replacement.Parsed, replacement.Raw, replacementReceipt, messageRFC822ID(replacement.Parsed)), + ) + if err != nil { + return a.reportGmailDraftAcceptedLocalFailure( + emit, intent, claimed, replacementReceipt, replacement.Raw, err, + ) + } + finish() + defer refresh() + output, err := a.gmailDraftLifecycleOutput(evidenceCtx, published, "edited", nil, nil) + if err != nil { + return draftReplyError("draft_read_failed", err) + } + return emitGmailDraftLifecycleOutput(emit, cliStreamStdout, intent, output) +} + +func (a *storeAPIAdapter) runGmailDraftDelete( + ctx context.Context, + intent draftLifecycleIntent, + draft store.GmailDraft, + client gmail.DraftAPI, + finish func(), + refresh func(), + emit func(api.CLIRunEvent) error, +) error { + claimed, err := a.store.ClaimGmailDraftContext(ctx, intent.DraftID, intent.Revision, store.GmailDraftOperationDelete, nil) + if err != nil { + return draftReplyError("claim_failed", err) + } + err = client.DeleteDraft(ctx, draft.CurrentReceipt.GmailDraftID) + if err != nil { + evidenceCtx, cancelEvidence := localDraftEvidenceContext(ctx) + defer cancelEvidence() + code := gmailWriteErrorCode(err) + var writeErr *gmail.DraftWriteError + if errors.As(err, &writeErr) && writeErr.Code == "draft_absent" { + return a.completeGmailDraftDelete(ctx, intent, claimed, gmailDraftAbsentOutcome, finish, refresh, emit) + } + if errors.As(err, &writeErr) && (writeErr.State == gmail.DraftStateRejected || writeErr.State == gmail.DraftStateCancelled) { + active, abortErr := a.store.AbortGmailDraftContext(evidenceCtx, intent.DraftID, intent.Revision) + if abortErr != nil { + return draftReplyError("local_persistence_failed", abortErr) + } + output, outputErr := a.gmailDraftLifecycleOutput(evidenceCtx, active, code, nil, nil) + if outputErr == nil { + _ = emitGmailDraftLifecycleOutput(emit, cliStreamStderr, intent, output) + } + return draftReplyError(code, gmailWriteCause(err)) + } + return a.recordGmailDraftUncertainOutcome(ctx, intent, claimed, code, err, emit) + } + return a.completeGmailDraftDelete(ctx, intent, claimed, gmailDraftDeleteOutcome, finish, refresh, emit) +} + +func messageRFC822ID(parsed *msgmime.Message) string { + id := msgmime.NormalizeMessageID(parsed.MessageID) + if id == "" { + return "" + } + return "<" + id + ">" +} + +func parseDraftSendAsArgs(args []string) (string, bool, error) { + if !api.IsCLIRunDraftSendAs(args) { + return "", false, draftReplyError("invalid_args", errors.New("expected draft-send-as")) + } + var account string + jsonOutput := false + for _, arg := range args[1:] { + if strings.HasPrefix(arg, "--json") { + if arg != "--json" && arg != "--json=true" { + return "", false, draftReplyError("invalid_args", errors.New("--json accepts no value")) + } + jsonOutput = true + continue + } + if strings.HasPrefix(arg, "--log-") || arg == "--verbose" { + continue + } + if account != "" || !utf8.ValidString(arg) || strings.TrimSpace(arg) == "" || strings.ContainsAny(arg, "\x00\r\n") { + return "", false, draftReplyError("invalid_args", errors.New("draft-send-as requires one account")) + } + account = arg + } + if account == "" { + return "", false, draftReplyError("invalid_args", errors.New("draft-send-as requires one account")) + } + return account, jsonOutput, nil +} + +type gmailSendAsOutput struct { + SourceID int64 `json:"source_id"` + Account string `json:"account"` + Entries []gmailSendAsRow `json:"send_as"` +} + +type gmailSendAsRow struct { + Email string `json:"email"` + DisplayName string `json:"display_name,omitempty"` + Primary bool `json:"primary"` + Default bool `json:"default"` + VerificationStatus string `json:"verification_status"` + ConfirmedIdentity bool `json:"confirmed_identity"` +} + +func (a *storeAPIAdapter) runCLIDraftSendAs(ctx context.Context, req api.CLIRunRequest, emit func(api.CLIRunEvent) error) error { + if req.Grant != nil { + return draftReplyNotPermitted(errors.New("gmail send-as listing is owner-only")) + } + if len(req.Env) != 0 || req.Cwd != "" { + return draftReplyError("invalid_args", errors.New("draft-send-as accepts no environment or working directory")) + } + account, jsonOutput, err := parseDraftSendAsArgs(req.Args) + if err != nil { + return err + } + source, err := sourceops.ResolveExactOne(a.store, sourceops.Selector{Account: account, SourceType: "gmail"}) + if err != nil { + return draftReplyError("invalid_source", err) + } + if err := gmailDraftScopeGate(ctx, a.config, source, oauth.ScopesGmailSendAsList); err != nil { + return err + } + clientFactory := a.gmailDraftClientFactory + if clientFactory == nil { + clientFactory = defaultGmailDraftClientFactory + } + client, err := clientFactory(ctx, source) + if err != nil { + return draftReplyError("invalid_source", err) + } + defer func() { _ = client.Close() }() + entries, err := client.ListSendAs(ctx) + if err != nil { + return draftReplyError(gmailReadErrorCode(err), err) + } + identities, err := a.store.ListAccountIdentitiesContext(ctx, source.ID) + if err != nil { + return draftReplyError("invalid_from", err) + } + output := gmailSendAsOutput{SourceID: source.ID, Account: source.Identifier, Entries: make([]gmailSendAsRow, len(entries))} + confirmed, _ := confirmedDraftIdentities(identities) + for i, entry := range entries { + _, isConfirmed := confirmed[store.NormalizeIdentifierForCompare(entry.Email)] + output.Entries[i] = gmailSendAsRow{ + Email: entry.Email, DisplayName: entry.DisplayName, Primary: entry.Primary, + Default: entry.Default, VerificationStatus: entry.VerificationStatus, + ConfirmedIdentity: isConfirmed, + } + } + if emit == nil { + return nil + } + if jsonOutput { + data, err := jsonv2.Marshal(output) + if err != nil { + return draftReplyError("output_failed", err) + } + return emit(api.CLIRunEvent{Type: cliStreamStdout, Data: string(data) + "\n"}) + } + var data strings.Builder + for _, entry := range output.Entries { + fmt.Fprintf(&data, "%s\t%s\tprimary=%t\tdefault=%t\tverification=%s\tconfirmed=%t\n", + textutil.SanitizeTerminal(entry.Email), textutil.SanitizeTerminal(entry.DisplayName), + entry.Primary, entry.Default, textutil.SanitizeTerminal(entry.VerificationStatus), entry.ConfirmedIdentity) + } + return emit(api.CLIRunEvent{Type: cliStreamStdout, Data: data.String()}) +} diff --git a/cmd/msgvault/cmd/serve_gmail_draft_review_test.go b/cmd/msgvault/cmd/serve_gmail_draft_review_test.go new file mode 100644 index 000000000..38ddf5b4e --- /dev/null +++ b/cmd/msgvault/cmd/serve_gmail_draft_review_test.go @@ -0,0 +1,1436 @@ +package cmd + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/agentgrant" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/gmail" + msgmime "go.kenn.io/msgvault/internal/mime" + "go.kenn.io/msgvault/internal/store" + "go.kenn.io/msgvault/internal/testutil" + testemail "go.kenn.io/msgvault/internal/testutil/email" +) + +type scriptedGmailDraftClient struct { + createDraft *gmail.Draft + createErr error + getDraft *gmail.Draft + getErr error + updateDraft *gmail.Draft + updateErr error + updateHook func() + deleteErr error + deleteHook func() + sendAs []gmail.SendAs + sendAsErr error + + createCalls int + getCalls int + updateCalls int + deleteCalls int + listCalls int +} + +func (c *scriptedGmailDraftClient) CreateDraft(_ context.Context, raw []byte, threadID string) (*gmail.Draft, error) { + c.createCalls++ + if c.createErr != nil { + return nil, c.createErr + } + if c.createDraft != nil { + return c.createDraft, nil + } + return &gmail.Draft{ + ID: "gmail-draft-created", + Message: gmail.RawMessage{ + ID: "gmail-message-created", ThreadID: threadID, Raw: append([]byte(nil), raw...), + }, + }, nil +} + +func (c *scriptedGmailDraftClient) GetDraft(context.Context, string) (*gmail.Draft, error) { + c.getCalls++ + if c.getErr != nil { + return nil, c.getErr + } + if c.getDraft == nil { + return nil, errors.New("test Gmail draft was not configured") + } + return c.getDraft, nil +} + +func (c *scriptedGmailDraftClient) UpdateDraft(context.Context, string, []byte, string) (*gmail.Draft, error) { + c.updateCalls++ + if c.updateHook != nil { + c.updateHook() + } + if c.updateErr != nil { + return nil, c.updateErr + } + if c.updateDraft == nil { + return nil, errors.New("test Gmail update was not configured") + } + return c.updateDraft, nil +} + +func (c *scriptedGmailDraftClient) DeleteDraft(context.Context, string) error { + c.deleteCalls++ + if c.deleteHook != nil { + c.deleteHook() + } + return c.deleteErr +} + +func (c *scriptedGmailDraftClient) ListSendAs(context.Context) ([]gmail.SendAs, error) { + c.listCalls++ + if c.sendAsErr != nil { + return nil, c.sendAsErr + } + return c.sendAs, nil +} + +func (c *scriptedGmailDraftClient) Close() error { return nil } + +type gmailDraftTestFixture struct { + store *store.Store + source *store.Source + conversationID int64 + parentID int64 + client *scriptedGmailDraftClient + adapter *storeAPIAdapter +} + +func newGmailDraftTestFixture(t *testing.T) gmailDraftTestFixture { + t.Helper() + return newGmailDraftTestFixtureWithStore(t, testutil.NewTestStore) +} + +func newSQLiteGmailDraftTestFixture(t *testing.T) gmailDraftTestFixture { + t.Helper() + return newGmailDraftTestFixtureWithStore(t, testutil.NewSQLiteTestStore) +} + +func newGmailDraftTestFixtureWithStore(t *testing.T, newStore func(*testing.T) *store.Store) gmailDraftTestFixture { + t.Helper() + cfg := &config.Config{ + Data: config.DataConfig{DataDir: t.TempDir()}, + OAuth: config.OAuthConfig{ServiceAccountKey: "synthetic-service-account"}, + } + + st := newStore(t) + source, err := st.GetOrCreateSource("gmail", "owner@example.test") + require.NoError(t, err) + require.NoError(t, st.AddAccountIdentity(source.ID, source.Identifier, "manual")) + conversationID, err := st.EnsureConversation(source.ID, "gmail-thread-1", "Question") + require.NoError(t, err) + senderID, err := st.EnsureParticipant("sender@example.test", "Sender", "example.test") + require.NoError(t, err) + ownerID, err := st.EnsureParticipant(source.Identifier, "", "example.test") + require.NoError(t, err) + parentRaw := []byte("From: Sender \r\n" + + "To: " + source.Identifier + "\r\n" + + "Subject: Question\r\n" + + "Message-ID: \r\n\r\n" + + "Parent body\r\n") + parentID, err := st.PersistMessage(&store.MessagePersistData{ + Message: &store.Message{ + SourceID: source.ID, SourceMessageID: "gmail-parent-1", + ConversationID: conversationID, + RFC822MessageID: sql.NullString{String: "gmail-parent@example.test", Valid: true}, + MessageType: store.MessageTypeEmail, + SenderID: sql.NullInt64{Int64: senderID, Valid: true}, + }, + Conversation: &store.ConversationPersistData{ + SourceConversationID: "gmail-thread-1", ConversationType: "email_thread", Title: "Question", + }, + BodyText: sql.NullString{String: "Parent body", Valid: true}, RawMIME: parentRaw, + Recipients: []store.RecipientSet{ + {Type: "from", ParticipantIDs: []int64{senderID}, EmailAddresses: []string{"sender@example.test"}}, + {Type: "to", ParticipantIDs: []int64{ownerID}, EmailAddresses: []string{source.Identifier}}, + }, + }) + require.NoError(t, err) + + client := &scriptedGmailDraftClient{ + sendAs: []gmail.SendAs{ + {Email: source.Identifier, Primary: true, Default: true, VerificationStatus: "accepted"}, + {Email: "alias@example.test", VerificationStatus: "accepted"}, + }, + } + adapter := &storeAPIAdapter{ + store: st, + config: cfg, + logger: testLoggerValue(), + gmailDraftPolicy: []config.GmailDraftSource{{SourceID: source.ID, Enabled: true}}, + gmailDraftClientFactory: func(context.Context, *store.Source) (gmail.DraftAPI, error) { + return client, nil + }, + } + return gmailDraftTestFixture{ + store: st, source: source, conversationID: conversationID, + parentID: parentID, client: client, adapter: adapter, + } +} + +func gmailDraftTestRaw(body, messageID string) []byte { + return []byte(fmt.Sprintf( + "From: owner@example.test\r\nTo: sender@example.test\r\nSubject: Re: Question\r\nMessage-ID: <%s>\r\n\r\n%s\r\n", + messageID, body, + )) +} + +func gmailDraftTestRawWithAttachment(messageID string) []byte { + return testemail.NewMessage(). + From("owner@example.test"). + To("sender@example.test"). + Subject("Re: Question"). + Header("Message-ID", "<"+messageID+">"). + Body("external with attachment"). + WithAttachment("notes.txt", "text/plain", []byte("attachment bytes")). + CRLF(). + Bytes() +} + +func (f gmailDraftTestFixture) create(t *testing.T, body string, asJSON bool) ([]api.CLIRunEvent, error) { + t.Helper() + return f.createContext(t.Context(), t, body, asJSON) +} + +func (f gmailDraftTestFixture) createContext(ctx context.Context, t *testing.T, body string, asJSON bool) ([]api.CLIRunEvent, error) { + t.Helper() + args := []string{ + api.CLIRunDraftReplyCommand, strconv.FormatInt(f.parentID, 10), + "--from", f.source.Identifier, "--body", body, + } + if asJSON { + args = append(args, "--json") + } + var events []api.CLIRunEvent + err := f.adapter.runCLIReplyDraft(ctx, api.CLIRunRequest{Args: args}, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + return events, err +} + +func (f gmailDraftTestFixture) seedDraft(t *testing.T) store.GmailDraft { + t.Helper() + raw := gmailDraftTestRaw("original", "gmail-original@example.test") + parsed, err := msgmime.Parse(raw) + require.NoError(t, err) + receipt := store.GmailDraftReceipt{ + SourceID: f.source.ID, GmailDraftID: "gmail-draft-managed", + GmailMessageID: "gmail-message-original", ThreadID: "gmail-thread-1", + } + draft, err := f.store.PersistGmailDraftContext( + t.Context(), receipt, gmailDraftParticipants(parsed), + gmailDraftMessagePersistData(f.source.ID, f.parentID, parsed, raw, receipt, messageRFC822ID(parsed)), + ) + require.NoError(t, err) + f.client.getDraft = &gmail.Draft{ + ID: receipt.GmailDraftID, + Message: gmail.RawMessage{ID: receipt.GmailMessageID, ThreadID: receipt.ThreadID, Raw: raw}, + } + return draft +} + +func (f gmailDraftTestFixture) lifecycle(t *testing.T, operation string, draft store.GmailDraft, body string) ([]api.CLIRunEvent, error) { + t.Helper() + args := []string{ + operation, draft.DraftID, "--revision", strconv.FormatInt(draft.Revision, 10), + } + if operation == api.CLIRunDraftEditCommand { + args = append(args, "--body", body) + } + args = append(args, "--json") + var events []api.CLIRunEvent + err := f.adapter.runCLIDraftLifecycle(t.Context(), api.CLIRunRequest{Args: args}, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + return events, err +} + +func TestGmailDraftCreateAndSendAsUseLocalBehavior(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + fixture := newGmailDraftTestFixture(t) + + events, err := fixture.create(t, "created body", true) + require.NoError(err) + require.Len(events, 1) + var created gmailDraftReplyOutput + require.NoError(json.Unmarshal([]byte(events[0].Data), &created)) + assert.Equal(gmailDraftStatusCreated, created.Status) + assert.Equal("gmail-draft-created", created.GmailDraftID) + assert.Equal(int64(1), created.Revision) + assert.Equal(1, fixture.client.createCalls) + draft, err := fixture.store.GetGmailDraftContext(t.Context(), created.DraftID) + require.NoError(err) + assert.Equal(created.GmailMessageID, draft.CurrentReceipt.GmailMessageID) + + var labelCount int + require.NoError(fixture.store.DB().QueryRow(fixture.store.Rebind(` + SELECT COUNT(*) FROM message_labels ml + JOIN labels l ON l.id = ml.label_id + WHERE ml.message_id = ? AND l.source_label_id = 'DRAFT' + `), draft.CurrentMessageID).Scan(&labelCount)) + assert.Equal(1, labelCount) + + var sendAsEvents []api.CLIRunEvent + err = fixture.adapter.runCLIDraftSendAs(t.Context(), api.CLIRunRequest{ + Args: []string{api.CLIRunDraftSendAsCommand, fixture.source.Identifier, "--json"}, + }, func(event api.CLIRunEvent) error { + sendAsEvents = append(sendAsEvents, event) + return nil + }) + require.NoError(err) + require.Len(sendAsEvents, 1) + var sendAs gmailSendAsOutput + require.NoError(json.Unmarshal([]byte(sendAsEvents[0].Data), &sendAs)) + require.Len(sendAs.Entries, 2) + assert.True(sendAs.Entries[0].ConfirmedIdentity) +} + +func TestGmailDraftReplyAllInfersSenderAndIndexesCc(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture := newGmailDraftTestFixture(t) + raw, err := fixture.store.GetMessageRaw(fixture.parentID) + requirements.NoError(err) + raw = []byte("Cc: copy@example.test\r\n" + string(raw)) + requirements.NoError(fixture.store.UpsertMessageRaw(fixture.parentID, raw)) + var events []api.CLIRunEvent + err = fixture.adapter.runCLIReplyDraft(t.Context(), api.CLIRunRequest{ + Args: []string{"draft-reply", strconv.FormatInt(fixture.parentID, 10), "--all", "--body", "reply", "--json"}, + }, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + requirements.NoError(err) + requirements.Len(events, 1) + var created gmailDraftReplyOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &created)) + message, err := fixture.store.GetMessageContext(t.Context(), created.MessageID) + requirements.NoError(err) + assertions.Equal("owner@example.test", message.From) + assertions.Equal([]string{"Sender "}, message.To) + assertions.Equal([]string{"copy@example.test"}, message.Cc) + assertions.Equal("gmail-thread-1", created.ThreadID) + + // A shared token avoids backend differences in email punctuation handling. + matches, total, err := fixture.store.SearchMessages("copy", 0, 10) + requirements.NoError(err) + requirements.Equal(int64(1), total) + requirements.Len(matches, 1) + assertions.Equal(created.MessageID, matches[0].ID) +} + +func TestGmailDraftRejectsComposeAndCrossSourceReply(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + fixture := newGmailDraftTestFixture(t) + other, err := fixture.store.GetOrCreateSource("gmail", "other@example.test") + requirements.NoError(err) + requirements.NoError(fixture.store.AddAccountIdentity(other.ID, other.Identifier, "manual")) + fixture.adapter.gmailDraftPolicy = append(fixture.adapter.gmailDraftPolicy, + config.GmailDraftSource{SourceID: other.ID, Enabled: true}) + err = fixture.adapter.runCLIComposeDraft(t.Context(), api.CLIRunRequest{ + Args: []string{"draft-compose", "--source-id", strconv.FormatInt(fixture.source.ID, 10), "--to", "recipient@example.test"}, + }, nil) + requirements.ErrorContains(err, "draft_disabled") + requirements.EqualError(errors.Unwrap(err), "draft-compose requires an IMAP source") + err = fixture.adapter.runCLIReplyDraft(t.Context(), api.CLIRunRequest{ + Args: []string{"draft-reply", strconv.FormatInt(fixture.parentID, 10), "--source-id", strconv.FormatInt(other.ID, 10), "--body", "reply"}, + }, nil) + requirements.ErrorContains(err, "draft_disabled") + assertions.Zero(fixture.client.createCalls) +} + +func TestGmailDraftSendAsFailureIsReportedLocally(t *testing.T) { + fixture := newGmailDraftTestFixture(t) + fixture.client.sendAsErr = errors.New("send-as unavailable") + err := fixture.adapter.runCLIDraftSendAs(t.Context(), api.CLIRunRequest{ + Args: []string{api.CLIRunDraftSendAsCommand, fixture.source.Identifier}, + }, nil) + require.Error(t, err) + assert.Equal(t, "provider_refused", err.Error()) +} + +func TestGmailDraftLifecyclePublishesEditAndDelete(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + fixture := newGmailDraftTestFixture(t) + draft := fixture.seedDraft(t) + fixture.client.updateDraft = &gmail.Draft{ + ID: "gmail-draft-managed", + Message: gmail.RawMessage{ID: "gmail-message-edited", ThreadID: "gmail-thread-1"}, + } + events, err := fixture.lifecycle(t, api.CLIRunDraftEditCommand, draft, "edited") + require.NoError(err) + require.Len(events, 1) + assert.Contains(events[0].Data, `"status":"edited"`) + updated, err := fixture.store.GetGmailDraftContext(t.Context(), draft.DraftID) + require.NoError(err) + assert.Equal(int64(2), updated.Revision) + assert.Equal("gmail-message-edited", updated.CurrentReceipt.GmailMessageID) + + fixture.client.getDraft = &gmail.Draft{ + ID: "gmail-draft-managed", + Message: gmail.RawMessage{ID: "gmail-message-edited", ThreadID: "gmail-thread-1"}, + } + events, err = fixture.lifecycle(t, api.CLIRunDraftDeleteCommand, updated, "") + require.NoError(err) + require.Len(events, 1) + assert.Contains(events[0].Data, `"status":"deleted"`) + deleted, err := fixture.store.GetGmailDraftContext(t.Context(), draft.DraftID) + require.NoError(err) + assert.NotNil(deleted.DiscardedAt) +} + +func TestGmailDraftRecoverIsNotSupported(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + fixture := newGmailDraftTestFixture(t) + draft := fixture.seedDraft(t) + + events, err := fixture.lifecycle(t, api.CLIRunDraftRecoverCommand, draft, "") + require.Error(err) + assert.Equal("not_supported", err.Error()) + assert.Empty(events) + assert.Zero(fixture.client.getCalls + fixture.client.updateCalls + fixture.client.deleteCalls) + latest, err := fixture.store.GetGmailDraftContext(t.Context(), draft.DraftID) + require.NoError(err) + assert.Equal(draft.Revision, latest.Revision) + assert.Nil(latest.DiscardedAt) +} + +func delegatedGmailDraftGrant(f gmailDraftTestFixture, permissions []agentgrant.Permission, sourceType, identifier string, senderKeys ...[]string) *agentgrant.Grant { + source := agentgrant.SourceRef{ID: f.source.ID, Type: sourceType, Identifier: identifier, SenderKeys: []string{"owner@example.test"}} + if len(senderKeys) > 0 { + source.SenderKeys = senderKeys[0] + } + return &agentgrant.Grant{ + ID: "gmail-grant", + Permissions: permissions, + Sources: []agentgrant.SourceRef{source}, + } +} + +func TestGmailDraftLifecycleDelegatedGrantManagesDraft(t *testing.T) { + for _, tc := range []struct { + name string + operation string + permission agentgrant.Permission + body string + }{ + {name: "get with create", operation: api.CLIRunDraftGetCommand, permission: agentgrant.PermissionDraftCreate}, + {name: "get with edit", operation: api.CLIRunDraftGetCommand, permission: agentgrant.PermissionDraftEdit}, + {name: "get with delete", operation: api.CLIRunDraftGetCommand, permission: agentgrant.PermissionDraftDelete}, + {name: "edit with edit", operation: api.CLIRunDraftEditCommand, permission: agentgrant.PermissionDraftEdit, body: "edited"}, + {name: "delete with delete", operation: api.CLIRunDraftDeleteCommand, permission: agentgrant.PermissionDraftDelete}, + } { + t.Run(tc.name, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + fixture := newGmailDraftTestFixture(t) + draft := fixture.seedDraft(t) + if tc.operation == api.CLIRunDraftEditCommand { + fixture.client.updateDraft = &gmail.Draft{ + ID: "gmail-draft-managed", + Message: gmail.RawMessage{ID: "gmail-message-edited", ThreadID: "gmail-thread-1"}, + } + } + args := []string{tc.operation, draft.DraftID} + if tc.operation != api.CLIRunDraftGetCommand { + args = append(args, "--revision", strconv.FormatInt(draft.Revision, 10)) + } + if tc.body != "" { + args = append(args, "--body", tc.body) + } + args = append(args, "--json") + grant := delegatedGmailDraftGrant(fixture, []agentgrant.Permission{tc.permission}, fixture.source.SourceType, fixture.source.Identifier) + var events []api.CLIRunEvent + err := fixture.adapter.runCLIDraftLifecycle(t.Context(), api.CLIRunRequest{Args: args, Grant: grant}, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + require.NoError(err) + require.Len(events, 1) + var output gmailDraftLifecycleOutput + require.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + if tc.permission == agentgrant.PermissionDraftDelete { + assert.Empty(output.Content) + assert.Empty(output.RawMIME) + assert.Empty(output.CandidateContent) + } else { + assert.NotEmpty(output.Content) + assert.NotEmpty(output.RawMIME) + } + switch tc.operation { + case api.CLIRunDraftGetCommand: + assert.Equal("gmail", output.Provider) + assert.Zero(fixture.client.getCalls + fixture.client.updateCalls + fixture.client.deleteCalls) + case api.CLIRunDraftEditCommand: + assert.Equal("edited", output.Status) + assert.Equal(int64(2), output.Revision) + assert.Equal(1, fixture.client.getCalls) + assert.Equal(1, fixture.client.updateCalls) + case api.CLIRunDraftDeleteCommand: + assert.Equal("deleted", output.Status) + assert.Equal("discarded", output.Lifecycle) + assert.Equal(1, fixture.client.deleteCalls) + } + }) + } +} + +func TestGmailDraftLifecycleDelegatedSenderScopeFailsClosed(t *testing.T) { + for _, access := range []struct { + operation string + permission agentgrant.Permission + }{ + {api.CLIRunDraftGetCommand, agentgrant.PermissionDraftCreate}, + {api.CLIRunDraftGetCommand, agentgrant.PermissionDraftEdit}, + {api.CLIRunDraftGetCommand, agentgrant.PermissionDraftDelete}, + {api.CLIRunDraftEditCommand, agentgrant.PermissionDraftEdit}, + {api.CLIRunDraftDeleteCommand, agentgrant.PermissionDraftDelete}, + } { + for _, tc := range []struct { + name string + senderKeys []string + raw []byte + }{ + {name: "different From on same source", senderKeys: []string{"owner@example.test"}, raw: []byte("From: other@example.test\r\nTo: sender@example.test\r\nSubject: Re: Question\r\n\r\noriginal\r\n")}, + {name: "empty sender keys", senderKeys: []string{}}, + {name: "absent From", senderKeys: []string{"owner@example.test"}, raw: []byte("To: sender@example.test\r\nSubject: Re: Question\r\n\r\noriginal\r\n")}, + {name: "malformed From", senderKeys: []string{"owner@example.test"}, raw: []byte("From: not-an-address\r\nTo: sender@example.test\r\nSubject: Re: Question\r\n\r\noriginal\r\n")}, + } { + t.Run(access.operation+"/"+string(access.permission)+"/"+tc.name, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + fixture := newGmailDraftTestFixture(t) + draft := fixture.seedDraft(t) + if tc.raw != nil { + require.NoError(fixture.store.UpsertMessageRaw(draft.CurrentMessageID, tc.raw)) + } + factoryCalls := 0 + factory := fixture.adapter.gmailDraftClientFactory + fixture.adapter.gmailDraftClientFactory = func(ctx context.Context, source *store.Source) (gmail.DraftAPI, error) { + factoryCalls++ + return factory(ctx, source) + } + grant := delegatedGmailDraftGrant(fixture, []agentgrant.Permission{access.permission}, fixture.source.SourceType, fixture.source.Identifier, tc.senderKeys) + args := []string{access.operation, draft.DraftID, "--json"} + if access.operation != api.CLIRunDraftGetCommand { + args = append(args, "--revision", "99") + } + if access.operation == api.CLIRunDraftEditCommand { + args = append(args, "--body", "edited") + } + var events []api.CLIRunEvent + err := fixture.adapter.runCLIDraftLifecycle(t.Context(), api.CLIRunRequest{ + Args: args, Grant: grant, + }, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + require.Error(err) + assert.Equal("not_permitted", err.Error()) + if tc.name == "absent From" { + assert.Contains(errors.Unwrap(err).Error(), "exactly one From address") + } + assert.Empty(events) + assert.Zero(factoryCalls) + assert.Zero(fixture.client.getCalls + fixture.client.updateCalls + fixture.client.deleteCalls) + }) + } + } +} + +func TestGmailDraftLifecycleDelegatedDenialPrecedesProviderWork(t *testing.T) { + for _, tc := range []struct { + name string + operation string + permissions []agentgrant.Permission + sourceType string + identifier string + draftID string + wrongRevision bool + emptyPolicy bool + }{ + {name: "edit with create", operation: api.CLIRunDraftEditCommand, permissions: []agentgrant.Permission{agentgrant.PermissionDraftCreate}}, + {name: "delete with edit", operation: api.CLIRunDraftDeleteCommand, permissions: []agentgrant.Permission{agentgrant.PermissionDraftEdit}}, + {name: "get with wrong identifier", operation: api.CLIRunDraftGetCommand, permissions: []agentgrant.Permission{agentgrant.PermissionDraftCreate}, identifier: "other@example.test"}, + {name: "edit with wrong source type", operation: api.CLIRunDraftEditCommand, permissions: []agentgrant.Permission{agentgrant.PermissionDraftEdit}, sourceType: "imap"}, + {name: "edit with wrong permission and revision", operation: api.CLIRunDraftEditCommand, permissions: []agentgrant.Permission{agentgrant.PermissionDraftCreate}, wrongRevision: true}, + {name: "edit with wrong permission and policy", operation: api.CLIRunDraftEditCommand, permissions: []agentgrant.Permission{agentgrant.PermissionDraftCreate}, emptyPolicy: true}, + {name: "unknown draft ID", operation: api.CLIRunDraftGetCommand, permissions: []agentgrant.Permission{agentgrant.PermissionDraftCreate}, draftID: "draft-unknown"}, + {name: "Gmail recovery", operation: api.CLIRunDraftRecoverCommand, permissions: []agentgrant.Permission{agentgrant.PermissionDraftEdit, agentgrant.PermissionDraftDelete}}, + } { + t.Run(tc.name, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + fixture := newGmailDraftTestFixture(t) + draft := fixture.seedDraft(t) + factoryCalls := 0 + factory := fixture.adapter.gmailDraftClientFactory + fixture.adapter.gmailDraftClientFactory = func(ctx context.Context, source *store.Source) (gmail.DraftAPI, error) { + factoryCalls++ + return factory(ctx, source) + } + if tc.emptyPolicy { + fixture.adapter.gmailDraftPolicy = nil + } + identifier := fixture.source.Identifier + if tc.identifier != "" { + identifier = tc.identifier + } + sourceType := fixture.source.SourceType + if tc.sourceType != "" { + sourceType = tc.sourceType + } + grant := delegatedGmailDraftGrant(fixture, tc.permissions, sourceType, identifier) + draftID := draft.DraftID + if tc.draftID != "" { + draftID = tc.draftID + } + args := []string{tc.operation, draftID} + if tc.operation != api.CLIRunDraftGetCommand { + revision := draft.Revision + if tc.wrongRevision { + revision++ + } + args = append(args, "--revision", strconv.FormatInt(revision, 10)) + } + if tc.operation == api.CLIRunDraftEditCommand { + args = append(args, "--body", "delegated") + } + var events []api.CLIRunEvent + err := fixture.adapter.runCLIDraftLifecycle(t.Context(), api.CLIRunRequest{Args: args, Grant: grant}, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + require.Error(err) + assert.Equal("not_permitted", err.Error()) + assert.Empty(events) + assert.Equal(0, factoryCalls) + assert.Zero(fixture.client.getCalls + fixture.client.updateCalls + fixture.client.deleteCalls) + latest, loadErr := fixture.store.GetGmailDraftContext(t.Context(), draft.DraftID) + require.NoError(loadErr) + assert.Equal(draft.Revision, latest.Revision) + assert.Nil(latest.DiscardedAt) + }) + } +} + +func TestGmailDraftDelegatedUncertainEditStaysPending(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + fixture := newGmailDraftTestFixture(t) + draft := fixture.seedDraft(t) + fixture.client.updateErr = &gmail.DraftWriteError{ + State: gmail.DraftStateRemoteUnknown, Code: "remote_unknown", Err: errors.New("response lost"), + } + grant := delegatedGmailDraftGrant(fixture, []agentgrant.Permission{agentgrant.PermissionDraftEdit}, fixture.source.SourceType, fixture.source.Identifier) + args := []string{api.CLIRunDraftEditCommand, draft.DraftID, "--revision", strconv.FormatInt(draft.Revision, 10), "--body", "candidate", "--json"} + var events []api.CLIRunEvent + err := fixture.adapter.runCLIDraftLifecycle(t.Context(), api.CLIRunRequest{Args: args, Grant: grant}, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + require.Error(err) + assert.Equal("remote_unknown", err.Error()) + require.Len(events, 1) + assert.Equal(cliStreamStderr, events[0].Type) + var pendingOutput gmailDraftLifecycleOutput + require.NoError(json.Unmarshal([]byte(events[0].Data), &pendingOutput)) + assert.Equal("edit", pendingOutput.PendingOperation) + assert.Contains(pendingOutput.CandidateContent, "candidate") + + latest, err := fixture.store.GetGmailDraftContext(t.Context(), draft.DraftID) + require.NoError(err) + require.NotNil(latest.Pending) + assert.Equal("edit", latest.Pending.Operation) + updateCalls := fixture.client.updateCalls + getArgs := []string{api.CLIRunDraftGetCommand, draft.DraftID, "--json"} + events = nil + err = fixture.adapter.runCLIDraftLifecycle(t.Context(), api.CLIRunRequest{Args: getArgs, Grant: grant}, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + require.NoError(err) + require.Len(events, 1) + var getOutput gmailDraftLifecycleOutput + require.NoError(json.Unmarshal([]byte(events[0].Data), &getOutput)) + assert.Equal("edit", getOutput.PendingOperation) + assert.Contains(getOutput.CandidateContent, "candidate") + assert.Equal(updateCalls, fixture.client.updateCalls) + + grant.Permissions = []agentgrant.Permission{agentgrant.PermissionDraftDelete} + for _, asJSON := range []bool{false, true} { + args := []string{api.CLIRunDraftGetCommand, draft.DraftID} + if asJSON { + args = append(args, "--json") + } + events = nil + err = fixture.adapter.runCLIDraftLifecycle(t.Context(), api.CLIRunRequest{Args: args, Grant: grant}, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + require.NoError(err) + require.Len(events, 1) + assert.NotContains(events[0].Data, `"content"`) + assert.NotContains(events[0].Data, "content:\noriginal") + assert.NotContains(events[0].Data, "candidate") + assert.NotContains(events[0].Data, `"raw_mime"`) + assert.Contains(events[0].Data, "remote_unknown") + } +} + +func TestGmailDraftLifecycleDelegatedStaleRevisionStopsBeforeProvider(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + fixture := newGmailDraftTestFixture(t) + draft := fixture.seedDraft(t) + factoryCalls := 0 + factory := fixture.adapter.gmailDraftClientFactory + fixture.adapter.gmailDraftClientFactory = func(ctx context.Context, source *store.Source) (gmail.DraftAPI, error) { + factoryCalls++ + return factory(ctx, source) + } + grant := delegatedGmailDraftGrant(fixture, []agentgrant.Permission{agentgrant.PermissionDraftEdit}, fixture.source.SourceType, fixture.source.Identifier) + args := []string{api.CLIRunDraftEditCommand, draft.DraftID, "--revision", "2", "--body", "stale"} + var events []api.CLIRunEvent + err := fixture.adapter.runCLIDraftLifecycle(t.Context(), api.CLIRunRequest{Args: args, Grant: grant}, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + require.Error(err) + assert.Equal("revision_mismatch", err.Error()) + assert.Empty(events) + assert.Zero(factoryCalls) + assert.Zero(fixture.client.getCalls + fixture.client.updateCalls + fixture.client.deleteCalls) + latest, err := fixture.store.GetGmailDraftContext(t.Context(), draft.DraftID) + require.NoError(err) + assert.Equal(draft.Revision, latest.Revision) + assert.Nil(latest.DiscardedAt) +} + +func TestGmailDraftDeleteFinishFailureIsRetryableWithoutProviderMutation(t *testing.T) { + for _, test := range []struct { + name string + absent bool + blockedBy string + wantStatus string + wantCode string + }{ + {name: "provider confirms delete", wantStatus: "deleted", wantCode: "deleted"}, + {name: "inspection confirms absence", absent: true, wantStatus: "already_absent", wantCode: "already_absent"}, + {name: "disabled grant keeps confirmed delete pending", blockedBy: "grant", wantStatus: "deleted", wantCode: "deleted"}, + {name: "changed source keeps confirmed delete pending", blockedBy: "source", wantStatus: "deleted", wantCode: "deleted"}, + } { + t.Run(test.name, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + fixture := newSQLiteGmailDraftTestFixture(t) + draft := fixture.seedDraft(t) + if test.absent { + fixture.client.getErr = &gmail.NotFoundError{Path: "/drafts/gmail-draft-managed"} + } + _, err := fixture.store.DB().Exec(` + CREATE TRIGGER fail_gmail_draft_finish + BEFORE UPDATE OF discarded_at ON gmail_drafts + WHEN NEW.discarded_at IS NOT NULL + BEGIN + SELECT RAISE(FAIL, 'injected Gmail draft finish failure'); + END + `) + require.NoError(err) + + events, err := fixture.lifecycle(t, api.CLIRunDraftDeleteCommand, draft, "") + require.Error(err) + assert.Equal("cleanup_local_failed", err.Error()) + require.Len(events, 1) + var pendingOutput gmailDraftLifecycleOutput + require.NoError(json.Unmarshal([]byte(events[0].Data), &pendingOutput)) + assert.Equal("pending", pendingOutput.Status) + assert.Equal(test.wantCode, pendingOutput.PendingCode) + assert.Equal(draft.CurrentReceipt.GmailMessageID, pendingOutput.Receipt.GmailMessageID) + require.NotNil(pendingOutput.ProviderObservation) + assert.Equal(test.wantCode, pendingOutput.ProviderObservation.Code) + + pending, err := fixture.store.GetGmailDraftContext(t.Context(), draft.DraftID) + require.NoError(err) + require.NotNil(pending.Pending) + assert.Equal(test.wantCode, pending.Pending.Code) + getCalls := fixture.client.getCalls + deleteCalls := fixture.client.deleteCalls + + _, err = fixture.store.DB().Exec("DROP TRIGGER fail_gmail_draft_finish") + require.NoError(err) + if test.blockedBy != "" { + switch test.blockedBy { + case "grant": + fixture.adapter.gmailDraftPolicy = []config.GmailDraftSource{{SourceID: fixture.source.ID, Enabled: false}} + case "source": + _, err = fixture.store.DB().Exec("UPDATE sources SET source_type = ? WHERE id = ?", "imap", fixture.source.ID) + require.NoError(err) + } + + events, err = fixture.lifecycle(t, api.CLIRunDraftDeleteCommand, draft, "") + require.Error(err) + assert.Equal("draft_disabled", err.Error()) + assert.Empty(events) + assert.Equal(getCalls, fixture.client.getCalls) + assert.Equal(deleteCalls, fixture.client.deleteCalls) + + blocked, readErr := fixture.store.GetGmailDraftContext(t.Context(), draft.DraftID) + require.NoError(readErr) + require.NotNil(blocked.Pending) + assert.Equal(test.wantCode, blocked.Pending.Code) + assert.Nil(blocked.DiscardedAt) + + if test.blockedBy == "grant" { + fixture.adapter.gmailDraftPolicy = []config.GmailDraftSource{{SourceID: fixture.source.ID, Enabled: true}} + } else { + _, err = fixture.store.DB().Exec("UPDATE sources SET source_type = ? WHERE id = ?", sourceTypeGmail, fixture.source.ID) + require.NoError(err) + } + } + events, err = fixture.lifecycle(t, api.CLIRunDraftDeleteCommand, draft, "") + require.NoError(err) + require.Len(events, 1) + var finishedOutput gmailDraftLifecycleOutput + require.NoError(json.Unmarshal([]byte(events[0].Data), &finishedOutput)) + assert.Equal(test.wantStatus, finishedOutput.Status) + assert.Equal(getCalls, fixture.client.getCalls) + assert.Equal(deleteCalls, fixture.client.deleteCalls) + + finished, err := fixture.store.GetGmailDraftContext(t.Context(), draft.DraftID) + require.NoError(err) + assert.NotNil(finished.DiscardedAt) + assert.Nil(finished.Pending) + }) + } +} + +func TestGmailDraftUncertainOutcomeRecordFailureReturnsLocalPersistenceCode(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + fixture := newSQLiteGmailDraftTestFixture(t) + draft := fixture.seedDraft(t) + _, err := fixture.store.DB().Exec(` + CREATE TRIGGER fail_gmail_draft_uncertain_outcome + BEFORE UPDATE OF pending_code ON gmail_drafts + WHEN NEW.pending_code = 'remote_unknown' + BEGIN + SELECT RAISE(FAIL, 'injected Gmail uncertain outcome failure'); + END + `) + require.NoError(err) + fixture.client.updateErr = &gmail.DraftWriteError{ + State: gmail.DraftStateRemoteUnknown, Code: "remote_unknown", Err: errors.New("response lost"), + } + + events, err := fixture.lifecycle(t, api.CLIRunDraftEditCommand, draft, "candidate") + require.Error(err) + assert.Equal("local_persistence_failed", err.Error()) + require.Len(events, 1) + var output gmailDraftLifecycleOutput + require.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assert.Equal("pending", output.Status) + assert.Equal(store.GmailDraftOperationEdit, output.PendingOperation) + assert.Equal("remote_unknown", output.PendingCode) + assert.Equal(draft.CurrentReceipt.GmailMessageID, output.Receipt.GmailMessageID) + assert.Contains(output.CandidateContent, "candidate") + + latest, err := fixture.store.GetGmailDraftContext(t.Context(), draft.DraftID) + require.NoError(err) + require.NotNil(latest.Pending) + assert.Empty(latest.Pending.Code) + assert.Equal(1, fixture.client.updateCalls) +} + +func TestGmailDraftDeleteOutcomeRecordFailureIsRetryableAfterReinspection(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + fixture := newSQLiteGmailDraftTestFixture(t) + draft := fixture.seedDraft(t) + _, err := fixture.store.DB().Exec(` + CREATE TRIGGER fail_gmail_draft_delete_outcome + BEFORE UPDATE OF pending_code ON gmail_drafts + WHEN NEW.pending_code = 'deleted' + BEGIN + SELECT RAISE(FAIL, 'injected Gmail delete outcome failure'); + END + `) + require.NoError(err) + + events, err := fixture.lifecycle(t, api.CLIRunDraftDeleteCommand, draft, "") + require.Error(err) + assert.Equal("local_persistence_failed", err.Error()) + require.Len(events, 1) + pending, err := fixture.store.GetGmailDraftContext(t.Context(), draft.DraftID) + require.NoError(err) + require.NotNil(pending.Pending) + assert.Empty(pending.Pending.Code) + deleteCalls := fixture.client.deleteCalls + + _, err = fixture.store.DB().Exec("DROP TRIGGER fail_gmail_draft_delete_outcome") + require.NoError(err) + fixture.client.getErr = &gmail.NotFoundError{Path: "/drafts/gmail-draft-managed"} + events, err = fixture.lifecycle(t, api.CLIRunDraftDeleteCommand, draft, "") + require.NoError(err) + require.Len(events, 1) + assert.Equal(deleteCalls, fixture.client.deleteCalls) + + finished, err := fixture.store.GetGmailDraftContext(t.Context(), draft.DraftID) + require.NoError(err) + assert.NotNil(finished.DiscardedAt) + assert.Nil(finished.Pending) +} + +func TestGmailDraftExternalAdoptionReturnsFailure(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + fixture := newGmailDraftTestFixture(t) + draft := fixture.seedDraft(t) + fixture.client.getDraft = &gmail.Draft{ + ID: "gmail-draft-managed", + Message: gmail.RawMessage{ + ID: "gmail-message-external", ThreadID: "gmail-thread-1", + Raw: gmailDraftTestRaw("external", "gmail-external@example.test"), + }, + } + events, err := fixture.lifecycle(t, api.CLIRunDraftEditCommand, draft, "candidate") + require.Error(err) + assert.Equal("changed_externally", err.Error()) + require.Len(events, 1) + assert.Equal(cliStreamStderr, events[0].Type) + var output gmailDraftLifecycleOutput + require.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assert.Equal("changed_externally", output.Status) + require.NotNil(output.ProviderObservation) + assert.Equal("gmail-message-external", output.ProviderObservation.GmailMessageID) + assert.Equal(0, fixture.client.updateCalls) + adopted, err := fixture.store.GetGmailDraftContext(t.Context(), draft.DraftID) + require.NoError(err) + assert.Equal(int64(2), adopted.Revision) + assert.Equal("gmail-message-external", adopted.CurrentReceipt.GmailMessageID) +} + +func TestGmailDraftDelegatedExternalAdoptionChecksSender(t *testing.T) { + for _, operation := range []string{api.CLIRunDraftEditCommand, api.CLIRunDraftDeleteCommand} { + for _, sender := range []string{"owner@example.test", "other@example.test"} { + t.Run(operation+"/"+sender, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + fixture := newGmailDraftTestFixture(t) + draft := fixture.seedDraft(t) + fixture.client.getDraft.Message.ID = "gmail-message-external" + fixture.client.getDraft.Message.Raw = []byte(strings.Replace(string(gmailDraftTestRaw("external body", "external@example.test")), "From: owner@example.test", "From: "+sender, 1)) + permissions := []agentgrant.Permission{agentgrant.PermissionDraftEdit} + args := []string{operation, draft.DraftID, "--revision", "1", "--json"} + if operation == api.CLIRunDraftEditCommand { + args = append(args, "--body", "candidate") + } else { + permissions = []agentgrant.Permission{agentgrant.PermissionDraftCreate, agentgrant.PermissionDraftDelete} + } + grant := delegatedGmailDraftGrant(fixture, permissions, fixture.source.SourceType, fixture.source.Identifier) + var events []api.CLIRunEvent + err := fixture.adapter.runCLIDraftLifecycle(t.Context(), api.CLIRunRequest{Args: args, Grant: grant}, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + require.Error(err) + if sender == "other@example.test" { + assert.Equal("not_permitted", err.Error()) + assert.Empty(events) + } else { + assert.Equal("changed_externally", err.Error()) + require.Len(events, 1) + assert.Contains(events[0].Data, "external body") + } + assert.Zero(fixture.client.updateCalls + fixture.client.deleteCalls) + adopted, err := fixture.store.GetGmailDraftContext(t.Context(), draft.DraftID) + require.NoError(err) + assert.Equal(int64(2), adopted.Revision) + }) + } + } +} + +func TestGmailDraftExternalAdoptionPersistsAttachments(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + fixture := newGmailDraftTestFixture(t) + draft := fixture.seedDraft(t) + fixture.client.getDraft = &gmail.Draft{ + ID: "gmail-draft-managed", + Message: gmail.RawMessage{ + ID: "gmail-message-external-attachment", ThreadID: "gmail-thread-1", + Raw: gmailDraftTestRawWithAttachment("gmail-external-attachment@example.test"), + }, + } + + events, err := fixture.lifecycle(t, api.CLIRunDraftEditCommand, draft, "candidate") + require.Error(err) + assert.Equal("changed_externally", err.Error()) + require.Len(events, 1) + + adopted, err := fixture.store.GetGmailDraftContext(t.Context(), draft.DraftID) + require.NoError(err) + message, err := fixture.store.GetMessageContext(t.Context(), adopted.CurrentMessageID) + require.NoError(err) + require.Len(message.Attachments, 1) + assert.True(message.HasAttachments) + assert.Equal("notes.txt", message.Attachments[0].Filename) + assert.Equal("text/plain", message.Attachments[0].MimeType) + assert.Equal(int64(len("attachment bytes")), message.Attachments[0].Size) + assert.NotEmpty(message.Attachments[0].ContentHash) + + _, err = fixture.lifecycle(t, api.CLIRunDraftEditCommand, adopted, "candidate") + require.Error(err) + assert.Equal("invalid_draft", err.Error()) + assert.Equal(0, fixture.client.updateCalls) +} + +func TestGmailDraftCancellationClearsClaim(t *testing.T) { + for _, operation := range []string{api.CLIRunDraftEditCommand, api.CLIRunDraftDeleteCommand} { + t.Run(operation, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + fixture := newGmailDraftTestFixture(t) + draft := fixture.seedDraft(t) + ctx, cancel := context.WithCancel(t.Context()) + fixture.client.updateDraft = &gmail.Draft{ + ID: "gmail-draft-managed", + Message: gmail.RawMessage{ID: "gmail-message-edited", ThreadID: "gmail-thread-1"}, + } + fixture.client.updateErr = &gmail.DraftWriteError{ + State: gmail.DraftStateCancelled, Code: "cancelled", Err: context.Canceled, + } + if operation == api.CLIRunDraftDeleteCommand { + fixture.client.updateErr = nil + fixture.client.deleteErr = &gmail.DraftWriteError{ + State: gmail.DraftStateCancelled, Code: "cancelled", Err: context.Canceled, + } + } + fixture.client.updateHook = cancel + if operation == api.CLIRunDraftDeleteCommand { + fixture.client.updateHook = nil + fixture.client.deleteErr = &gmail.DraftWriteError{ + State: gmail.DraftStateCancelled, Code: "cancelled", Err: context.Canceled, + } + } + fixture.client.getDraft = &gmail.Draft{ + ID: "gmail-draft-managed", + Message: gmail.RawMessage{ID: "gmail-message-original", ThreadID: "gmail-thread-1"}, + } + if operation == api.CLIRunDraftDeleteCommand { + fixture.client.deleteErr = &gmail.DraftWriteError{ + State: gmail.DraftStateCancelled, Code: "cancelled", Err: context.Canceled, + } + fixture.client.deleteHook = cancel + } + events, err := fixture.lifecycleContext(ctx, operation, draft, "candidate") + require.Error(err) + assert.Equal("cancelled", err.Error()) + assert.Len(events, 1) + latest, loadErr := fixture.store.GetGmailDraftContext(t.Context(), draft.DraftID) + require.NoError(loadErr) + assert.Nil(latest.Pending) + }) + } +} + +func TestGmailDraftAcceptedResponseSurvivesCancellation(t *testing.T) { + for _, operation := range []string{api.CLIRunDraftEditCommand, api.CLIRunDraftDeleteCommand} { + t.Run(operation, func(t *testing.T) { + require := require.New(t) + fixture := newGmailDraftTestFixture(t) + draft := fixture.seedDraft(t) + ctx, cancel := context.WithCancel(t.Context()) + if operation == api.CLIRunDraftEditCommand { + fixture.client.updateDraft = &gmail.Draft{ + ID: "gmail-draft-managed", + Message: gmail.RawMessage{ID: "gmail-message-edited", ThreadID: "gmail-thread-1"}, + } + fixture.client.updateHook = cancel + } else { + fixture.client.deleteHook = cancel + } + events, err := fixture.lifecycleContext(ctx, operation, draft, "edited") + require.NoError(err) + require.Len(events, 1) + latest, loadErr := fixture.store.GetGmailDraftContext(t.Context(), draft.DraftID) + require.NoError(loadErr) + if operation == api.CLIRunDraftEditCommand { + require.Equal(int64(2), latest.Revision) + require.Equal("gmail-message-edited", latest.CurrentReceipt.GmailMessageID) + } else { + require.NotNil(latest.DiscardedAt) + } + require.Nil(latest.Pending) + }) + } +} + +func TestGmailDraftLifecycleRefreshRunsAfterOutput(t *testing.T) { + for _, operation := range []string{ + api.CLIRunDraftEditCommand, + api.CLIRunDraftDeleteCommand, + api.CLIRunDraftEditCommand + " external adoption", + } { + t.Run(operation, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + fixture := newGmailDraftTestFixture(t) + draft := fixture.seedDraft(t) + switch operation { + case api.CLIRunDraftEditCommand: + fixture.client.updateDraft = &gmail.Draft{ + ID: "gmail-draft-managed", + Message: gmail.RawMessage{ID: "gmail-message-edited", ThreadID: "gmail-thread-1"}, + } + case api.CLIRunDraftEditCommand + " external adoption": + fixture.client.getDraft = &gmail.Draft{ + ID: "gmail-draft-managed", + Message: gmail.RawMessage{ + ID: "gmail-message-external", ThreadID: "gmail-thread-1", + Raw: gmailDraftTestRaw("external", "gmail-external@example.test"), + }, + } + } + + var events []api.CLIRunEvent + refreshSawOutput := false + fixture.adapter.draftCacheRefresh = func(ctx context.Context, _ string) error { + refreshSawOutput = len(events) == 1 + require.NoError(ctx.Err()) + return nil + } + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + args := []string{api.CLIRunDraftDeleteCommand, draft.DraftID, "--revision", strconv.FormatInt(draft.Revision, 10), "--json"} + if operation == api.CLIRunDraftEditCommand || strings.HasSuffix(operation, "external adoption") { + args[0] = api.CLIRunDraftEditCommand + args = append(args, "--body", "candidate") + } + err := fixture.adapter.runCLIDraftLifecycle(ctx, api.CLIRunRequest{Args: args}, func(event api.CLIRunEvent) error { + events = append(events, event) + cancel() + return nil + }) + if strings.HasSuffix(operation, "external adoption") { + require.ErrorContains(err, "changed_externally") + } else { + require.NoError(err) + } + require.Len(events, 1) + assert.True(refreshSawOutput) + }) + } +} + +func TestGmailDraftCreateOutputsReceiptBeforeCacheRefreshAfterCancellation(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + fixture := newGmailDraftTestFixture(t) + var events []api.CLIRunEvent + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + fixture.adapter.draftCacheRefresh = func(refreshCtx context.Context, _ string) error { + assert.Len(events, 1) + require.ErrorIs(ctx.Err(), context.Canceled) + assert.NoError(refreshCtx.Err()) + return nil + } + args := []string{ + api.CLIRunDraftReplyCommand, strconv.FormatInt(fixture.parentID, 10), + "--from", fixture.source.Identifier, "--body", "created after cancellation", "--json", + } + err := fixture.adapter.runCLIReplyDraft(ctx, api.CLIRunRequest{Args: args}, func(event api.CLIRunEvent) error { + events = append(events, event) + cancel() + return nil + }) + require.NoError(err) + require.Len(events, 1) + var output gmailDraftReplyOutput + require.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assert.Equal(gmailDraftStatusCreated, output.Status) + assert.Equal("gmail-draft-created", output.GmailDraftID) +} + +func (f gmailDraftTestFixture) lifecycleContext(ctx context.Context, operation string, draft store.GmailDraft, body string) ([]api.CLIRunEvent, error) { + args := []string{operation, draft.DraftID, "--revision", strconv.FormatInt(draft.Revision, 10)} + if operation == api.CLIRunDraftEditCommand { + args = append(args, "--body", body) + } + args = append(args, "--json") + var events []api.CLIRunEvent + err := f.adapter.runCLIDraftLifecycle(ctx, api.CLIRunRequest{Args: args}, func(event api.CLIRunEvent) error { + events = append(events, event) + return nil + }) + return events, err +} + +func TestGmailDraftAcceptedReplacementReceiptIsOutputWhenPublicationFails(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + fixture := newGmailDraftTestFixture(t) + draft := fixture.seedDraft(t) + conflictRaw := gmailDraftTestRaw("conflict", "gmail-conflict@example.test") + conflictSender, err := fixture.store.EnsureParticipant("owner@example.test", "", "example.test") + require.NoError(err) + conflictTo, err := fixture.store.EnsureParticipant("sender@example.test", "", "example.test") + require.NoError(err) + _, err = fixture.store.PersistMessage(&store.MessagePersistData{ + Message: &store.Message{ + SourceID: fixture.source.ID, SourceMessageID: "gmail-message-edited", + ConversationID: fixture.conversationID, MessageType: store.MessageTypeEmail, + SenderID: sql.NullInt64{Int64: conflictSender, Valid: true}, + }, + BodyText: sql.NullString{String: "conflict", Valid: true}, RawMIME: conflictRaw, + Recipients: []store.RecipientSet{ + {Type: "from", ParticipantIDs: []int64{conflictSender}, EmailAddresses: []string{"owner@example.test"}}, + {Type: "to", ParticipantIDs: []int64{conflictTo}, EmailAddresses: []string{"sender@example.test"}}, + }, + }) + require.NoError(err) + fixture.client.updateDraft = &gmail.Draft{ + ID: "gmail-draft-managed", + Message: gmail.RawMessage{ID: "gmail-message-edited", ThreadID: "gmail-thread-1"}, + } + + events, err := fixture.lifecycle(t, api.CLIRunDraftEditCommand, draft, "candidate") + require.Error(err) + assert.Equal("accepted_local_failed", err.Error()) + require.Len(events, 1) + var output gmailDraftLifecycleOutput + require.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assert.Equal("gmail-message-edited", output.PendingReplacementGmailMessageID) + require.NotNil(output.ProviderObservation) + assert.Equal("gmail-message-edited", output.ProviderObservation.GmailMessageID) + latest, err := fixture.store.GetGmailDraftContext(t.Context(), draft.DraftID) + require.NoError(err) + require.NotNil(latest.Pending) + assert.Equal("gmail-message-edited", latest.Pending.ReplacementGmailMessageID) + + var human api.CLIRunEvent + require.NoError(emitGmailDraftLifecycleOutput(func(event api.CLIRunEvent) error { + human = event + return nil + }, cliStreamStderr, draftLifecycleIntent{}, output)) + assert.Contains(human.Data, "pending replacement Gmail message ID: gmail-message-edited") + assert.Contains(human.Data, "acknowledged replacement receipt:") +} + +func TestGmailDraftAcceptedReplacementReceiptIsOutputWhenOutcomeRecordFails(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + fixture := newSQLiteGmailDraftTestFixture(t) + draft := fixture.seedDraft(t) + _, err := fixture.store.DB().Exec(` + CREATE TRIGGER fail_gmail_draft_outcome + BEFORE UPDATE OF pending_code ON gmail_drafts + WHEN NEW.pending_code = 'accepted_local_failed' + BEGIN + SELECT RAISE(FAIL, 'injected Gmail outcome failure'); + END + `) + require.NoError(err) + fixture.client.updateDraft = &gmail.Draft{ + ID: "gmail-draft-managed", + Message: gmail.RawMessage{ID: "gmail-message-edited", ThreadID: "gmail-thread-1"}, + } + + events, err := fixture.lifecycle(t, api.CLIRunDraftEditCommand, draft, "candidate") + require.Error(err) + assert.Equal("accepted_local_failed", err.Error()) + require.Len(events, 1) + var output gmailDraftLifecycleOutput + require.NoError(json.Unmarshal([]byte(events[0].Data), &output)) + assert.Equal("accepted_local_failed", output.Status) + assert.Equal("gmail-message-original", output.Receipt.GmailMessageID) + assert.Equal("gmail-message-edited", output.PendingReplacementGmailMessageID) + require.NotNil(output.ProviderObservation) + assert.Equal("gmail-message-edited", output.ProviderObservation.GmailMessageID) + + latest, err := fixture.store.GetGmailDraftContext(t.Context(), draft.DraftID) + require.NoError(err) + require.NotNil(latest.Pending) + assert.Empty(latest.Pending.ReplacementGmailMessageID) +} + +func TestGmailDraftRemoteUnknownCreateHumanOutputIncludesRFC822ID(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + fixture := newGmailDraftTestFixture(t) + fixture.client.createErr = &gmail.DraftWriteError{ + State: gmail.DraftStateRemoteUnknown, Code: "remote_unknown", Err: errors.New("response lost"), + } + + events, err := fixture.create(t, "unknown create", false) + require.Error(err) + assert.Equal("remote_unknown", err.Error()) + require.Len(events, 1) + assert.Equal(cliStreamStderr, events[0].Type) + assert.Contains(events[0].Data, "RFC822 Message-ID: <") + assert.Contains(events[0].Data, "inspect operation") + var count int + require.NoError(fixture.store.DB().QueryRow("SELECT COUNT(*) FROM gmail_drafts").Scan(&count)) + assert.Zero(count) +} + +func TestGmailDraftPendingDeleteReconciles(t *testing.T) { + for _, code := range []string{"", "remote_unknown", "local_persistence_failed"} { + for _, absent := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/absent=%t", code, absent), func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := newGmailDraftTestFixture(t) + draft := f.seedDraft(t) + _, err := f.store.ClaimGmailDraftContext(t.Context(), draft.DraftID, draft.Revision, store.GmailDraftOperationDelete, nil) + require.NoError(err) + if code != "" { + require.NoError(f.store.RecordGmailDraftOutcomeContext(t.Context(), draft.DraftID, draft.Revision, code, "")) + } + if absent { + f.client.getErr = &gmail.NotFoundError{Path: "/drafts/gmail-draft-managed"} + } + _, err = f.lifecycle(t, api.CLIRunDraftDeleteCommand, draft, "") + require.NoError(err) + latest, err := f.store.GetGmailDraftContext(t.Context(), draft.DraftID) + require.NoError(err) + assert.Nil(latest.Pending) + assert.NotNil(latest.DiscardedAt) + assert.Equal(1, f.client.getCalls) + if absent { + assert.Zero(f.client.deleteCalls) + } else { + assert.Equal(1, f.client.deleteCalls) + } + }) + } + } +} + +func TestGmailDraftPendingEditReconciles(t *testing.T) { + for _, observed := range []string{"original", "replacement", "synced_replacement", "external", "absent"} { + t.Run(observed, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := newGmailDraftTestFixture(t) + draft := f.seedDraft(t) + raw := gmailDraftTestRaw("candidate", "candidate@example.test") + _, err := f.store.ClaimGmailDraftContext(t.Context(), draft.DraftID, draft.Revision, store.GmailDraftOperationEdit, raw) + require.NoError(err) + code, replacementID := "remote_unknown", "" + if observed == "replacement" || observed == "synced_replacement" { + code, replacementID = "accepted_local_failed", "gmail-message-replacement" + } + require.NoError(f.store.RecordGmailDraftOutcomeContext(t.Context(), draft.DraftID, draft.Revision, code, replacementID)) + switch observed { + case "replacement", "synced_replacement": + f.client.getDraft.Message.ID = "gmail-message-replacement" + f.client.getDraft.Message.Raw = raw + case "external": + f.client.getDraft.Message.ID = "gmail-message-external" + f.client.getDraft.Message.Raw = gmailDraftTestRaw("external", "external@example.test") + case "absent": + f.client.getErr = &gmail.NotFoundError{Path: "/drafts/gmail-draft-managed"} + } + var syncedMessageID int64 + if observed == "synced_replacement" { + senderID, senderErr := f.store.EnsureParticipant("owner@example.test", "", "example.test") + require.NoError(senderErr) + syncedMessageID, err = f.store.PersistMessage(&store.MessagePersistData{ + Message: &store.Message{ + SourceID: f.source.ID, SourceMessageID: "gmail-message-replacement", + MessageType: store.MessageTypeEmail, ConversationID: f.conversationID, + SenderID: sql.NullInt64{Int64: senderID, Valid: true}, + }, + BodyText: sql.NullString{String: "candidate", Valid: true}, RawMIME: raw, + }) + require.NoError(err) + } + f.client.updateDraft = &gmail.Draft{ID: "gmail-draft-managed", Message: gmail.RawMessage{ID: "gmail-message-new", ThreadID: "gmail-thread-1"}} + events, err := f.lifecycle(t, api.CLIRunDraftEditCommand, draft, "new body") + switch observed { + case "original": + require.NoError(err) + assert.Equal(1, f.client.updateCalls) + case "replacement", "synced_replacement": + require.ErrorContains(err, "revision_mismatch") + require.Len(events, 1) + assert.Contains(events[0].Data, `"status":"recovered"`) + assert.Zero(f.client.updateCalls) + case "external": + require.ErrorContains(err, "changed_externally") + assert.Zero(f.client.updateCalls) + case "absent": + require.ErrorContains(err, "provider_absent") + assert.Zero(f.client.updateCalls) + } + latest, err := f.store.GetGmailDraftContext(t.Context(), draft.DraftID) + require.NoError(err) + if observed == "absent" { + require.NotNil(latest.Pending) + assert.Equal(raw, latest.Pending.Raw) + _, err = f.lifecycle(t, api.CLIRunDraftDeleteCommand, latest, "") + require.NoError(err) + latest, err = f.store.GetGmailDraftContext(t.Context(), draft.DraftID) + require.NoError(err) + assert.NotNil(latest.DiscardedAt) + assert.Zero(f.client.deleteCalls) + } + assert.Nil(latest.Pending) + if observed == "replacement" || observed == "synced_replacement" || observed == "external" { + assert.Equal(f.client.getDraft.Message.ID, latest.CurrentReceipt.GmailMessageID) + if observed == "synced_replacement" { + assert.Equal(syncedMessageID, latest.CurrentMessageID) + } + assert.Equal(int64(2), latest.Revision) + message, err := f.store.GetMessageContext(t.Context(), latest.CurrentMessageID) + require.NoError(err) + wantBody := "candidate" + if observed == "external" { + wantBody = "external" + } + assert.Contains(message.BodyText, wantBody) + } + }) + } +} + +func TestGmailDraftSendAsRefusesDelegatedGrant(t *testing.T) { + f := newGmailDraftTestFixture(t) + err := f.adapter.runCLIDraftSendAs(t.Context(), api.CLIRunRequest{ + Args: []string{api.CLIRunDraftSendAsCommand, f.source.Identifier}, Grant: &agentgrant.Grant{}, + }, nil) + require.ErrorContains(t, err, "not_permitted") + assert.Zero(t, f.client.listCalls) +} + +func TestGmailDraftLifecycleReportsStoreReadFailure(t *testing.T) { + f := newSQLiteGmailDraftTestFixture(t) + draft := f.seedDraft(t) + _, err := f.store.DB().Exec("DROP TABLE gmail_drafts") + require.NoError(t, err) + err = f.adapter.runCLIDraftLifecycle(t.Context(), api.CLIRunRequest{Args: []string{api.CLIRunDraftGetCommand, draft.DraftID}}, nil) + require.ErrorContains(t, err, "draft_read_failed") +} diff --git a/cmd/msgvault/cmd/serve_gmail_draft_test.go b/cmd/msgvault/cmd/serve_gmail_draft_test.go new file mode 100644 index 000000000..9aa3d4692 --- /dev/null +++ b/cmd/msgvault/cmd/serve_gmail_draft_test.go @@ -0,0 +1,125 @@ +package cmd + +import ( + "errors" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/gmail" +) + +func TestAuthorizeGmailDraftRequiresEnabledSourcePolicy(t *testing.T) { + requirements := require.New(t) + requirements.ErrorContains(authorizeGmailDraft(nil, 42, "gmail"), "draft_disabled") + requirements.ErrorContains(authorizeGmailDraft([]config.GmailDraftSource{{SourceID: 42}}, 42, "gmail"), "draft_disabled") + requirements.NoError(authorizeGmailDraft([]config.GmailDraftSource{{SourceID: 42, Enabled: true}}, 42, "gmail")) + requirements.ErrorContains(authorizeGmailDraft([]config.GmailDraftSource{{SourceID: 42, Enabled: true}}, 42, "imap"), "draft_disabled") +} + +func TestValidateGmailSendAsRequiresPrimaryOrAccepted(t *testing.T) { + requirements := require.New(t) + entries := []gmail.SendAs{ + {Email: "pending@example.com", VerificationStatus: "pending"}, + {Email: "accepted@example.com", VerificationStatus: "accepted"}, + {Email: "primary@example.com", Primary: true, VerificationStatus: "pending"}, + } + requirements.NoError(validateGmailSendAs(entries, "ACCEPTED@example.com")) + requirements.NoError(validateGmailSendAs(entries, "primary@example.com")) + requirements.ErrorContains(validateGmailSendAs(entries, "pending@example.com"), "invalid_from") + requirements.ErrorContains(validateGmailSendAs(entries, "missing@example.com"), "invalid_from") +} + +func TestParseDraftSendAsArgs(t *testing.T) { + assertions := assert.New(t) + requirements := require.New(t) + account, jsonOutput, err := parseDraftSendAsArgs([]string{"draft-send-as", "alice@example.com", "--json"}) + requirements.NoError(err) + assertions.Equal("alice@example.com", account) + assertions.True(jsonOutput) + _, _, err = parseDraftSendAsArgs([]string{"draft-send-as"}) + requirements.ErrorContains(err, "invalid_args") +} + +func TestGmailReadErrorCodeUsesProviderMessages(t *testing.T) { + tests := []struct { + name string + err error + want string + }{ + { + name: "access token scope insufficient", + err: errors.New("googleapi: Error 403: ACCESS_TOKEN_SCOPE_INSUFFICIENT"), + want: "insufficient_scope", + }, + { + name: "insufficient authentication scopes", + err: errors.New("googleapi: Error 403: insufficient authentication scopes"), + want: "insufficient_scope", + }, + { + name: "insufficient permission", + err: errors.New("googleapi: Error 403: Insufficient Permission"), + want: "insufficient_scope", + }, + { + name: "generic insufficient text", + err: errors.New("provider returned insufficient data"), + want: "provider_refused", + }, + { + name: "draft not found", + err: &gmail.NotFoundError{Path: "/drafts/draft-1"}, + want: "provider_absent", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, gmailReadErrorCode(tt.err)) + }) + } +} + +func TestEmitGmailDraftLifecycleHumanPendingOutput(t *testing.T) { + assertions := assert.New(t) + requirements := require.New(t) + output := gmailDraftLifecycleOutput{ + Status: "pending", + Provider: "gmail", + DraftID: "draft-1", + Revision: 2, + Lifecycle: "active", + Receipt: gmailDraftLifecycleReceipt{GmailDraftID: "gmail-draft-1", GmailMessageID: "gmail-message-1", ThreadID: "thread-1"}, + Content: "current content\n", + CandidateContent: "candidate content\n", + PendingOperation: "edit", + PendingCode: "remote_unknown", + Observation: &gmailDraftLifecycleObservation{ + State: "unknown", + Code: "remote_unknown", + GmailDraftID: "gmail-draft-1", + GmailMessageID: "gmail-message-1", + ThreadID: "thread-1", + Present: true, + }, + } + + var event api.CLIRunEvent + err := emitGmailDraftLifecycleOutput(func(got api.CLIRunEvent) error { + event = got + return nil + }, cliStreamStderr, draftLifecycleIntent{}, output) + requirements.NoError(err) + assertions.Equal(cliStreamStderr, event.Type) + assertions.Contains(event.Data, "receipt (revision 2): gmail_draft_id=gmail-draft-1") + assertions.Contains(event.Data, "pending operation: edit") + assertions.Contains(event.Data, "candidate content:\ncandidate content") + assertions.Contains(event.Data, "old provider receipt: state=unknown code=remote_unknown") + assertions.Contains(event.Data, "provider outcome: remote_unknown") + assertions.Contains(event.Data, "retry draft-edit with --revision 2 to reconcile with Gmail") + assertions.Equal(1, strings.Count(event.Data, "provider outcome:")) +} diff --git a/cmd/msgvault/cmd/serve_lifecycle.go b/cmd/msgvault/cmd/serve_lifecycle.go index 978d3b2fe..ed920792c 100644 --- a/cmd/msgvault/cmd/serve_lifecycle.go +++ b/cmd/msgvault/cmd/serve_lifecycle.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "log/slog" "net" "net/http" "os" @@ -63,22 +64,38 @@ func newLifecycleCommand(name string, hidden bool) *cobra.Command { case "start": cmd.Short = "Start msgvault daemon in the background" cmd.RunE = func(cmd *cobra.Command, _ []string) error { - return runServeStart(cmd, cfg) + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + return runServeStart(cmd, state.cfg) } case statusValue: cmd.Short = "Show msgvault daemon status" cmd.RunE = func(cmd *cobra.Command, _ []string) error { - return runServeStatusWithAPIKey(cmd, cfg.Data.DataDir, cfg.Server.APIKey) + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + return runServeStatusWithAPIKey(cmd, state.cfg.Data.DataDir, bestEffortLifecycleAPIKey(state.cfg)) } case "stop": cmd.Short = "Stop msgvault daemon" cmd.RunE = func(cmd *cobra.Command, _ []string) error { - return runServeStopWithAPIKey(cmd, cfg.Data.DataDir, cfg.Server.APIKey) + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + return runServeStopWithAPIKey(cmd, state.cfg.Data.DataDir, bestEffortLifecycleAPIKey(state.cfg)) } case "restart": cmd.Short = "Restart msgvault daemon in the background" cmd.RunE = func(cmd *cobra.Command, _ []string) error { - return runServeRestart(cmd, cfg) + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + return runServeRestart(cmd, state.cfg) } default: panic("unknown daemon lifecycle command: " + name) @@ -86,6 +103,15 @@ func newLifecycleCommand(name string, hidden bool) *cobra.Command { return cmd } +// bestEffortLifecycleAPIKey lets status omit authenticated health details and +// stop use runtime shutdown credentials when the configured key is unavailable. +func bestEffortLifecycleAPIKey(cfg *config.Config) string { + if err := cfg.ResolveServerKey(); err != nil { + return "" + } + return cfg.Server.AuthenticationKey() +} + func addServeLifecycleCommands(parent *cobra.Command) { for _, name := range []string{"start", statusValue, "stop", "restart"} { parent.AddCommand(newLifecycleCommand(name, true)) @@ -248,17 +274,26 @@ type backgroundDaemonStartPreparation struct { type backgroundServeStartOptions struct { ExecutablePath string CacheBuildIntent startupCacheBuildIntent + Invocation *invocationOptions } func prepareBackgroundDaemonStart( c *config.Config, + restartPolicy string, incompatibleGuidance string, + logger *slog.Logger, ) (backgroundDaemonStartPreparation, error) { + if _, err := resolveServeBind(c.Server.BindAddr); err != nil { + return backgroundDaemonStartPreparation{}, err + } + if err := c.ResolveServerKey(); err != nil { + return backgroundDaemonStartPreparation{}, err + } if rt := findDaemonRuntime(c.Data.DataDir); rt != nil { - if !shouldUpgradeDaemonRuntimeWithPolicy(rt, Version, c.Server.DaemonAutoRestart) { + if !shouldUpgradeDaemonRuntimeWithPolicy(rt, Version, restartPolicy) { return backgroundDaemonStartPreparation{Reusable: rt}, nil } - if err := stopDaemonRuntimeForUpgrade(*c, rt); err != nil { + if err := stopDaemonRuntimeForUpgrade(*c, rt, logger); err != nil { return backgroundDaemonStartPreparation{}, fmt.Errorf("stop older daemon before restart: %w", err) } } @@ -267,10 +302,10 @@ func prepareBackgroundDaemonStart( return backgroundDaemonStartPreparation{}, fmt.Errorf("inspect daemon runtimes: %w", compatErr) } if foundIncompatible { - if !shouldUpgradeIncompatibleDaemonRuntimeWithPolicy(rt, Version, c.Server.DaemonAutoRestart) { + if !shouldUpgradeIncompatibleDaemonRuntimeWithPolicy(rt, Version, restartPolicy) { return backgroundDaemonStartPreparation{}, incompatibleDaemonError(compatErr, incompatibleGuidance) } - if err := stopDaemonRuntimeForUpgrade(*c, rt); err != nil { + if err := stopDaemonRuntimeForUpgrade(*c, rt, logger); err != nil { return backgroundDaemonStartPreparation{}, fmt.Errorf("stop older daemon before restart: %w", err) } } @@ -285,17 +320,17 @@ func prepareBackgroundDaemonStart( } if foundLegacy { if legacyCompatErr == nil { - if !shouldUpgradeDaemonRuntimeWithPolicy(legacy, Version, c.Server.DaemonAutoRestart) { + if !shouldUpgradeDaemonRuntimeWithPolicy(legacy, Version, restartPolicy) { return backgroundDaemonStartPreparation{Reusable: legacy}, nil } } else if !shouldUpgradeIncompatibleDaemonRuntimeWithPolicy( - legacy, Version, c.Server.DaemonAutoRestart, + legacy, Version, restartPolicy, ) { return backgroundDaemonStartPreparation{}, incompatibleDaemonError( legacyCompatErr, incompatibleGuidance, ) } - if err := stopDaemonRuntimeForUpgrade(*c, legacy); err != nil { + if err := stopDaemonRuntimeForUpgrade(*c, legacy, logger); err != nil { return backgroundDaemonStartPreparation{}, fmt.Errorf("stop older daemon before restart: %w", err) } } @@ -324,6 +359,9 @@ func runServeStartWithOptions(cmd *cobra.Command, c *config.Config, opts backgro if c == nil { return errors.New("nil config") } + if inv := invocationFromCommand(cmd); inv != nil { + opts.Invocation = &inv.options + } if err := os.MkdirAll(c.Data.DataDir, 0o700); err != nil { return fmt.Errorf("create data directory: %w", err) } @@ -335,7 +373,7 @@ func runServeStartWithOptions(cmd *cobra.Command, c *config.Config, opts backgro } defer func() { _ = launchLock.Unlock() }() - prep, err := prepareBackgroundDaemonStart(c, "run `msgvault daemon stop` before starting this version") + prep, err := prepareBackgroundDaemonStart(c, c.Server.DaemonAutoRestart, "run `msgvault daemon stop` before starting this version", loggerFromContext(cmd.Context())) if err != nil { return err } @@ -391,7 +429,10 @@ func runServeRestart(cmd *cobra.Command, c *config.Config) error { if c == nil { return errors.New("nil config") } - if err := stopLiveDaemonsWithAPIKey(cmd, c.Data.DataDir, c.Server.APIKey, true); err != nil { + if err := prepareServeConfig(c); err != nil { + return err + } + if err := stopLiveDaemonsWithAPIKey(cmd, c.Data.DataDir, c.Server.AuthenticationKey(), true); err != nil { return err } return runServeStart(cmd, c) @@ -402,6 +443,7 @@ func stopLiveDaemons(cmd *cobra.Command, dataDir string, quietNoDaemon bool) err } func stopLiveDaemonsWithAPIKey(cmd *cobra.Command, dataDir string, apiKey string, quietNoDaemon bool) error { + logger := loggerFromContext(cmd.Context()) records, err := listLiveDaemonRuntimeRecords(dataDir) if err != nil { return err @@ -415,7 +457,7 @@ func stopLiveDaemonsWithAPIKey(cmd *cobra.Command, dataDir string, apiKey string stopped := 0 skipped := 0 for _, rec := range records { - if err := stopDaemonRuntimeRecord(cmd.OutOrStdout(), dataDir, rec, apiKey, serveStopGraceTimeout); err != nil { + if err := stopDaemonRuntimeRecord(cmd.OutOrStdout(), dataDir, rec, apiKey, serveStopGraceTimeout, logger); err != nil { if !errors.Is(err, errDaemonIdentityUnconfirmed) { return fmt.Errorf("stop pid %d: %w", rec.PID, err) } @@ -435,12 +477,15 @@ func stopLiveDaemonsWithAPIKey(cmd *cobra.Command, dataDir string, apiKey string return nil } -func stopDaemonRuntimeForUpgradeImpl(c config.Config, rt *DaemonRuntime) error { +func stopDaemonRuntimeForUpgradeImpl(c config.Config, rt *DaemonRuntime, logger *slog.Logger) error { if rt == nil { return nil } + if err := prepareServeConfig(&c); err != nil { + return fmt.Errorf("validate replacement daemon: %w", err) + } if err := stopDaemonRuntimeRecord(os.Stdout, c.Data.DataDir, rt.Record, - c.Server.APIKey, serveStopGraceTimeout); err != nil { + c.Server.AuthenticationKey(), serveStopGraceTimeout, logger); err != nil { return fmt.Errorf("stop pid %d: %w", rt.Record.PID, err) } return nil @@ -452,10 +497,11 @@ func stopDaemonRuntimeRecord( rec daemon.RuntimeRecord, apiKey string, grace time.Duration, + logger *slog.Logger, ) error { switch runtimeRecordIdentity(rec) { case createTimeMatch: - return stopDaemonProcess(out, rec, apiKey, grace) + return stopDaemonProcess(out, rec, apiKey, grace, logger) case createTimeMismatch: proof, err := probeDaemonRuntimeIdentity(context.Background(), rec) if err != nil { @@ -509,7 +555,8 @@ func processIdentityConfirmed(rec daemon.RuntimeRecord) bool { return processCreateTimeMatches(rec.PID, rec.Metadata[runtimeCreateTime]) } -func stopDaemonProcess(out io.Writer, rec daemon.RuntimeRecord, apiKey string, grace time.Duration) error { +func stopDaemonProcess(out io.Writer, rec daemon.RuntimeRecord, apiKey string, grace time.Duration, logger *slog.Logger) error { + logger = repairLogger(logger) if !processIdentityConfirmed(rec) { return fmt.Errorf("cannot confirm pid %d is the recorded msgvault daemon", rec.PID) } @@ -861,7 +908,11 @@ func startServeBackgroundProcess(c *config.Config, opts backgroundServeStartOpti } defer func() { _ = devNull.Close() }() - child := newServeBackgroundCommandForRun(exe, serveBackgroundChildArgs()...) + var childOptions invocationOptions + if opts.Invocation != nil { + childOptions = *opts.Invocation + } + child := newServeBackgroundCommandForRun(exe, serveBackgroundChildArgs(childOptions)...) child.Env = withStartupCacheBuildIntent( append(os.Environ(), "MSGVAULT_HOME="+c.HomeDir, serveBackgroundChildEnv+"=1"), opts.CacheBuildIntent, @@ -954,31 +1005,35 @@ func stopBackgroundServeStartup(proc *backgroundServeProcess, grace time.Duratio } } -func serveBackgroundChildArgs() []string { +func serveBackgroundChildArgs(options ...invocationOptions) []string { + var o invocationOptions + if len(options) > 0 { + o = options[0] + } args := make([]string, 0, 16) - if cfgFile != "" { - args = append(args, "--config", cfgFile) + if o.cfgFile != "" { + args = append(args, "--config", o.cfgFile) } - if homeDir != "" { - args = append(args, "--home", homeDir) + if o.homeDir != "" { + args = append(args, "--home", o.homeDir) } - if verbose { + if o.verbose { args = append(args, "--verbose") } - if logFile != "" { - args = append(args, "--log-file", logFile) + if o.logFile != "" { + args = append(args, "--log-file", o.logFile) } - if logLevel != "" { - args = append(args, "--log-level", logLevel) + if o.logLevel != "" { + args = append(args, "--log-level", o.logLevel) } - if noLogFile { + if o.noLogFile { args = append(args, "--no-log-file") } - if logSQL { + if o.logSQL { args = append(args, "--log-sql") } - if logSQLSlow != 0 { - args = append(args, "--log-sql-slow-ms", strconv.FormatInt(logSQLSlow, 10)) + if o.logSQLSlow != 0 { + args = append(args, "--log-sql-slow-ms", strconv.FormatInt(o.logSQLSlow, 10)) } return append(args, "serve") } diff --git a/cmd/msgvault/cmd/serve_lifecycle_test.go b/cmd/msgvault/cmd/serve_lifecycle_test.go index 4f51ba855..f53ee59e2 100644 --- a/cmd/msgvault/cmd/serve_lifecycle_test.go +++ b/cmd/msgvault/cmd/serve_lifecycle_test.go @@ -17,6 +17,7 @@ import ( "sync" "sync/atomic" "testing" + "testing/synctest" "time" "github.com/spf13/cobra" @@ -40,7 +41,8 @@ func TestNativeOperationRecoveryRunsBeforeDaemonServices(t *testing.T) { _, err = st.StartSync(source.ID, "incremental") require.NoError(err) _, err = st.StartCardDAVSyncRunContext(t.Context(), store.CardDAVSyncRunStart{ - Trigger: store.CardDAVSyncTriggerScheduled, + AccountID: store.DefaultCardDAVAccountID, + Trigger: store.CardDAVSyncTriggerScheduled, }) require.NoError(err) ledgers := []struct { @@ -100,15 +102,19 @@ func TestDaemonAndServeLifecycleCommandSurfaces(t *testing.T) { } func TestDaemonAndServeStatusHaveIdenticalBehavior(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) dataDir := t.TempDir() oldCfg := cfg cfg = lifecycleTestConfig(dataDir) t.Cleanup(func() { cfg = oldCfg }) + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) run := func(args ...string) (string, error) { root := newTestRootCmd() + root.SetContext(testCtx) root.SilenceUsage = true root.AddCommand(newDaemonCommand()) compatServe := &cobra.Command{Use: "serve"} @@ -118,7 +124,7 @@ func TestDaemonAndServeStatusHaveIdenticalBehavior(t *testing.T) { root.SetOut(&stdout) root.SetErr(io.Discard) root.SetArgs(args) - err := root.ExecuteContext(context.Background()) + err := root.ExecuteContext(testCtx) return stdout.String(), err } @@ -249,6 +255,8 @@ func TestRunServeStatusIncludesVectorHealth(t *testing.T) { } func TestServeStatusCommandUsesAuthenticatedHealthForOperationDetails(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) dataDir := t.TempDir() @@ -304,7 +312,7 @@ func TestServeStatusCommandUsesAuthenticatedHealthForOperationDetails(t *testing t.Cleanup(func() { cfg = oldCfg }) cmd, stdout, stderr := lifecycleTestCommand() - cmd.SetContext(context.Background()) + cmd.SetContext(testInvocationContext(context.Background(), cfg, invocationOptions{})) statusCmd, _, err := serveCmd.Find([]string{"status"}) require.NoError(err, "find serve status") require.NoError(statusCmd.RunE(cmd, nil), "serve status") @@ -318,6 +326,59 @@ func TestServeStatusCommandUsesAuthenticatedHealthForOperationDetails(t *testing assert.Empty(stderr.String(), "status must not write to stderr") } +func TestDaemonStatusContinuesWhenServerAPIKeyFileIsUnavailable(t *testing.T) { + clearServerKeyEnvironment(t) + require := require.New(t) + assert := assert.New(t) + dataDir := t.TempDir() + cfg := lifecycleTestConfig(dataDir) + cfg.Server.APIKeyFile = filepath.Join(dataDir, "missing-api-key") + + var publicHealthRequests atomic.Int32 + mux := http.NewServeMux() + mux.Handle("/api/ping", daemon.NewPingHandler(daemon.PingHandlerOptions{ + Service: daemonService, + Version: Version, + })) + mux.HandleFunc("/api/v1/health", func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "unauthorized", http.StatusUnauthorized) + }) + mux.HandleFunc("/health", func(w http.ResponseWriter, _ *http.Request) { + publicHealthRequests.Add(1) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"status":"ok","vector":{"status":"initializing"}}`)) + }) + server := httptest.NewServer(mux) + t.Cleanup(server.Close) + host, portText, err := net.SplitHostPort(server.Listener.Addr().String()) + require.NoError(err) + port, err := strconv.Atoi(portText) + require.NoError(err) + _, err = daemonRuntimeStore(dataDir).Write(daemon.RuntimeRecord{ + PID: os.Getpid(), + Network: daemon.NetworkTCP, + Address: net.JoinHostPort(host, portText), + Service: daemonService, + Version: Version, + Metadata: map[string]string{ + runtimeHost: host, + runtimePort: strconv.Itoa(port), + runtimeAPIVersion: strconv.Itoa(daemonAPIVersion), + runtimeAPISchemaVersion: api.APISchemaVersion, + runtimeCreateTime: matchingProcessCreateTime(t), + }, + }) + require.NoError(err) + + command, stdout, _ := lifecycleTestCommand() + command.SetContext(testInvocationContext(t.Context(), cfg, invocationOptions{})) + status := newLifecycleCommand("status", false) + require.NoError(status.RunE(command, nil)) + assert.Contains(stdout.String(), "msgvault running at") + assert.Contains(stdout.String(), "vector: initializing") + assert.Positive(publicHealthRequests.Load(), "public health details remain available without the configured key") +} + func TestFetchDaemonOperationUsesAuthenticatedHealth(t *testing.T) { assert := assert.New(t) require := require.New(t) @@ -514,7 +575,7 @@ func TestStopDaemonRuntimeRecordRejectsUnprovedCreateTimeMismatch(t *testing.T) }, } - err = stopDaemonRuntimeRecord(io.Discard, dataDir, rec, "configured-api-key", 10*time.Millisecond) + err = stopDaemonRuntimeRecord(io.Discard, dataDir, rec, "configured-api-key", 10*time.Millisecond, testDiscardLogger()) require.ErrorIs(err, errDaemonIdentityUnconfirmed, "unproved mismatch must be rejected") assert.Positive(proofRequests.Load(), "mismatched endpoint is challenged") @@ -545,6 +606,10 @@ func TestStopLiveDaemonsUsesLegacyShutdownWhenCreateTimeSkewed(t *testing.T) { testStopLiveDaemonsUsesAuthenticatedHTTP(t, "6000", 5_000, true, false) } +func TestDaemonStopUsesRuntimeTokenWhenServerAPIKeyFileIsUnavailable(t *testing.T) { + testStopLiveDaemons(t, "", 0, false, true, true) +} + func testStopLiveDaemonsUsesAuthenticatedHTTP( t *testing.T, recordedCreateTime string, @@ -553,6 +618,20 @@ func testStopLiveDaemonsUsesAuthenticatedHTTP( identityEndpointSupported bool, ) { t.Helper() + testStopLiveDaemons(t, recordedCreateTime, liveCreateTime, liveCreateTimeOK, + identityEndpointSupported, false) +} + +func testStopLiveDaemons( + t *testing.T, + recordedCreateTime string, + liveCreateTime int64, + liveCreateTimeOK bool, + identityEndpointSupported bool, + serverAPIKeyFileUnavailable bool, +) { + t.Helper() + clearServerKeyEnvironment(t) require := require.New(t) assert := assert.New(t) dataDir := t.TempDir() @@ -594,7 +673,7 @@ func testStopLiveDaemonsUsesAuthenticatedHTTP( shutdownTokens <- r.Header.Get(api.DaemonShutdownTokenHeader) w.WriteHeader(http.StatusAccepted) go func() { - time.Sleep(25 * time.Millisecond) + time.Sleep(25 * time.Millisecond) //nolint:kennlint // holds the OS ownership lock past the reply releaseOwner.Do(func() { _ = owner.Close() }) }() default: @@ -624,9 +703,15 @@ func testStopLiveDaemonsUsesAuthenticatedHTTP( }) require.NoError(err, "write runtime record") cmd, stdout, _ := lifecycleTestCommand() - - require.NoError(stopLiveDaemonsWithAPIKey(cmd, dataDir, "configured-api-key", false), - "stop daemon with indeterminate process identity") + cfg := lifecycleTestConfig(dataDir) + if serverAPIKeyFileUnavailable { + cfg.Server.APIKeyFile = filepath.Join(dataDir, "missing-api-key") + } else { + cfg.Server.APIKey = "configured-api-key" + } + cmd.SetContext(testInvocationContext(t.Context(), cfg, invocationOptions{})) + stop := newLifecycleCommand("stop", false) + require.NoError(stop.RunE(cmd, nil), "stop daemon with indeterminate process identity") select { case got := <-shutdownTokens: @@ -701,7 +786,7 @@ func TestStopDaemonRuntimeRecordNeverSignalsProvedCreateTimeMismatchOnShutdownFa }, } - err = stopDaemonRuntimeRecord(io.Discard, dataDir, rec, "configured-api-key", 10*time.Millisecond) + err = stopDaemonRuntimeRecord(io.Discard, dataDir, rec, "configured-api-key", 10*time.Millisecond, testDiscardLogger()) require.Error(err, "failed authenticated shutdown remains an error") require.ErrorContains(err, tt.wantError) @@ -922,7 +1007,11 @@ func TestRunServeStartAlreadyRunningWritesOnlyStdout(t *testing.T) { err, "write runtime") cmd, stdout, stderr := lifecycleTestCommand() - require.NoError(runServeStart(cmd, lifecycleTestConfig(dataDir))) + cfg := lifecycleTestConfig(dataDir) + cfg.Server.BindAddr = "0.0.0.0" + require.NoError(runServeStart(cmd, cfg)) + _, err = os.Stat(cfg.ServerKeyFilePath()) + require.ErrorIs(err, os.ErrNotExist, "reusing a daemon must not mint a replacement key") assert.Equal( "msgvault already running at http://"+net.JoinHostPort(server.Host, portText)+ " (pid "+strconv.Itoa(os.Getpid())+")\n", @@ -1029,7 +1118,7 @@ func TestRunServeStartDoesNotDowngradeNewerDaemon(t *testing.T) { require.NoError( err, "write runtime") - stubStopDaemonRuntimeForUpgrade(t, func(config.Config, *DaemonRuntime) error { + stubStopDaemonRuntimeForUpgrade(t, func(config.Config, *DaemonRuntime, *slog.Logger) error { require.Fail("older CLI must not stop a newer daemon") return nil }) @@ -1073,7 +1162,7 @@ func TestRunServeStartUpgradesOlderDaemon(t *testing.T) { err, "write runtime") var stoppedPID int - stubStopDaemonRuntimeForUpgrade(t, func(_ config.Config, rt *DaemonRuntime) error { + stubStopDaemonRuntimeForUpgrade(t, func(_ config.Config, rt *DaemonRuntime, _ *slog.Logger) error { stoppedPID = rt.Record.PID return nil }) @@ -1107,6 +1196,67 @@ func TestRunServeStartUpgradesOlderDaemon(t *testing.T) { assert.Empty(stderr.String()) } +func TestRunServeStartIgnoresDisabledDaemonAutoStart(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + withTestVersion(t, "v1.1.0") + dataDir := t.TempDir() + server := httptestPingDaemon(t) + portText := strconv.Itoa(server.Port) + _, err := daemonRuntimeStore(dataDir).Write(daemon.RuntimeRecord{ + PID: os.Getpid(), + Network: daemon.NetworkTCP, + Address: net.JoinHostPort(server.Host, portText), + Service: daemonService, + Version: "v1.0.0", + Metadata: map[string]string{ + runtimeHost: server.Host, + runtimePort: portText, + runtimeAPIVersion: strconv.Itoa(daemonAPIVersion), + runtimeAPISchemaVersion: api.APISchemaVersion, + runtimeCreateTime: matchingProcessCreateTime(t), + }, + }) + require.NoError(err, "write runtime") + + var stoppedPID int + stubStopDaemonRuntimeForUpgrade(t, func(_ config.Config, rt *DaemonRuntime, _ *slog.Logger) error { + stoppedPID = rt.Record.PID + return nil + }) + waitCh := make(chan error) + stubStartServeBackgroundProcess(t, func(*config.Config, backgroundServeStartOptions) (*backgroundServeProcess, error) { + return &backgroundServeProcess{ + PID: 777, + LogPath: "/tmp/msgvault-serve.log", + Wait: waitCh, + }, nil + }) + stubWaitForBackgroundServeReady(t, func( + context.Context, + string, + <-chan error, + time.Duration, + ) (*DaemonRuntime, bool, error) { + return &DaemonRuntime{ + Record: daemon.RuntimeRecord{PID: 777}, + Host: "127.0.0.1", + Port: 9090, + }, true, nil + }) + c := lifecycleTestConfig(dataDir) + c.Server.DaemonAutoStart = new(false) + cmd, stdout, stderr := lifecycleTestCommand() + require.NoError(runServeStart(cmd, c)) + assert.Equal(os.Getpid(), stoppedPID, "explicit start still stops an older daemon") + assert.Equal( + "msgvault running at http://127.0.0.1:9090 (pid 777)\n"+ + "Logs: /tmp/msgvault-serve.log\n", + stdout.String()) + assert.Empty(stderr.String()) +} + func TestRunServeStartHonorsNeverAutoRestartPolicy(t *testing.T) { assert := assert.New(t) require := @@ -1133,7 +1283,7 @@ func TestRunServeStartHonorsNeverAutoRestartPolicy(t *testing.T) { require.NoError( err, "write runtime") - stubStopDaemonRuntimeForUpgrade(t, func(config.Config, *DaemonRuntime) error { + stubStopDaemonRuntimeForUpgrade(t, func(config.Config, *DaemonRuntime, *slog.Logger) error { require.FailNow("never policy must not stop a compatible daemon") return errors.New("unreachable") }) @@ -1178,7 +1328,7 @@ func TestRunServeStartUpgradesOlderIncompatibleDaemon(t *testing.T) { err, "write runtime") var stoppedPID int - stubStopDaemonRuntimeForUpgrade(t, func(_ config.Config, rt *DaemonRuntime) error { + stubStopDaemonRuntimeForUpgrade(t, func(_ config.Config, rt *DaemonRuntime, _ *slog.Logger) error { stoppedPID = rt.Record.PID return nil }) @@ -1237,7 +1387,7 @@ func TestRunServeStartRefusesNewerIncompatibleDaemon(t *testing.T) { require.NoError( err, "write runtime") - stubStopDaemonRuntimeForUpgrade(t, func(config.Config, *DaemonRuntime) error { + stubStopDaemonRuntimeForUpgrade(t, func(config.Config, *DaemonRuntime, *slog.Logger) error { require.FailNow("older CLI must not stop a newer incompatible daemon") return errors.New("unreachable") }) @@ -1299,6 +1449,73 @@ func TestRunServeRestartStartsWhenNoDaemonIsRunning(t *testing.T) { assert.Empty(t, stderr.String()) } +func TestDaemonReplacementRejectsInvalidConfigBeforeStopping(t *testing.T) { + for _, tt := range []struct { + name string + bind string + agentAccess bool + upgrade bool + wantError string + }{ + {"restart missing interface", "iface:msgvault-nonexistent-restart-interface", false, false, "resolve bind interface"}, + {"restart missing agent key", "127.0.0.1", true, false, "agent_access"}, + {"upgrade missing agent key", "127.0.0.1", true, true, "agent_access"}, + } { + t.Run(tt.name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + dataDir := t.TempDir() + standIn := startBlockingDaemonStandIn(t) + created, ok := processCreateTimeMillis(standIn.Process.Pid) + require.True(ok, "read stand-in process create time") + record := daemon.RuntimeRecord{ + PID: standIn.Process.Pid, + Network: daemon.NetworkTCP, + Address: net.JoinHostPort("127.0.0.1", "1"), + Service: daemonService, + Version: Version, + Metadata: map[string]string{ + runtimeHost: "127.0.0.1", + runtimePort: "1", + runtimeAPIVersion: strconv.Itoa(daemonAPIVersion), + runtimeAPISchemaVersion: api.APISchemaVersion, + runtimeCreateTime: strconv.FormatInt(created, 10), + runtimeShutdownToken: "stand-in-shutdown-token", + }, + } + _, err := daemonRuntimeStore(dataDir).Write(record) + require.NoError(err, "write live daemon runtime") + + var shutdownRequests atomic.Int32 + previousShutdown := requestDaemonShutdownForRun + requestDaemonShutdownForRun = func(daemon.RuntimeRecord) (bool, error) { + shutdownRequests.Add(1) + if err := standIn.Process.Kill(); err != nil { + return false, err + } + _ = standIn.Wait() + return true, nil + } + t.Cleanup(func() { requestDaemonShutdownForRun = previousShutdown }) + stubStartServeBackgroundProcess(t, func(*config.Config, backgroundServeStartOptions) (*backgroundServeProcess, error) { + return nil, errors.New("invalid replacement must not be launched") + }) + cfg := lifecycleTestConfig(dataDir) + cfg.Server.BindAddr = tt.bind + cfg.Server.AgentAccess = tt.agentAccess + if tt.upgrade { + err = stopDaemonRuntimeForUpgradeImpl(*cfg, daemonRuntimeFromRecord(record), slog.New(slog.DiscardHandler)) + } else { + cmd, _, _ := lifecycleTestCommand() + err = runServeRestart(cmd, cfg) + } + require.ErrorContains(err, tt.wantError) + assert.Zero(shutdownRequests.Load(), "reject an invalid replacement before stopping the daemon") + assert.True(daemon.ProcessAlive(standIn.Process.Pid), "the existing daemon must keep running") + }) + } +} + func TestRunServeStartNotReadyPrintsWebUIURLForFixedPort(t *testing.T) { dataDir := t.TempDir() waitCh := make(chan error) @@ -1509,7 +1726,7 @@ func TestNewDaemonIdleTrackerOnlyRunsForBackgroundServeChild(t *testing.T) { tracker := newDaemonIdleTracker(cfg, func() { require.FailNow(t, "foreground serve must not arm idle shutdown") - }) + }, testDiscardLogger()) assert.Nil(t, tracker) } @@ -1520,7 +1737,7 @@ func TestNewDaemonIdleTrackerUsesServerConfigTimeout(t *testing.T) { cfg.Server.DaemonIdleTimeout = 20 * time.Millisecond fired := make(chan struct{}) - tracker := newDaemonIdleTracker(cfg, func() { close(fired) }) + tracker := newDaemonIdleTracker(cfg, func() { close(fired) }, testDiscardLogger()) require.NotNil(t, tracker) go tracker.Run(t.Context()) @@ -1540,7 +1757,7 @@ func TestNewDaemonIdleTrackerEnvOverrideDisables(t *testing.T) { tracker := newDaemonIdleTracker(cfg, func() { require.FailNow(t, "idle tracker fired despite env disable") - }) + }, testDiscardLogger()) assert.Nil(t, tracker) } @@ -1554,6 +1771,15 @@ func lifecycleTestCommand() (*cobra.Command, *bytes.Buffer, *bytes.Buffer) { return cmd, stdout, stderr } +func clearServerKeyEnvironment(t *testing.T) { + t.Helper() + for _, name := range []string{"MSGVAULT_API_KEY", "MSGVAULT_API_KEY_FILE", "MSGVAULT_API_KEY_ENV"} { + value, _ := os.LookupEnv(name) + t.Setenv(name, value) + require.NoError(t, os.Unsetenv(name), "clear %s", name) + } +} + func runtimeDataDirFile(t *testing.T) string { t.Helper() path := filepath.Join(t.TempDir(), "data-file") @@ -1570,7 +1796,7 @@ func withTestVersion(t *testing.T, version string) { func stubStopDaemonRuntimeForUpgrade( t *testing.T, - fn func(config.Config, *DaemonRuntime) error, + fn func(config.Config, *DaemonRuntime, *slog.Logger) error, ) { t.Helper() old := stopDaemonRuntimeForUpgrade @@ -1702,15 +1928,17 @@ func TestWaitForDaemonExitWithProgressExplainsLongStops(t *testing.T) { } func TestWaitForDaemonExitWithProgressGivesUpAtGrace(t *testing.T) { - restoreStopWaitPacing(t, 5*time.Millisecond, 10*time.Millisecond) - out := &bytes.Buffer{} + synctest.Test(t, func(t *testing.T) { + restoreStopWaitPacing(t, 5*time.Millisecond, 10*time.Millisecond) + out := &bytes.Buffer{} - exited := waitForDaemonExitWithProgress(out, daemon.RuntimeRecord{PID: 4242}, nil, - 50*time.Millisecond, time.Millisecond, - func(daemon.RuntimeRecord) bool { return true }) + exited := waitForDaemonExitWithProgress(out, daemon.RuntimeRecord{PID: 4242}, nil, + 50*time.Millisecond, time.Millisecond, + func(daemon.RuntimeRecord) bool { return true }) - assert.False(t, exited, "wait must give up at the grace deadline") - assert.Contains(t, out.String(), "Waiting up to") + assert.False(t, exited, "wait must give up at the grace deadline") + assert.Contains(t, out.String(), "Waiting up to") + }) } func TestWaitForDaemonExitWithProgressQuietOnFastExit(t *testing.T) { diff --git a/cmd/msgvault/cmd/serve_lifecycle_windows_test.go b/cmd/msgvault/cmd/serve_lifecycle_windows_test.go index 6f15ca4ee..a161b6959 100644 --- a/cmd/msgvault/cmd/serve_lifecycle_windows_test.go +++ b/cmd/msgvault/cmd/serve_lifecycle_windows_test.go @@ -17,6 +17,8 @@ import ( "golang.org/x/sys/windows" ) +const suspendedProcessObservationBudget = 500 * time.Millisecond + func TestWindowsBackgroundProcessDoesNotRunBeforeJobAttachment(t *testing.T) { assert := assert.New(t) require := require.New(t) @@ -39,14 +41,14 @@ func TestWindowsBackgroundProcessDoesNotRunBeforeJobAttachment(t *testing.T) { assert.Never(func() bool { _, statErr := os.Stat(pidPath) return statErr == nil || !errors.Is(statErr, os.ErrNotExist) - }, 500*time.Millisecond, 10*time.Millisecond, + }, suspendedProcessObservationBudget, 10*time.Millisecond, "daemon work must not begin before Job Object attachment") require.NoError(tree.Attach(cmd.Process), "attach and resume parent helper") require.Eventually(func() bool { _, statErr := os.Stat(pidPath) return statErr == nil - }, 10*time.Second, 25*time.Millisecond, "blocking child PID") + }, serveLifecycleTestTimeout, 25*time.Millisecond, "blocking child PID") } func TestStopBackgroundServeStartupTerminatesWindowsProcessTree(t *testing.T) { @@ -76,7 +78,7 @@ func TestStopBackgroundServeStartupTerminatesWindowsProcessTree(t *testing.T) { } childPID, readErr = strconv.Atoi(strings.TrimSpace(string(contents))) return readErr == nil && childPID > 0 - }, 10*time.Second, 25*time.Millisecond, "blocking child PID") + }, serveLifecycleTestTimeout, 25*time.Millisecond, "blocking child PID") child, err := windows.OpenProcess(windows.SYNCHRONIZE, false, uint32(childPID)) require.NoError(err, "open blocking child") t.Cleanup(func() { _ = windows.CloseHandle(child) }) diff --git a/cmd/msgvault/cmd/serve_meetings.go b/cmd/msgvault/cmd/serve_meetings.go new file mode 100644 index 000000000..84d584be6 --- /dev/null +++ b/cmd/msgvault/cmd/serve_meetings.go @@ -0,0 +1,29 @@ +package cmd + +import ( + "context" + + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/meetingcontent" + "go.kenn.io/msgvault/internal/store" +) + +func (a *storeAPIAdapter) GetMeetingContextContext( + ctx context.Context, scope store.MeetingQueryScope, options meetingcontent.PacketOptions, +) (*meetingcontent.PacketResult, error) { + return a.store.GetMeetingContextContext(ctx, scope, options) +} + +func (a *storeAPIAdapter) ListMeetingActionsContext( + ctx context.Context, request store.MeetingActionsQuery, +) (*meetingcontent.ActionsPage, error) { + return a.store.ListMeetingActionsContext(ctx, request) +} + +func (a *storeAPIAdapter) GetMeetingMetricsContext( + ctx context.Context, scope store.MeetingQueryScope, +) (*meetingcontent.Metrics, error) { + return a.store.GetMeetingMetricsContext(ctx, scope) +} + +var _ api.MeetingStore = (*storeAPIAdapter)(nil) diff --git a/cmd/msgvault/cmd/serve_meetings_test.go b/cmd/msgvault/cmd/serve_meetings_test.go new file mode 100644 index 000000000..b94cd2a9e --- /dev/null +++ b/cmd/msgvault/cmd/serve_meetings_test.go @@ -0,0 +1,44 @@ +package cmd + +import ( + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/testutil" +) + +func TestStoreAPIAdapterServesMeetingMetricsThroughRealDaemonRoute(t *testing.T) { + st := testutil.NewTestStore(t) + srv := api.NewServerWithOptions(api.ServerOptions{ + Config: &config.Config{}, Store: &storeAPIAdapter{store: st}, Logger: slog.New(slog.DiscardHandler), + }) + request := httptest.NewRequest(http.MethodPost, "/api/v1/meetings/metrics", strings.NewReader(`{}`)) + request.Header.Set("Content-Type", "application/json") + response := httptest.NewRecorder() + + srv.Router().ServeHTTP(response, request) + + require.Equal(t, http.StatusOK, response.Code, response.Body.String()) + archiveUID, err := st.ArchiveUID() + require.NoError(t, err) + assert.JSONEq(t, `{ + "schema_version":1, + "archive_uid":"`+archiveUID+`", + "totals":{"meeting_count":0,"known_duration_count":0,"unknown_duration_count":0,"total_known_seconds":0,"average_known_seconds":null}, + "first_meeting_at":null, + "last_meeting_at":null, + "undated_count":0, + "duration_by_basis":[], + "months":[], + "scope":{"kind":"direct"} + }`, response.Body.String()) +} + +var _ api.MeetingStore = (*storeAPIAdapter)(nil) diff --git a/cmd/msgvault/cmd/serve_ownership.go b/cmd/msgvault/cmd/serve_ownership.go index 954c7b717..867d51353 100644 --- a/cmd/msgvault/cmd/serve_ownership.go +++ b/cmd/msgvault/cmd/serve_ownership.go @@ -50,7 +50,12 @@ func claimServeOwnership( return nil, err } } - record, shutdownToken, err := writeDaemonRuntime(cfg.Data.DataDir, host, port, version, cfg.Server.APIKey) + // A rejected contender must not mint a key that changes how clients + // authenticate to the daemon that already owns this archive. + if err := cfg.PrepareServerKey(); err != nil { + return nil, errors.Join(err, lock.Close(), daemonLock.Close()) + } + record, shutdownToken, err := writeDaemonRuntime(cfg.Data.DataDir, host, port, version, cfg.Server.AuthenticationKey()) if err != nil { _ = lock.Close() _ = daemonLock.Close() diff --git a/cmd/msgvault/cmd/serve_ownership_test.go b/cmd/msgvault/cmd/serve_ownership_test.go index 184aeae03..8c91956d2 100644 --- a/cmd/msgvault/cmd/serve_ownership_test.go +++ b/cmd/msgvault/cmd/serve_ownership_test.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -38,7 +39,9 @@ func TestServeOwnershipEnsureRuntimeRecordRepublishesMissingRecord(t *testing.T) dataDir := t.TempDir() cfg := &config.Config{Data: config.DataConfig{DataDir: dataDir}} - owner, err := claimServeOwnership(context.Background(), cfg, "127.0.0.1", 8123, "v-test") + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx + owner, err := claimServeOwnership(testCtx, cfg, "127.0.0.1", 8123, "v-test") require.NoError(err, "claimServeOwnership") t.Cleanup(func() { require.NoError(owner.Close(), "close ownership") }) @@ -70,68 +73,46 @@ func TestServeOwnershipEnsureRuntimeRecordRepublishesMissingRecord(t *testing.T) } func TestRuntimeRecordHeartbeatRepublishesUntilCancelled(t *testing.T) { - require := require.New(t) - - dataDir := t.TempDir() - cfg := &config.Config{Data: config.DataConfig{DataDir: dataDir}} - owner, err := claimServeOwnership(context.Background(), cfg, "127.0.0.1", 8123, "v-test") - require.NoError(err, "claimServeOwnership") - t.Cleanup(func() { require.NoError(owner.Close(), "close ownership") }) - - path, err := daemonRuntimeStore(dataDir).Path(owner.record.PID) - require.NoError(err, "runtime record path") - require.NoError(os.Remove(path), "remove runtime record") - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - done := make(chan struct{}) - go func() { - defer close(done) - runtimeRecordHeartbeat(ctx, owner, 10*time.Millisecond) - }() - - require.Eventually(func() bool { + synctest.Test(t, func(t *testing.T) { + require := require.New(t) + dataDir := t.TempDir() + cfg := &config.Config{Data: config.DataConfig{DataDir: dataDir}} + owner, err := claimServeOwnership(context.Background(), cfg, "127.0.0.1", 8123, "v-test") + require.NoError(err, "claimServeOwnership") + path, err := daemonRuntimeStore(dataDir).Path(owner.record.PID) + require.NoError(err, "runtime record path") + require.NoError(os.Remove(path), "remove runtime record") + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { defer close(done); runtimeRecordHeartbeat(ctx, owner, 10*time.Millisecond) }() + t.Cleanup(func() { cancel(); <-done; require.NoError(owner.Close(), "close ownership") }) + synctest.Sleep(10 * time.Millisecond) + synctest.Wait() _, statErr := os.Stat(path) - return statErr == nil - }, 5*time.Second, 10*time.Millisecond, "heartbeat republishes the pruned record") - - cancel() - select { - case <-done: - case <-time.After(5 * time.Second): - require.FailNow("heartbeat did not stop after context cancellation") - } + require.NoError(statErr, "heartbeat republishes the pruned record") + }) } func TestRuntimeRecordHeartbeatDoesNotRepublishAfterOwnershipClose(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - dataDir := t.TempDir() - cfg := &config.Config{Data: config.DataConfig{DataDir: dataDir}} - owner, err := claimServeOwnership(context.Background(), cfg, "127.0.0.1", 8123, "v-test") - require.NoError(err, "claimServeOwnership") - - path, err := daemonRuntimeStore(dataDir).Path(owner.record.PID) - require.NoError(err, "runtime record path") - - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - defer close(done) - runtimeRecordHeartbeat(ctx, owner, time.Millisecond) - }() - t.Cleanup(func() { - cancel() - <-done - }) - - require.NoError(owner.Close(), "close ownership") - require.NoError(owner.SetStartupPhase("still starting"), "startup phase update after close") - assert.Never(func() bool { + synctest.Test(t, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + dataDir := t.TempDir() + cfg := &config.Config{Data: config.DataConfig{DataDir: dataDir}} + owner, err := claimServeOwnership(context.Background(), cfg, "127.0.0.1", 8123, "v-test") + require.NoError(err, "claimServeOwnership") + path, err := daemonRuntimeStore(dataDir).Path(owner.record.PID) + require.NoError(err, "runtime record path") + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { defer close(done); runtimeRecordHeartbeat(ctx, owner, time.Millisecond) }() + t.Cleanup(func() { cancel(); <-done; require.NoError(owner.Close(), "close ownership") }) + require.NoError(owner.Close(), "close ownership") + require.NoError(owner.SetStartupPhase("still starting"), "startup phase update after close") + synctest.Sleep(100 * time.Millisecond) _, statErr := os.Stat(path) - return statErr == nil - }, 100*time.Millisecond, time.Millisecond, "closed ownership must stay unpublished") + assert.ErrorIs(statErr, os.ErrNotExist, "closed ownership must stay unpublished") + }) } func TestRuntimeRecordHeartbeatSerializesStartupPhaseUpdates(t *testing.T) { @@ -140,14 +121,16 @@ func TestRuntimeRecordHeartbeatSerializesStartupPhaseUpdates(t *testing.T) { dataDir := t.TempDir() cfg := &config.Config{Data: config.DataConfig{DataDir: dataDir}} - owner, err := claimServeOwnership(context.Background(), cfg, "127.0.0.1", 8123, "v-test") + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx + owner, err := claimServeOwnership(testCtx, cfg, "127.0.0.1", 8123, "v-test") require.NoError(err, "claimServeOwnership") t.Cleanup(func() { require.NoError(owner.Close(), "close ownership") }) path, err := daemonRuntimeStore(dataDir).Path(owner.record.PID) require.NoError(err, "runtime record path") - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(testCtx) done := make(chan struct{}) go func() { defer close(done) @@ -177,8 +160,10 @@ func TestClaimServeOwnershipLocksAndPublishesRuntime(t *testing.T) { dataDir := t.TempDir() cfg := &config.Config{Data: config.DataConfig{DataDir: dataDir}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx - owner, err := claimServeOwnership(context.Background(), cfg, "127.0.0.1", 8123, "v-test") + owner, err := claimServeOwnership(testCtx, cfg, "127.0.0.1", 8123, "v-test") require.NoError( err, "claimServeOwnership") @@ -215,7 +200,9 @@ func TestServeOwnershipStartupPhaseUpdatesRuntimeRecord(t *testing.T) { dataDir := t.TempDir() cfg := &config.Config{Data: config.DataConfig{DataDir: dataDir}} - owner, err := claimServeOwnership(context.Background(), cfg, "127.0.0.1", 8123, "v-test") + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx + owner, err := claimServeOwnership(testCtx, cfg, "127.0.0.1", 8123, "v-test") require.NoError(err, "claimServeOwnership") t.Cleanup(func() { require.NoError(owner.Close(), "close ownership") }) @@ -246,7 +233,9 @@ func TestServeOwnershipStartupCacheBuildOutcomeUpdatesRuntimeRecord(t *testing.T dataDir := t.TempDir() cfg := &config.Config{Data: config.DataConfig{DataDir: dataDir}} - owner, err := claimServeOwnership(context.Background(), cfg, "127.0.0.1", 8123, "v-test") + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx + owner, err := claimServeOwnership(testCtx, cfg, "127.0.0.1", 8123, "v-test") require.NoError(err, "claimServeOwnership") t.Cleanup(func() { require.NoError(owner.Close(), "close ownership") }) @@ -272,12 +261,14 @@ func TestServeOwnershipStartupCacheBuildOutcomeUpdatesRuntimeRecord(t *testing.T func TestClaimServeOwnershipRejectsSecondOwner(t *testing.T) { dataDir := t.TempDir() cfg := &config.Config{Data: config.DataConfig{DataDir: dataDir}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx first, err := tryAcquireWriteOwnerLock(dataDir) require.NoError(t, err, "pre-held lock") t.Cleanup(func() { require.NoError(t, first.Close(), "close pre-held lock") }) - owner, err := claimServeOwnership(context.Background(), cfg, "127.0.0.1", 8123, "v-test") + owner, err := claimServeOwnership(testCtx, cfg, "127.0.0.1", 8123, "v-test") assert.Nil(t, owner, "ownership") require.ErrorAs(t, err, &writeOwnerLockHeldError{}, "error type") } @@ -290,8 +281,10 @@ func TestClaimServeOwnershipSkipsSQLiteLockForPostgreSQL(t *testing.T) { DataDir: dataDir, DatabaseURL: "postgres://user:pass@example.com:5432/msgvault", }} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx - owner, err := claimServeOwnership(context.Background(), cfg, "127.0.0.1", 8123, "v-test") + owner, err := claimServeOwnership(testCtx, cfg, "127.0.0.1", 8123, "v-test") require.NoError( err, "claimServeOwnership") @@ -320,14 +313,16 @@ func TestClaimServeOwnershipRejectsSecondPostgreSQLDaemon(t *testing.T) { DataDir: dataDir, DatabaseURL: "postgres://user:pass@example.com:5432/msgvault", }} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx - owner, err := claimServeOwnership(context.Background(), cfg, "127.0.0.1", 8123, "v-test") + owner, err := claimServeOwnership(testCtx, cfg, "127.0.0.1", 8123, "v-test") require.NoError( err, "claimServeOwnership") t.Cleanup(func() { require.NoError(owner.Close(), "close ownership") }) - second, err := claimServeOwnership(context.Background(), cfg, "127.0.0.1", 8124, "v-test") + second, err := claimServeOwnership(testCtx, cfg, "127.0.0.1", 8124, "v-test") assert.Nil(second, "second owner") require.Error(err, "second PostgreSQL daemon should be rejected") assert.Contains(err.Error(), "daemon", "error names daemon ownership") diff --git a/cmd/msgvault/cmd/serve_people_sweep.go b/cmd/msgvault/cmd/serve_people_sweep.go index b17b81b76..a1e998235 100644 --- a/cmd/msgvault/cmd/serve_people_sweep.go +++ b/cmd/msgvault/cmd/serve_people_sweep.go @@ -5,6 +5,7 @@ import ( "errors" "net/http" "os" + "path/filepath" "time" "github.com/google/uuid" @@ -111,9 +112,10 @@ func newProductionPersonSweepWorker( func newProductionStructuredRunner( cfg *config.Config, st *store.Store, ) (*peoplesweep.Runner, error) { - registry, err := peoplesweep.NewDriverRegistry( + registry, err := peoplesweep.NewDriverRegistryWithCodexAuthHome( http.DefaultClient, peoplesweep.NewCodexCommandStarter(), peoplesweep.NewReleasedCodexIsolationGate(), + filepath.Join(cfg.TokensDir(), "people-codex"), ) if err != nil { return nil, err diff --git a/cmd/msgvault/cmd/serve_reply_draft.go b/cmd/msgvault/cmd/serve_reply_draft.go index 9264e90b3..c9dc8c5e3 100644 --- a/cmd/msgvault/cmd/serve_reply_draft.go +++ b/cmd/msgvault/cmd/serve_reply_draft.go @@ -6,6 +6,7 @@ import ( "encoding/json/v2" "errors" "fmt" + "net/mail" "strconv" "strings" "time" @@ -15,11 +16,11 @@ import ( "go.kenn.io/msgvault/internal/config" imaplib "go.kenn.io/msgvault/internal/imap" "go.kenn.io/msgvault/internal/mime" + "go.kenn.io/msgvault/internal/sourceops" "go.kenn.io/msgvault/internal/store" ) var ( - errDraftReplyFromRequired = errors.New("--from is required") errDraftReplyBodyRequired = errors.New("--body is required") ) @@ -29,23 +30,32 @@ const ( ) type draftReplyIntent struct { - MessageID int64 - From string - Body string - JSON bool + MessageID int64 + From string + Body string + JSON bool + ReplyAll bool + Account string + SourceID int64 + SourceIDSet bool } // draftReplyTarget is the archived parent message and the granted source // mailbox that will hold the reply. type draftReplyTarget struct { - parent *store.APIMessage - source *store.Source - mailbox string - raw []byte + parent *store.APIMessage + parentSource *store.Source + source *store.Source + mailbox string + raw []byte + forward bool + attachmentWrites *[]store.AttachmentWrite } type draftReplyOutput struct { Status string `json:"status"` + DraftID string `json:"draft_id,omitempty"` + Revision int64 `json:"revision,omitzero"` MessageID int64 `json:"message_id,omitzero"` OperationRef string `json:"operation_ref"` RFC822MessageID string `json:"rfc822_message_id"` @@ -75,7 +85,7 @@ func parseDraftReplyArgs(args []string) (draftReplyIntent, error) { } var intent draftReplyIntent var positional string - var fromSet, bodySet, jsonSet bool + var fromSet, bodySet, jsonSet, allSet, accountSet, sourceIDSet bool rest := args[1:] for len(rest) > 0 { arg := rest[0] @@ -90,21 +100,39 @@ func parseDraftReplyArgs(args []string) (draftReplyIntent, error) { } name, value, hasValue := strings.Cut(nameValue, "=") switch name { - case "from", "body": + case draftFromFlag, "body", "account", "source-id": if !hasValue { if len(rest) == 0 { return invalidDraftReplyArgs("--%s requires a value", name) } value, rest = rest[0], rest[1:] } - if (name == "from" && fromSet) || (name == "body" && bodySet) { + if (name == draftFromFlag && fromSet) || (name == "body" && bodySet) || + (name == "account" && accountSet) || (name == "source-id" && sourceIDSet) { return invalidDraftReplyArgs("--%s given more than once", name) } - if name == "from" { + switch name { + case draftFromFlag: intent.From, fromSet = value, true - } else { + case "body": intent.Body, bodySet = value, true + case "account": + if strings.TrimSpace(value) == "" { + return invalidDraftReplyArgs("--account must not be empty") + } + intent.Account, accountSet = strings.TrimSpace(value), true + case "source-id": + id, parseErr := strconv.ParseInt(strings.TrimSpace(value), 10, 64) + if parseErr != nil || id <= 0 { + return invalidDraftReplyArgs("source ID must be a positive integer") + } + intent.SourceID, intent.SourceIDSet, sourceIDSet = id, true, true + } + case "all": + if allSet || (hasValue && value != "true") { + return invalidDraftReplyArgs("--all accepts one flag without a value") } + intent.ReplyAll, allSet = true, true case "json": if jsonSet { return invalidDraftReplyArgs("--json given more than once") @@ -124,8 +152,8 @@ func parseDraftReplyArgs(args []string) (draftReplyIntent, error) { return invalidDraftReplyArgs("unknown flag --%s", name) } } - if !fromSet || intent.From == "" { - return invalidDraftReplyArgs("--from is required") + if fromSet && intent.From == "" { + return invalidDraftReplyArgs("--from must not be empty") } if !bodySet { return invalidDraftReplyArgs("--body is required") @@ -135,6 +163,9 @@ func parseDraftReplyArgs(args []string) (draftReplyIntent, error) { return invalidDraftReplyArgs("message ID must be a positive integer") } intent.MessageID = id + if intent.SourceIDSet && intent.Account != "" { + return invalidDraftReplyArgs("--account and --source-id are mutually exclusive") + } return intent, nil } @@ -161,13 +192,187 @@ func authorizeIMAPDraft(policy []config.IMAPDraftSource, sourceID int64, sourceT return "", draftReplyError("draft_disabled", fmt.Errorf("source %d has no enabled [[imap.drafts]] grant", sourceID)) } -func hasConfirmedSourceIdentity(identities []store.AccountIdentity, address string) bool { +func draftSourceRef(source *store.Source) agentgrant.SourceRef { + return agentgrant.SourceRef{ID: source.ID, Type: source.SourceType, Identifier: source.Identifier} +} + +func parseDraftSender(value string) (*mail.Address, string, error) { + addresses, err := mail.ParseAddressList(strings.TrimSpace(value)) + if err != nil || len(addresses) != 1 || addresses[0] == nil || addresses[0].Address == "" { + return nil, "", errors.New("expected exactly one mailbox identity") + } + address := addresses[0] + if !strings.Contains(address.Address, "@") || strings.ContainsAny(address.Address, "\r\n") { + return nil, "", errors.New("mailbox identity is malformed") + } + return address, store.NormalizeIdentifierForCompare(address.Address), nil +} + +func confirmedDraftIdentities(identities []store.AccountIdentity) (map[string]string, []string) { + selected := make(map[string]string, len(identities)) + all := make([]string, 0, len(identities)) for _, identity := range identities { - if !identity.ConfirmedAt.IsZero() && store.EqualIdentifier(identity.Address, address) { - return true + if identity.ConfirmedAt.IsZero() { + continue } + address, key, err := parseDraftSender(identity.Address) + if err != nil { + continue + } + if _, seen := selected[key]; seen { + continue + } + selected[key] = address.String() + all = append(all, address.Address) } - return false + return selected, all +} + +func (a *storeAPIAdapter) selectDraftSender( + identities []store.AccountIdentity, + requested string, + grant *agentgrant.Grant, + source *store.Source, +) (string, []string, error) { + eligible, selfAddresses := confirmedDraftIdentities(identities) + ref := draftSourceRef(source) + if requested != "" { + address, key, err := parseDraftSender(requested) + if err != nil { + return "", nil, draftReplyError("invalid_from", err) + } + if _, ok := eligible[key]; !ok { + return "", nil, draftReplyError("invalid_from", errors.New("--from is not a confirmed identity on the selected source")) + } + if grant != nil && !grant.AllowsSender(agentgrant.PermissionDraftCreate, ref, key) { + return "", nil, draftReplyNotPermitted(errors.New("selected sender is not in the grant")) + } + return address.String(), selfAddresses, nil + } + + candidates := make([]string, 0, len(eligible)) + for key, value := range eligible { + if grant != nil && !grant.AllowsSender(agentgrant.PermissionDraftCreate, ref, key) { + continue + } + candidates = append(candidates, value) + } + if len(candidates) == 0 { + if grant != nil { + return "", nil, draftReplyNotPermitted(errors.New("the grant has no eligible sender identity on the selected source")) + } + return "", nil, draftReplyError("invalid_from", errors.New("the selected source has no confirmed mailbox identity")) + } + if len(candidates) > 1 { + return "", nil, draftReplyError("from_ambiguous", errors.New("--from is required when the selected source has multiple eligible identities")) + } + return candidates[0], selfAddresses, nil +} + +// resolveDraftTarget performs source, grant, sender, policy, and provider +// configuration checks before it reads an archived parent or opens IMAP. +func (a *storeAPIAdapter) resolveDraftTarget( + ctx context.Context, + parentID *int64, + account string, + sourceID int64, + sourceIDSet bool, + requestedFrom string, + grant *agentgrant.Grant, +) (draftReplyTarget, string, []string, error) { + var parentSource *store.Source + if parentID != nil { + var err error + parentSource, err = a.store.GetMessageSourceContext(ctx, *parentID) + if err != nil { + if grant != nil { + return draftReplyTarget{}, "", nil, draftReplyNotPermitted(errors.New("parent source is not available")) + } + return draftReplyTarget{}, "", nil, draftReplyError("invalid_parent", fmt.Errorf("load parent source: %w", err)) + } + if err := authorizeDelegatedDraftSource(grant, parentSource); err != nil { + return draftReplyTarget{}, "", nil, err + } + } + + var source *store.Source + var err error + if sourceIDSet || strings.TrimSpace(account) != "" { + source, err = sourceops.ResolveExactOne(a.store, sourceops.Selector{ + Account: account, SourceID: sourceID, SourceIDSet: sourceIDSet, + }) + if err != nil { + if grant != nil { + return draftReplyTarget{}, "", nil, draftReplyNotPermitted(errors.New("destination source is not available")) + } + return draftReplyTarget{}, "", nil, draftReplyError("invalid_source", fmt.Errorf("resolve destination source: %w", err)) + } + } else if parentSource != nil && (parentSource.SourceType == "imap" || parentSource.SourceType == "gmail") { + source = parentSource + } else if parentSource != nil { + return draftReplyTarget{}, "", nil, draftReplyError("invalid_source", errors.New("an offline parent requires --account or --source-id for a live IMAP destination")) + } else { + return draftReplyTarget{}, "", nil, draftReplyError("invalid_source", errors.New("--account or --source-id is required")) + } + + if err := authorizeDelegatedDraftSource(grant, source); err != nil { + return draftReplyTarget{}, "", nil, err + } + identities, err := a.store.ListAccountIdentitiesContext(ctx, source.ID) + if err != nil { + return draftReplyTarget{}, "", nil, draftReplyError("invalid_from", fmt.Errorf("list identities for source %d: %w", source.ID, err)) + } + from, selfAddresses, err := a.selectDraftSender(identities, requestedFrom, grant, source) + if err != nil { + return draftReplyTarget{}, "", nil, err + } + var mailbox string + switch source.SourceType { + case "imap": + mailbox, err = authorizeIMAPDraft(a.draftPolicy, source.ID, source.SourceType) + if err != nil { + return draftReplyTarget{}, "", nil, err + } + if !source.SyncConfig.Valid { + return draftReplyTarget{}, "", nil, draftReplyError("invalid_source", fmt.Errorf("source %d has no sync config", source.ID)) + } + imapConfig, err := imaplib.ConfigFromJSON(source.SyncConfig.String) + if err != nil { + return draftReplyTarget{}, "", nil, draftReplyError("invalid_source", fmt.Errorf("source %d sync config: %w", source.ID, err)) + } + if imapConfig.Identifier() != source.Identifier { + return draftReplyTarget{}, "", nil, draftReplyError("invalid_source", fmt.Errorf("source %d sync config identifier does not match the source", source.ID)) + } + case "gmail": + if parentSource == nil { + return draftReplyTarget{}, "", nil, draftReplyError("draft_disabled", errors.New("draft-compose requires an IMAP source")) + } + if parentSource.ID != source.ID { + return draftReplyTarget{}, "", nil, draftReplyError("draft_disabled", errors.New("gmail draft replies must use the parent source")) + } + if err := authorizeGmailDraft(a.gmailDraftPolicy, source.ID, source.SourceType); err != nil { + return draftReplyTarget{}, "", nil, err + } + default: + return draftReplyTarget{}, "", nil, draftReplyError("draft_disabled", fmt.Errorf("source %d is a %q source", source.ID, source.SourceType)) + } + + target := draftReplyTarget{parentSource: parentSource, source: source, mailbox: mailbox} + if parentID != nil { + parent, err := a.store.GetMessageContext(ctx, *parentID) + if err != nil { + return draftReplyTarget{}, "", nil, draftReplyError("invalid_parent", fmt.Errorf("load message %d: %w", *parentID, err)) + } + if !store.IsEmailMessageType(parent.MessageType) { + return draftReplyTarget{}, "", nil, draftReplyError("invalid_parent", errors.New("parent message is not an email")) + } + raw, err := a.store.GetMessageRawContext(ctx, parent.ID) + if err != nil { + return draftReplyTarget{}, "", nil, draftReplyError("invalid_parent", fmt.Errorf("load raw MIME for message %d: %w", parent.ID, err)) + } + target.parent, target.raw = parent, raw + } + return target, from, selfAddresses, nil } func (a *storeAPIAdapter) runCLIReplyDraft( @@ -182,70 +387,30 @@ func (a *storeAPIAdapter) runCLIReplyDraft( if err != nil { return err } - target, err := a.resolveDraftReplyTarget(ctx, intent, req.Grant) + target, from, selfAddresses, err := a.resolveDraftTarget( + ctx, &intent.MessageID, intent.Account, intent.SourceID, intent.SourceIDSet, + intent.From, req.Grant, + ) if err != nil { return err } - reply, err := imaplib.BuildReply(target.raw, intent.From, intent.Body, time.Now(), "") + reply, err := imaplib.BuildReplyWithOptions(target.raw, from, intent.Body, imaplib.ReplyOptions{ + ReplyAll: intent.ReplyAll, SelfAddresses: selfAddresses, + }, time.Now(), "") if err != nil { return draftReplyError("invalid_reply_metadata", err) } - if len(reply.Parsed.From) != 1 || len(reply.Parsed.To) == 0 { - return draftReplyError("invalid_reply_metadata", errors.New("composed reply needs one From and at least one To address")) - } - messageIDValue := mime.NormalizeMessageID(reply.Parsed.MessageID) - if messageIDValue == "" { - return draftReplyError("invalid_reply_metadata", errors.New("composed reply has no usable Message-ID")) - } - messageIDValue = "<" + messageIDValue + ">" - - // Hold the source's sync lock from APPEND through local publication so a - // concurrent sync cannot reconcile a stale mailbox snapshot over the draft. - execution, err := a.store.AcquireSyncExecutionContext(ctx, target.source.ID) - if err != nil { - if errors.Is(err, store.ErrSyncAlreadyActive) { - return draftReplyError("sync_active", fmt.Errorf("source %d: %w", target.source.ID, err)) + if target.source.SourceType == "gmail" { + if len(reply.Parsed.From) != 1 || len(reply.Parsed.To)+len(reply.Parsed.Cc)+len(reply.Parsed.Bcc) == 0 { + return draftReplyError("invalid_reply_metadata", errors.New("composed reply needs one From and at least one recipient")) } - return draftReplyError("sync_lock_failed", fmt.Errorf("source %d: %w", target.source.ID, err)) - } - defer func() { _ = execution.Release() }() - - receipt, err := a.appendDraftReply(ctx, target, reply.Raw, emit) - if err != nil { - return err - } - receiptModel := store.IMAPDraftReceipt{ - SourceID: target.source.ID, Mailbox: target.mailbox, - UIDValidity: receipt.UIDValidity, UID: receipt.UID, - } - result := draftReplyOutput{ - Status: draftReplyStatusCreated, - OperationRef: draftOperationRef(receiptModel), - RFC822MessageID: messageIDValue, - SourceID: target.source.ID, - Mailbox: target.mailbox, - UID: receipt.UID, - UIDValidity: receipt.UIDValidity, - } - localID, err := a.store.PersistIMAPDraftContext(ctx, receiptModel, draftReplyParticipants(reply.Parsed), func(ids []int64) *store.MessagePersistData { - return draftReplyPersistData(target, reply, receiptModel, messageIDValue, ids) - }) - if err != nil { - result.Status = draftReplyStatusLocalFailed - _ = emitDraftReplyOutput(emit, cliStreamStderr, intent.JSON, result) - return draftReplyError(draftReplyStatusLocalFailed, err) - } - result.MessageID = localID - if err := emitDraftReplyOutput(emit, cliStreamStdout, intent.JSON, result); err != nil { - return draftReplyError("output_failed", err) - } - // The draft is durable and reported. Free the source for syncs before the - // cache rebuild, which can take a while and needs no lock. - if err := execution.Release(); err != nil { - logger.Error("release source after draft", "source_id", target.source.ID, "error", err) + messageIDValue := mime.NormalizeMessageID(reply.Parsed.MessageID) + if messageIDValue == "" { + return draftReplyError("invalid_reply_metadata", errors.New("composed reply has no usable Message-ID")) + } + return a.runGmailReplyDraft(ctx, intent, target, reply, "<"+messageIDValue+">", emit) } - a.refreshDraftCache(ctx, target.source) - return nil + return a.createDraft(ctx, target, reply, intent.JSON, emit) } // refreshDraftCache runs the daemon's best-effort analytics rebuild once the @@ -259,7 +424,9 @@ func (a *storeAPIAdapter) refreshDraftCache(ctx context.Context, source *store.S if errors.Is(err, context.Canceled) { return } - logger.Error("draft analytics cache refresh failed", "source_id", source.ID, "error", err) + if a.logger != nil { + a.logger.Error("draft analytics cache refresh failed", "source_id", source.ID, "error", err) + } } } @@ -271,64 +438,89 @@ func authorizeDelegatedDraftSource(grant *agentgrant.Grant, source *store.Source if grant == nil { return nil } - ref := agentgrant.SourceRef{ID: source.ID, Type: source.SourceType, Identifier: source.Identifier} + ref := draftSourceRef(source) if !grant.Allows(agentgrant.PermissionDraftCreate, ref) { return draftReplyNotPermitted(fmt.Errorf("source %d is not in grant %s", source.ID, grant.ID)) } return nil } -// resolveDraftReplyTarget loads the parent, checks the operator grant, and -// confirms the sender identity. It runs before the sync lock is taken so a -// denied request never blocks a sync. -func (a *storeAPIAdapter) resolveDraftReplyTarget(ctx context.Context, intent draftReplyIntent, grant *agentgrant.Grant) (draftReplyTarget, error) { - parent, err := a.store.GetMessageContext(ctx, intent.MessageID) - if err != nil { - if grant != nil { - return draftReplyTarget{}, draftReplyNotPermitted(fmt.Errorf("load message %d: %w", intent.MessageID, err)) - } - return draftReplyTarget{}, draftReplyError("invalid_parent", fmt.Errorf("load message %d: %w", intent.MessageID, err)) +func (a *storeAPIAdapter) createDraft( + ctx context.Context, + target draftReplyTarget, + draft imaplib.ReplyDraft, + asJSON bool, + emit func(api.CLIRunEvent) error, +) error { + if len(draft.Parsed.From) != 1 || len(draft.Parsed.To)+len(draft.Parsed.Cc)+len(draft.Parsed.Bcc) == 0 { + return draftReplyError("invalid_reply_metadata", errors.New("draft needs one From and at least one recipient")) + } + messageIDValue := mime.NormalizeMessageID(draft.Parsed.MessageID) + if messageIDValue == "" { + return draftReplyError("invalid_reply_metadata", errors.New("composed draft has no usable Message-ID")) } - source, err := a.store.GetSourceByIDContext(ctx, parent.SourceID) + messageIDValue = "<" + messageIDValue + ">" + + // Hold the source's sync lock from APPEND through local publication so a + // concurrent sync cannot reconcile a stale mailbox snapshot over the draft. + execution, err := a.store.AcquireSyncExecutionContext(ctx, target.source.ID) if err != nil { - if grant != nil { - return draftReplyTarget{}, draftReplyNotPermitted(fmt.Errorf("load source %d: %w", parent.SourceID, err)) + if errors.Is(err, store.ErrSyncAlreadyActive) { + return draftReplyError("sync_active", fmt.Errorf("source %d: %w", target.source.ID, err)) } - return draftReplyTarget{}, draftReplyError("invalid_source", fmt.Errorf("load source %d: %w", parent.SourceID, err)) + return draftReplyError("sync_lock_failed", fmt.Errorf("source %d: %w", target.source.ID, err)) } - if err := authorizeDelegatedDraftSource(grant, source); err != nil { - return draftReplyTarget{}, err + defer func() { _ = execution.Release() }() + + clientFactory := a.draftClientFactory + if clientFactory == nil { + clientFactory = defaultDraftClientFactory } - mailbox, err := authorizeIMAPDraft(a.draftPolicy, source.ID, source.SourceType) + client, err := clientFactory(ctx, target.source) if err != nil { - return draftReplyTarget{}, err - } - if !source.SyncConfig.Valid { - return draftReplyTarget{}, draftReplyError("invalid_source", fmt.Errorf("source %d has no sync config", source.ID)) + return draftReplyError("invalid_source", fmt.Errorf("build IMAP client for source %d: %w", target.source.ID, err)) } - imapConfig, err := imaplib.ConfigFromJSON(source.SyncConfig.String) + defer func() { _ = client.Close() }() + receipt, err := a.appendDraftReplyWithClient(ctx, client, target, draft.Raw, emit) if err != nil { - return draftReplyTarget{}, draftReplyError("invalid_source", fmt.Errorf("source %d sync config: %w", source.ID, err)) + return err } - if imapConfig.Identifier() != source.Identifier { - return draftReplyTarget{}, draftReplyError("invalid_source", fmt.Errorf("source %d sync config identifier does not match the source", source.ID)) + receiptModel := store.IMAPDraftReceipt{ + SourceID: target.source.ID, Mailbox: target.mailbox, + UIDValidity: receipt.UIDValidity, UID: receipt.UID, } - raw, err := a.store.GetMessageRawContext(ctx, parent.ID) - if err != nil { - return draftReplyTarget{}, draftReplyError("invalid_parent", fmt.Errorf("load raw MIME for message %d: %w", parent.ID, err)) + result := draftReplyOutput{ + Status: draftReplyStatusCreated, + OperationRef: draftOperationRef(receiptModel), + RFC822MessageID: messageIDValue, + SourceID: target.source.ID, + Mailbox: target.mailbox, + UID: receipt.UID, + UIDValidity: receipt.UIDValidity, } - identities, err := a.store.ListAccountIdentitiesContext(ctx, source.ID) + evidenceCtx, cancelEvidence := localDraftEvidenceContext(ctx) + defer cancelEvidence() + draftRecord, err := a.store.PersistIMAPDraftContext(evidenceCtx, receiptModel, draftReplyParticipants(draft.Parsed), func(ids []int64) *store.MessagePersistData { + return draftReplyPersistData(target, draft, receiptModel, messageIDValue, ids) + }) if err != nil { - return draftReplyTarget{}, draftReplyError("invalid_from", fmt.Errorf("list identities for source %d: %w", source.ID, err)) + result.Status = draftReplyStatusLocalFailed + _ = emitDraftReplyOutput(emit, cliStreamStderr, asJSON, result) + return draftReplyError(draftReplyStatusLocalFailed, err) } - if !hasConfirmedSourceIdentity(identities, intent.From) { - return draftReplyTarget{}, draftReplyError("invalid_from", fmt.Errorf("--from is not a confirmed identity on source %d", source.ID)) + defer a.releaseDraftSourceAndRefreshCache(ctx, target.source, execution) + result.DraftID = draftRecord.DraftID + result.Revision = draftRecord.Revision + result.MessageID = draftRecord.CurrentMessageID + if err := emitDraftReplyOutput(emit, cliStreamStdout, asJSON, result); err != nil { + return draftReplyError("output_failed", err) } - return draftReplyTarget{parent: parent, source: source, mailbox: mailbox, raw: raw}, nil + _ = client.Close() + return nil } func defaultDraftClientFactory(ctx context.Context, source *store.Source) (*imaplib.Client, error) { - client, err := buildAPIClient(ctx, source, oauthManagerCache(), nil) + client, err := buildAPIClient(ctx, source, oauthManagerCache(invocationFromContext(ctx)), nil) if err != nil { return nil, err } @@ -339,23 +531,13 @@ func defaultDraftClientFactory(ctx context.Context, source *store.Source) (*imap return imapClient, nil } -// appendDraftReply sends the single APPEND. Any failure after this point -// leaves a state the operator must inspect before retrying. -func (a *storeAPIAdapter) appendDraftReply( +func (a *storeAPIAdapter) appendDraftReplyWithClient( ctx context.Context, + client *imaplib.Client, target draftReplyTarget, raw []byte, emit func(api.CLIRunEvent) error, ) (imaplib.DraftAppendResult, error) { - clientFactory := a.draftClientFactory - if clientFactory == nil { - clientFactory = defaultDraftClientFactory - } - client, err := clientFactory(ctx, target.source) - if err != nil { - return imaplib.DraftAppendResult{}, draftReplyError("invalid_source", fmt.Errorf("build IMAP client for source %d: %w", target.source.ID, err)) - } - defer func() { _ = client.Close() }() receipt, err := client.AppendDraft(ctx, target.mailbox, raw) if err != nil { if emit != nil { @@ -370,8 +552,12 @@ func (a *storeAPIAdapter) appendDraftReply( } func draftReplyParticipants(parsed *mime.Message) []store.ParticipantPersistData { - participants := make([]store.ParticipantPersistData, 0, len(parsed.From)+len(parsed.To)) - for _, address := range append(append([]mime.Address(nil), parsed.From...), parsed.To...) { + addresses := append([]mime.Address(nil), parsed.From...) + addresses = append(addresses, parsed.To...) + addresses = append(addresses, parsed.Cc...) + addresses = append(addresses, parsed.Bcc...) + participants := make([]store.ParticipantPersistData, 0, len(addresses)) + for _, address := range addresses { participants = append(participants, store.ParticipantPersistData{ EmailAddress: address.Email, DisplayName: address.Name, @@ -392,22 +578,44 @@ func draftReplyPersistData( ) *store.MessagePersistData { parsed := reply.Parsed fromCount := len(parsed.From) - toAddresses := make([]string, len(parsed.To)) - for i, address := range parsed.To { - toAddresses[i] = address.Email - } - conversationKey := target.parent.SourceConversationID - if conversationKey == "" { - conversationKey = fmt.Sprintf("draft-reply-%d", target.parent.ID) - } + toCount := len(parsed.To) + ccCount := len(parsed.Cc) + bccCount := len(parsed.Bcc) + toAddresses := addressStrings(parsed.To) + ccAddresses := addressStrings(parsed.Cc) + bccAddresses := addressStrings(parsed.Bcc) + fromAddresses := addressStrings(parsed.From) + var conversationKey string + var replyToMessageID sql.NullInt64 + if target.forward { // forwards start their own conversation + conversationKey = fmt.Sprintf("draft-forward-%d-%d-%s", receipt.SourceID, receipt.UIDValidity, store.IMAPDraftSourceMessageID(receipt)) + } else if target.parent != nil { + conversationKey = target.parent.SourceConversationID + replyToMessageID = sql.NullInt64{Int64: target.parent.ID, Valid: true} + if conversationKey == "" { + conversationKey = fmt.Sprintf("draft-reply-%d", target.parent.ID) + } + if target.parentSource != nil && target.parentSource.ID != target.source.ID { + conversationKey = fmt.Sprintf("draft-reply-%d-%d-%s", target.parentSource.ID, target.source.ID, conversationKey) + } + } else { + conversationKey = fmt.Sprintf("draft-compose-%d-%d-%s", receipt.SourceID, receipt.UIDValidity, store.IMAPDraftSourceMessageID(receipt)) + } + at := fromCount + fromIDs := ids[:fromCount] + toIDs := ids[at : at+toCount] + at += toCount + ccIDs := ids[at : at+ccCount] + at += ccCount + bccIDs := ids[at : at+bccCount] return &store.MessagePersistData{ Message: &store.Message{ SourceID: target.source.ID, SourceMessageID: store.IMAPDraftSourceMessageID(receipt), RFC822MessageID: sql.NullString{String: messageIDValue, Valid: true}, MessageType: "email", IsFromMe: true, IdentityDerivedIsFromMe: true, - SenderID: sql.NullInt64{Int64: ids[0], Valid: true}, - ReplyToMessageID: sql.NullInt64{Int64: target.parent.ID, Valid: true}, + SenderID: sql.NullInt64{Int64: fromIDs[0], Valid: true}, + ReplyToMessageID: replyToMessageID, Subject: sql.NullString{String: parsed.Subject, Valid: parsed.Subject != ""}, Snippet: sql.NullString{String: strings.TrimSpace(parsed.BodyText), Valid: parsed.BodyText != ""}, SentAt: sql.NullTime{Time: parsed.Date, Valid: !parsed.Date.IsZero()}, @@ -421,10 +629,19 @@ func draftReplyPersistData( BodyText: sql.NullString{String: parsed.BodyText, Valid: true}, RawMIME: reply.Raw, RawFormat: "mime", Recipients: []store.RecipientSet{ - {Type: "from", ParticipantIDs: ids[:fromCount], EmailAddresses: []string{parsed.From[0].Email}}, - {Type: "to", ParticipantIDs: ids[fromCount:], EmailAddresses: toAddresses}, + {Type: "from", ParticipantIDs: fromIDs, EmailAddresses: fromAddresses}, + {Type: "to", ParticipantIDs: toIDs, EmailAddresses: toAddresses}, + {Type: "cc", ParticipantIDs: ccIDs, EmailAddresses: ccAddresses}, + {Type: "bcc", ParticipantIDs: bccIDs, EmailAddresses: bccAddresses}, + }, + FTS: &store.FTSDoc{ + Subject: parsed.Subject, + Body: parsed.BodyText, + FromAddr: parsed.From[0].Email, + ToAddrs: strings.Join(toAddresses, " "), + CcAddrs: strings.Join(ccAddresses, " "), }, - FTS: &store.FTSDoc{Subject: parsed.Subject, Body: parsed.BodyText, FromAddr: parsed.From[0].Email, ToAddrs: strings.Join(toAddresses, " ")}, + MIMEAttachmentReplacement: target.attachmentWrites, } } @@ -437,8 +654,8 @@ func emitDraftReplyOutput(emit func(api.CLIRunEvent) error, stream string, asJSO case asJSON: text = string(marshalDraftReplyOutput(result)) + "\n" case result.Status == draftReplyStatusCreated: - text = fmt.Sprintf("created draft message %d (%s|%d|%d), operation %s\n", - result.MessageID, result.Mailbox, result.UIDValidity, result.UID, result.OperationRef) + text = fmt.Sprintf("created draft message %d (%s|%d|%d), operation %s, draft %s revision %d\n", + result.MessageID, result.Mailbox, result.UIDValidity, result.UID, result.OperationRef, result.DraftID, result.Revision) default: text = fmt.Sprintf("remote accepted; local persistence failed, inspect operation %s\n", result.OperationRef) } diff --git a/cmd/msgvault/cmd/serve_reply_draft_test.go b/cmd/msgvault/cmd/serve_reply_draft_test.go index a41581a39..c9f0b48b6 100644 --- a/cmd/msgvault/cmd/serve_reply_draft_test.go +++ b/cmd/msgvault/cmd/serve_reply_draft_test.go @@ -1,6 +1,7 @@ package cmd import ( + "bufio" "bytes" "context" "database/sql" @@ -9,6 +10,7 @@ import ( "fmt" "log/slog" "net" + "net/http" "net/http/httptest" "strconv" "strings" @@ -16,6 +18,7 @@ import ( "time" emersionimap "github.com/emersion/go-imap/v2" + "github.com/emersion/go-imap/v2/imapclient" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -71,6 +74,8 @@ func TestRunCLIReplyDraftUsesTypedRoute(t *testing.T) { func TestDraftPolicySnapshotRequiresDaemonRestart(t *testing.T) { cfg := config.Config{IMAP: config.IMAPConfig{Drafts: []config.IMAPDraftSource{{SourceID: 42, Enabled: true, Mailbox: "Drafts"}}}} + testCtx := testInvocationContext(t.Context(), &cfg, invocationOptions{}) + _ = testCtx snapshot := snapshotIMAPDraftPolicy(&cfg) cfg.IMAP.Drafts[0].Enabled = false assert.True(t, snapshot[0].Enabled) @@ -78,8 +83,17 @@ func TestDraftPolicySnapshotRequiresDaemonRestart(t *testing.T) { func TestConfirmedSourceIdentityRejectsMismatch(t *testing.T) { identities := []store.AccountIdentity{{Address: "user@example.com", ConfirmedAt: time.Now()}} - assert.False(t, hasConfirmedSourceIdentity(identities, "other@example.com")) - assert.True(t, hasConfirmedSourceIdentity(identities, "USER@example.com")) + eligible, _ := confirmedDraftIdentities(identities) + assert.NotContains(t, eligible, store.NormalizeIdentifierForCompare("other@example.com")) + assert.Contains(t, eligible, store.NormalizeIdentifierForCompare("USER@example.com")) + adapter := &storeAPIAdapter{} + _, _, err := adapter.selectDraftSender( + []store.AccountIdentity{ + {Address: "user@example.com", ConfirmedAt: time.Now()}, + {Address: "alias@example.com", ConfirmedAt: time.Now()}, + }, "", nil, &store.Source{ID: 1, SourceType: "imap", Identifier: "alice@example.com"}, + ) + assert.ErrorContains(t, err, "from_ambiguous") } // draftReplyFixture is one archived IMAP parent message on a source backed by @@ -163,6 +177,191 @@ func (f draftReplyFixture) grantedAdapter() *storeAPIAdapter { } } +func fetchDraftMailboxMessage( + t *testing.T, + config *imaplib.Config, + receipt store.IMAPDraftReceipt, +) ([]emersionimap.Flag, []byte) { + t.Helper() + requirements := require.New(t) + client, err := imapclient.DialInsecure(config.Addr(), nil) + requirements.NoError(err) + t.Cleanup(func() { _ = client.Close() }) + requirements.NoError(client.Login(testutil.IMAPTestUsername, testutil.IMAPTestPassword).Wait()) + _, err = client.Select(receipt.Mailbox, nil).Wait() + requirements.NoError(err) + section := &emersionimap.FetchItemBodySection{} + uidSet := emersionimap.UIDSetNum(emersionimap.UID(receipt.UID)) + fetched, err := client.Fetch(uidSet, &emersionimap.FetchOptions{ + UID: true, Flags: true, BodySection: []*emersionimap.FetchItemBodySection{section}, + }).Collect() + requirements.NoError(err) + requirements.Len(fetched, 1) + requirements.Equal(emersionimap.UID(receipt.UID), fetched[0].UID) + return fetched[0].Flags, fetched[0].FindBodySection(section) +} + +func TestDraftReplyOfflineParentWithLiveDestination(t *testing.T) { + assertions := assert.New(t) + requirements := require.New(t) + fixture := newDraftReplyFixture(t) + + parentSource, err := fixture.store.GetOrCreateSource("mbox", "imported-parent@example.test") + requirements.NoError(err) + conversationID, err := fixture.store.EnsureConversation(parentSource.ID, "offline-thread", "Imported question") + requirements.NoError(err) + senderID, err := fixture.store.EnsureParticipant("sender@example.test", "Sender", "example.test") + requirements.NoError(err) + ownerID, err := fixture.store.EnsureParticipant(testutil.IMAPTestUsername, "", "example.test") + requirements.NoError(err) + parentRaw := []byte("From: Sender \r\n" + + "To: " + testutil.IMAPTestUsername + "\r\n" + + "Subject: Imported question\r\n" + + "Message-ID: \r\n\r\n" + + "Imported body\r\n") + parentID, err := fixture.store.PersistMessage(&store.MessagePersistData{ + Message: &store.Message{ + SourceID: parentSource.ID, SourceMessageID: "mbox|offline-parent", + RFC822MessageID: sql.NullString{String: "offline-parent@example.test", Valid: true}, + ConversationID: conversationID, MessageType: store.MessageTypeEmail, + SenderID: sql.NullInt64{Int64: senderID, Valid: true}, + Subject: sql.NullString{String: "Imported question", Valid: true}, + SizeEstimate: int64(len(parentRaw)), + }, + BodyText: sql.NullString{String: "Imported body", Valid: true}, + RawMIME: parentRaw, + Recipients: []store.RecipientSet{ + {Type: "from", ParticipantIDs: []int64{senderID}, EmailAddresses: []string{"sender@example.test"}}, + {Type: "to", ParticipantIDs: []int64{ownerID}, EmailAddresses: []string{testutil.IMAPTestUsername}}, + }, + }) + requirements.NoError(err) + fixture.parentID = parentID + + adapter := fixture.grantedAdapter() + providerCalls := 0 + clientFactory := adapter.draftClientFactory + adapter.draftClientFactory = func(ctx context.Context, source *store.Source) (*imaplib.Client, error) { + providerCalls++ + return clientFactory(ctx, source) + } + + events, err := fixture.run(t, adapter, "--body", "reply body") + requirements.Error(err) + assertions.Empty(events) + assertions.Equal("invalid_source", err.Error()) + assertions.Zero(providerCalls) + + server := httptest.NewServer(api.NewServerWithOptions(api.ServerOptions{ + Config: &config.Config{ + HomeDir: t.TempDir(), + Server: config.ServerConfig{APIKey: "owner-test-key", AgentAccess: true}, + }, + Store: adapter, + Logger: slog.New(slog.DiscardHandler), + }).Router()) + t.Cleanup(server.Close) + + issue := func(sourceIDs ...int64) string { + senderSelections := map[string][]string{} + for _, sourceID := range sourceIDs { + if sourceID == fixture.source.ID { + senderSelections[strconv.FormatInt(sourceID, 10)] = []string{testutil.IMAPTestUsername} + } + } + body, err := json.Marshal(map[string]any{ + "label": "offline-reply-agent", "permissions": []string{"draft.create"}, + "source_ids": sourceIDs, "sender_selections": senderSelections, + }) + requirements.NoError(err) + request, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/agent-tokens", bytes.NewReader(body)) + requirements.NoError(err) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("X-Api-Key", "owner-test-key") + response, err := http.DefaultClient.Do(request) + requirements.NoError(err) + defer func() { _ = response.Body.Close() }() + requirements.Equal(http.StatusCreated, response.StatusCode) + var issued agentTokenIssueFixture + requirements.NoError(json.NewDecoder(response.Body).Decode(&issued)) + return issued.Secret + } + run := func(secret string) []api.CLIRunEvent { + args := []string{ + "draft-reply", strconv.FormatInt(parentID, 10), + "--source-id", strconv.FormatInt(fixture.source.ID, 10), + "--from", testutil.IMAPTestUsername, "--body", "reply body", "--json", + } + body, err := json.Marshal(map[string]any{"args": args}) + requirements.NoError(err) + request, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/cli/run", bytes.NewReader(body)) + requirements.NoError(err) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("X-Msgvault-Agent-Token", secret) + response, err := http.DefaultClient.Do(request) + requirements.NoError(err) + defer func() { _ = response.Body.Close() }() + requirements.Equal(http.StatusOK, response.StatusCode) + var events []api.CLIRunEvent + scanner := bufio.NewScanner(response.Body) + for scanner.Scan() { + var event api.CLIRunEvent + requirements.NoError(json.Unmarshal(scanner.Bytes(), &event)) + events = append(events, event) + } + requirements.NoError(scanner.Err()) + return events + } + events = run(issue(parentSource.ID, fixture.source.ID)) + requirements.Len(events, 2) + var result draftReplyOutput + requirements.NoError(json.Unmarshal([]byte(events[0].Data), &result)) + assertions.Equal(cliStreamStdout, events[0].Type) + assertions.Equal("complete", events[1].Type) + assertions.Equal(draftReplyStatusCreated, result.Status) + assertions.Equal(fixture.source.ID, result.SourceID) + assertions.Equal("Drafts", result.Mailbox) + assertions.NotZero(result.UID) + assertions.NotZero(result.UIDValidity) + assertions.Equal(int64(1), result.Revision) + assertions.Equal(1, providerCalls) + + draft, err := fixture.store.GetIMAPDraft(result.DraftID) + requirements.NoError(err) + assertions.Equal(result.MessageID, draft.CurrentMessageID) + assertions.Equal(result.UID, draft.CurrentReceipt.UID) + assertions.Equal(fixture.source.ID, draft.CurrentReceipt.SourceID) + + message, err := fixture.store.GetMessage(result.MessageID) + requirements.NoError(err) + assertions.Equal(fixture.source.ID, message.SourceID) + assertions.Equal( + "draft-reply-"+strconv.FormatInt(parentSource.ID, 10)+"-"+ + strconv.FormatInt(fixture.source.ID, 10)+"-offline-thread", + message.SourceConversationID, + ) + replyTo, err := fixture.store.GetMessageReplyToMessageIDContext(t.Context(), result.MessageID) + requirements.NoError(err) + requirements.True(replyTo.Valid) + assertions.Equal(parentID, replyTo.Int64) + + storedRaw, err := fixture.store.GetMessageRaw(result.MessageID) + requirements.NoError(err) + assertions.Contains(string(storedRaw), "In-Reply-To: ") + assertions.Contains(string(storedRaw), "References: ") + flags, fetchedRaw := fetchDraftMailboxMessage(t, fixture.config, draft.CurrentReceipt) + assertions.Contains(flags, emersionimap.FlagDraft) + assertions.Equal(storedRaw, fetchedRaw) + + for _, scope := range [][]int64{{fixture.source.ID}, {parentSource.ID}} { + events = run(issue(scope...)) + requirements.Len(events, 1) + assertions.Equal("error", events[0].Type) + assertions.Equal("not_permitted", events[0].Error) + } + assertions.Equal(1, providerCalls, "both source grants must be checked before provider work") +} + func (f draftReplyFixture) run(t *testing.T, adapter *storeAPIAdapter, flags ...string) ([]api.CLIRunEvent, error) { t.Helper() args := append([]string{"draft-reply", strconv.FormatInt(f.parentID, 10), "--from", testutil.IMAPTestUsername}, flags...) @@ -204,6 +403,7 @@ func TestDelegatedDraftReplyCreatesDraft(t *testing.T) { ID: fixture.source.ID, Type: fixture.source.SourceType, Identifier: fixture.source.Identifier, + SenderKeys: []string{store.NormalizeIdentifierForCompare(testutil.IMAPTestUsername)}, }}, } @@ -451,7 +651,10 @@ func TestAppendDraftReplyRetainsFailureCause(t *testing.T) { requirements := require.New(t) assertions := assert.New(t) var events []api.CLIRunEvent - _, err := adapter.appendDraftReply(t.Context(), draftReplyTarget{source: &store.Source{}, mailbox: tc.mailbox}, []byte("Subject: Reply\r\n\r\nreply body\r\n"), func(event api.CLIRunEvent) error { + client, err := adapter.draftClientFactory(t.Context(), &store.Source{}) + requirements.NoError(err) + t.Cleanup(func() { _ = client.Close() }) + _, err = adapter.appendDraftReplyWithClient(t.Context(), client, draftReplyTarget{source: &store.Source{}, mailbox: tc.mailbox}, []byte("Subject: Reply\r\n\r\nreply body\r\n"), func(event api.CLIRunEvent) error { events = append(events, event) return nil }) @@ -498,8 +701,9 @@ func TestDraftReplyCLIFailureOutput(t *testing.T) { Logger: slog.New(slog.DiscardHandler), }).Router()) t.Cleanup(server.Close) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) root := &cobra.Command{Use: "msgvault"} + root.SetContext(testCtx) root.AddCommand(newDraftReplyCommand()) silenceUsageInRunE(root) var stdout, stderr bytes.Buffer @@ -510,7 +714,7 @@ func TestDraftReplyCLIFailureOutput(t *testing.T) { args = append(args, "--json") } root.SetArgs(args) - requirements.Error(root.ExecuteContext(t.Context())) + requirements.Error(root.ExecuteContext(testCtx)) assertions.Empty(stdout.String()) assertions.Len(strings.Split(strings.TrimSpace(stderr.String()), "\n"), 1, stderr.String()) switch failure { @@ -558,7 +762,10 @@ func TestDelegatedDraftRefusesOutOfGrantSource(t *testing.T) { From: testutil.IMAPTestUsername, Body: "reply body", } - _, err := adapter.resolveDraftReplyTarget(t.Context(), intent, outOfScopeGrant) + target, selectedFrom, selfAddresses, err := adapter.resolveDraftTarget(t.Context(), &intent.MessageID, "", 0, false, intent.From, outOfScopeGrant) + assertions.Empty(target) + assertions.Empty(selectedFrom) + assertions.Empty(selfAddresses) requirements.Error(err) assertions.Equal("not_permitted", err.Error()) coded, ok := errors.AsType[*api.CLIRunCodedError](err) @@ -570,12 +777,15 @@ func TestDelegatedDraftRefusesOutOfGrantSource(t *testing.T) { // It verifies that authorizeDelegatedDraftSource runs BEFORE authorizeIMAPDraft // so an out-of-grant source cannot infer whether drafting is configured. func TestDraftRequiresBothChecks(t *testing.T) { + assertions := assert.New(t) + requirements := require.New(t) fixture := newDraftReplyFixture(t) inGrantRef := agentgrant.SourceRef{ ID: fixture.source.ID, Type: fixture.source.SourceType, Identifier: fixture.source.Identifier, + SenderKeys: []string{store.NormalizeIdentifierForCompare(testutil.IMAPTestUsername)}, } inGrant := &agentgrant.Grant{ ID: "g-in-grant", @@ -601,9 +811,12 @@ func TestDraftRequiresBothChecks(t *testing.T) { store: fixture.store, draftPolicy: nil, } - _, err := adapter.resolveDraftReplyTarget(t.Context(), intent, inGrant) - require.Error(t, err) - assert.Equal(t, "draft_disabled", err.Error()) + target, selectedFrom, selfAddresses, err := adapter.resolveDraftTarget(t.Context(), &intent.MessageID, "", 0, false, intent.From, inGrant) + assertions.Empty(target) + assertions.Empty(selectedFrom) + assertions.Empty(selfAddresses) + requirements.Error(err) + assertions.Equal("draft_disabled", err.Error()) }) t.Run("grant out-of-scope but draft policy exists returns not_permitted", func(t *testing.T) { @@ -611,8 +824,102 @@ func TestDraftRequiresBothChecks(t *testing.T) { // authorizeDelegatedDraftSource runs first, so the code is not_permitted, // not draft_disabled — the caller cannot infer whether drafting is configured. adapter := fixture.grantedAdapter() - _, err := adapter.resolveDraftReplyTarget(t.Context(), intent, outOfGrant) - require.Error(t, err) - assert.Equal(t, "not_permitted", err.Error()) + target, selectedFrom, selfAddresses, err := adapter.resolveDraftTarget(t.Context(), &intent.MessageID, "", 0, false, intent.From, outOfGrant) + assertions.Empty(target) + assertions.Empty(selectedFrom) + assertions.Empty(selfAddresses) + requirements.Error(err) + assertions.Equal("not_permitted", err.Error()) + }) + + t.Run("grant without the selected sender returns not_permitted", func(t *testing.T) { + adapter := fixture.grantedAdapter() + grantWithoutSender := &agentgrant.Grant{ + ID: "g-no-sender", + Permissions: []agentgrant.Permission{agentgrant.PermissionDraftCreate}, + Sources: []agentgrant.SourceRef{{ID: fixture.source.ID, Type: "imap", Identifier: fixture.source.Identifier}}, + } + target, selectedFrom, selfAddresses, err := adapter.resolveDraftTarget( + t.Context(), &intent.MessageID, "", 0, false, intent.From, grantWithoutSender, + ) + assertions.Empty(target) + assertions.Empty(selectedFrom) + assertions.Empty(selfAddresses) + requirements.Error(err) + assertions.Equal("not_permitted", err.Error()) + }) + + t.Run("automatic sender selection respects the frozen grant", func(t *testing.T) { + adapter := fixture.grantedAdapter() + grantWithoutCurrentSender := &agentgrant.Grant{ + ID: "g-other-sender", + Permissions: []agentgrant.Permission{agentgrant.PermissionDraftCreate}, + Sources: []agentgrant.SourceRef{{ID: fixture.source.ID, Type: "imap", Identifier: fixture.source.Identifier, SenderKeys: []string{"alias@example.com"}}}, + } + from := "" + target, selectedFrom, selfAddresses, err := adapter.resolveDraftTarget( + t.Context(), &intent.MessageID, "", 0, false, from, grantWithoutCurrentSender, + ) + assertions.Empty(target) + assertions.Empty(selectedFrom) + assertions.Empty(selfAddresses) + requirements.Error(err) + assertions.Equal("not_permitted", err.Error()) }) + + t.Run("destination resolution hides missing sources from grants", func(t *testing.T) { + adapter := fixture.grantedAdapter() + target, selectedFrom, selfAddresses, err := adapter.resolveDraftTarget( + t.Context(), &intent.MessageID, "", fixture.source.ID+999, true, intent.From, inGrant, + ) + assertions.Empty(target) + assertions.Empty(selectedFrom) + assertions.Empty(selfAddresses) + requirements.Error(err) + assertions.Equal("not_permitted", err.Error()) + }) + + t.Run("non-email parent is refused before MIME parsing", func(t *testing.T) { + _, updateErr := fixture.store.DB().Exec( + fixture.store.Rebind("UPDATE messages SET message_type = ? WHERE id = ?"), + store.MessageTypeGoogleChat, fixture.parentID, + ) + requirements.NoError(updateErr) + adapter := fixture.grantedAdapter() + _, _, _, err := adapter.resolveDraftTarget( + t.Context(), &intent.MessageID, "", 0, false, intent.From, inGrant, + ) + requirements.Error(err) + assertions.Equal("invalid_parent", err.Error()) + }) +} + +func TestDraftReplyPersistDataScopesCrossSourceConversation(t *testing.T) { + requirements := require.New(t) + parentRaw := []byte("From: sender@example.com\r\nMessage-ID: \r\nSubject: Imported\r\n\r\nold\r\n") + reply, err := imaplib.BuildReply(parentRaw, "owner@example.com", "reply", time.Now(), "draft@example.com") + requirements.NoError(err) + target := draftReplyTarget{ + parent: &store.APIMessage{ID: 7, SourceConversationID: "INBOX|9"}, + parentSource: &store.Source{ID: 1}, + source: &store.Source{ID: 2}, + } + data := draftReplyPersistData(target, reply, store.IMAPDraftReceipt{SourceID: 2, Mailbox: "Drafts", UIDValidity: 1, UID: 4}, "draft@example.com", []int64{1, 2}) + requirements.Equal("draft-reply-1-2-INBOX|9", data.Conversation.SourceConversationID) +} + +func TestDraftReplyPersistDataScopesComposeConversationByUIDValidity(t *testing.T) { + requirements := require.New(t) + parentRaw := []byte("From: sender@example.com\r\nMessage-ID: \r\nSubject: Imported\r\n\r\nold\r\n") + reply, err := imaplib.BuildReply(parentRaw, "owner@example.com", "compose", time.Now(), "draft@example.com") + requirements.NoError(err) + target := draftReplyTarget{source: &store.Source{ID: 2}} + first := draftReplyPersistData(target, reply, store.IMAPDraftReceipt{SourceID: 2, Mailbox: "Drafts", UIDValidity: 1, UID: 4}, "draft@example.com", []int64{1, 2}) + second := draftReplyPersistData(target, reply, store.IMAPDraftReceipt{SourceID: 2, Mailbox: "Drafts", UIDValidity: 2, UID: 4}, "draft@example.com", []int64{1, 2}) + + requirements.NotEqual(first.Conversation.SourceConversationID, second.Conversation.SourceConversationID) + requirements.Equal("draft-compose-2-1-Drafts|4", first.Conversation.SourceConversationID) + requirements.Equal("draft-compose-2-2-Drafts|4", second.Conversation.SourceConversationID) + requirements.Equal("Drafts|4", first.Message.SourceMessageID) + requirements.Equal("Drafts|4", second.Message.SourceMessageID) } diff --git a/cmd/msgvault/cmd/serve_test.go b/cmd/msgvault/cmd/serve_test.go index 4a923bfcd..17dc0fca8 100644 --- a/cmd/msgvault/cmd/serve_test.go +++ b/cmd/msgvault/cmd/serve_test.go @@ -3,6 +3,7 @@ package cmd import ( "bytes" "context" + "database/sql" "encoding/json" "errors" "fmt" @@ -17,6 +18,7 @@ import ( "runtime" "strconv" "strings" + "sync" "testing" "time" @@ -28,8 +30,10 @@ import ( "go.kenn.io/msgvault/internal/deletion" "go.kenn.io/msgvault/internal/discord" imaplib "go.kenn.io/msgvault/internal/imap" + "go.kenn.io/msgvault/internal/jobctx" "go.kenn.io/msgvault/internal/oauth" "go.kenn.io/msgvault/internal/personenrichment" + "go.kenn.io/msgvault/internal/provideridentity" "go.kenn.io/msgvault/internal/query" "go.kenn.io/msgvault/internal/scheduler" "go.kenn.io/msgvault/internal/store" @@ -77,7 +81,7 @@ func TestStoreAPIAdapterDeletePersonSuppressesCurrentIdentifiers(t *testing.T) { DueAt: now, })) lease, err := f.Store.ClaimWork(t.Context(), personenrichment.ClaimOptions{ - RunID: run.ID, Owner: "disabled-deletion-worker", ProviderName: profile.Name, + RunID: run.ID, Owner: "disabled-deletion-worker", ProviderName: profile.Name, ProfileFingerprint: profile.Fingerprint, Now: now, LeaseDuration: time.Minute, }) require.NoError(err) @@ -216,7 +220,7 @@ func TestStoreAPIAdapterDeletePersonRejectsRecordedAttemptKeyMismatch(t *testing DueAt: now, })) lease, err := f.Store.ClaimWork(t.Context(), personenrichment.ClaimOptions{ - RunID: run.ID, Owner: "attempt-key-worker", ProviderName: profile.Name, + RunID: run.ID, Owner: "attempt-key-worker", ProviderName: profile.Name, ProfileFingerprint: profile.Fingerprint, Now: now, LeaseDuration: time.Minute, }) require.NoError(err) @@ -317,6 +321,8 @@ func TestSchedulerWithConfig(t *testing.T) { {Email: "test3@gmail.com", Schedule: "invalid", Enabled: true}, }, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx var syncCalls []string sched := scheduler.New(func(ctx context.Context, email string) error { @@ -365,6 +371,8 @@ func TestServeOAuthValidationReportsNoProviders(t *testing.T) { } func TestRunServeStartsReadOnlyWithoutOAuthConfig(t *testing.T) { + cfg := testConfigValue() + oldCfg := cfg dataDir := t.TempDir() cfg = lifecycleTestConfig(dataDir) @@ -374,7 +382,7 @@ func TestRunServeStartsReadOnlyWithoutOAuthConfig(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() cmd := &cobra.Command{Use: serveCmd.Use} - cmd.SetContext(ctx) + cmd.SetContext(testInvocationContext(ctx, cfg, invocationOptions{})) errCh := make(chan error, 1) go func() { errCh <- runServe(cmd, nil) @@ -392,6 +400,8 @@ func TestRunServeStartsReadOnlyWithoutOAuthConfig(t *testing.T) { } func TestRunServeFailsPendingImportFromPreviousDaemon(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) oldCfg := cfg @@ -415,7 +425,7 @@ func TestRunServeFailsPendingImportFromPreviousDaemon(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) cmd := &cobra.Command{Use: serveCmd.Use} - cmd.SetContext(ctx) + cmd.SetContext(testInvocationContext(ctx, cfg, invocationOptions{})) errCh := make(chan error, 1) go func() { errCh <- runServe(cmd, nil) }() waitForServeHealth(t, c.Server.APIPort, errCh) @@ -438,6 +448,8 @@ func TestRunServeFailsPendingImportFromPreviousDaemon(t *testing.T) { } func TestRunServeImmediateCancellationWaitsForAPIStart(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) oldCfg := cfg dataDir := t.TempDir() @@ -468,7 +480,7 @@ func TestRunServeImmediateCancellationWaitsForAPIStart(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) cmd := &cobra.Command{Use: "serve"} - cmd.SetContext(ctx) + cmd.SetContext(testInvocationContext(ctx, cfg, invocationOptions{})) errCh := make(chan error, 1) go func() { errCh <- runServe(cmd, nil) }() @@ -494,6 +506,8 @@ func TestRunServeImmediateCancellationWaitsForAPIStart(t *testing.T) { } func TestRunServeAutoSelectsAPIPortWhenUnconfigured(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) @@ -506,7 +520,7 @@ func TestRunServeAutoSelectsAPIPortWhenUnconfigured(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() cmd := &cobra.Command{Use: serveCmd.Use} - cmd.SetContext(ctx) + cmd.SetContext(testInvocationContext(ctx, cfg, invocationOptions{})) errCh := make(chan error, 1) serveDone := make(chan struct{}) go func() { @@ -548,6 +562,8 @@ func TestRunServeAutoSelectsAPIPortWhenUnconfigured(t *testing.T) { } func TestRunServeServesHealthWhileAnalyticsBuildBlocked(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) oldCfg := cfg @@ -571,7 +587,7 @@ func TestRunServeServesHealthWhileAnalyticsBuildBlocked(t *testing.T) { defer cancel() t.Cleanup(cancel) cmd := &cobra.Command{Use: "serve"} - cmd.SetContext(ctx) + cmd.SetContext(testInvocationContext(ctx, cfg, invocationOptions{})) errCh := make(chan error, 1) go func() { errCh <- runServe(cmd, nil) @@ -617,6 +633,8 @@ func TestRunServeServesHealthWhileAnalyticsBuildBlocked(t *testing.T) { } func TestRunServeDuckDBReportsInitializingWithoutSQLFallback(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) oldCfg := cfg @@ -640,7 +658,7 @@ func TestRunServeDuckDBReportsInitializingWithoutSQLFallback(t *testing.T) { defer cancel() t.Cleanup(cancel) cmd := &cobra.Command{Use: serveCmd.Use} - cmd.SetContext(ctx) + cmd.SetContext(testInvocationContext(ctx, cfg, invocationOptions{})) errCh := make(chan error, 1) go func() { errCh <- runServe(cmd, nil) }() @@ -691,7 +709,7 @@ func TestRunServeDuckDBReportsInitializingWithoutSQLFallback(t *testing.T) { ) require.NoError(err, "POST SQL query") assert.Equal(http.StatusServiceUnavailable, resp.StatusCode, - "initializing DuckDB must report SQL engine unavailable") + "startup already owns cache recovery") _ = resp.Body.Close() cancel() @@ -704,6 +722,8 @@ func TestRunServeDuckDBReportsInitializingWithoutSQLFallback(t *testing.T) { } func TestRunServeAutoSwitchesToDuckDBAfterBackgroundBuild(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) oldCfg := cfg @@ -733,7 +753,7 @@ func TestRunServeAutoSwitchesToDuckDBAfterBackgroundBuild(t *testing.T) { }) cmd := &cobra.Command{Use: serveCmd.Use} - cmd.SetContext(ctx) + cmd.SetContext(testInvocationContext(ctx, cfg, invocationOptions{})) errCh := make(chan error, 1) go func() { errCh <- runServe(cmd, nil) }() @@ -792,6 +812,8 @@ func TestListenServeAPIHonorsAvailableExplicitPort(t *testing.T) { } func TestRunServeFailsBeforeArchiveWorkWhenAPIPortInUse(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) @@ -808,7 +830,7 @@ func TestRunServeFailsBeforeArchiveWorkWhenAPIPortInUse(t *testing.T) { t.Cleanup(func() { cfg = oldCfg }) cmd := &cobra.Command{Use: "serve"} - cmd.SetContext(context.Background()) + cmd.SetContext(testInvocationContext(context.Background(), cfg, invocationOptions{})) err = runServe(cmd, nil) require.Error(err, "runServe") @@ -899,7 +921,7 @@ func waitForServeHealth(t *testing.T, port int, errCh <-chan error) { return } } - time.Sleep(20 * time.Millisecond) + time.Sleep(20 * time.Millisecond) //nolint:kennlint // polls runServe's real TCP listener } require.FailNow(t, "serve health endpoint did not become ready") } @@ -923,40 +945,444 @@ func waitForServeHealthBounded(t *testing.T, port int, errCh <-chan error) { return } } - time.Sleep(20 * time.Millisecond) + time.Sleep(20 * time.Millisecond) //nolint:kennlint // polls runServe's real TCP listener } require.FailNow(t, "serve health endpoint did not become ready") } -func TestRunDaemonSQLQueryRebuildsStaleCacheOutOfProcess(t *testing.T) { +func TestRebuildCacheAfterManualSyncDefersUsableStaleCache(t *testing.T) { require := require.New(t) assert := assert.New(t) - dataDir := t.TempDir() - c := lifecycleTestConfig(dataDir) - s, err := store.Open(c.DatabaseDSN()) - require.NoError(err, "open store") - defer func() { _ = s.Close() }() - require.NoError(s.InitSchema(), "init schema") + c, s := openTestDaemonAnalyticsStore(t) + c.Analytics.AutoBuildCache = true + c.Analytics.MinRebuildInterval = 6 * time.Hour + state := &invocation{cfg: c} + _, err := s.DB().Exec(` + INSERT INTO sources (id, source_type, identifier) VALUES (1, 'gmail', 'user@example.com'); + INSERT INTO conversations (id, source_id, source_conversation_id, conversation_type) + VALUES (1, 1, 'thread-1', 'email_thread'); + INSERT INTO messages (id, source_id, source_message_id, conversation_id, message_type, sent_at) + VALUES (1, 1, 'message-1', 1, 'email', '2024-01-01 00:00:00'); + `) + require.NoError(err) + _, err = buildCache(c.DatabaseDSN(), c.AnalyticsDir(), true) + require.NoError(err) + _, err = s.DB().Exec(` + INSERT INTO messages (id, source_id, source_message_id, conversation_id, message_type, sent_at) + VALUES (2, 1, 'message-2', 1, 'email', '2024-01-02 00:00:00') + `) + require.NoError(err) + require.True(cacheNeedsBuild(c.DatabaseDSN(), c.AnalyticsDir()).NeedsBuild) + buildCacheBeforeMessagesExportHook = func() error { return errors.New("unexpected cache build") } + t.Cleanup(func() { buildCacheBeforeMessagesExportHook = nil }) + builderLock, err := cacheBuilderFileLock(c.AnalyticsDir()) + require.NoError(err) + locked, err := builderLock.TryLock() + require.NoError(err) + require.True(locked) + t.Cleanup(func() { require.NoError(builderLock.Unlock()) }) + done := make(chan error, 1) + go func() { done <- rebuildCacheAfterManualSync(c.DatabaseDSN(), state) }() + select { + case err := <-done: + require.NoError(err, "manual sync must finish while the builder lock is held") + case <-time.After(serveLifecycleTestTimeout): + require.FailNow("manual sync waited for the active cache builder") + } + staleness, err := cacheNeedsBuildForQuery(t.Context(), c.DatabaseDSN(), c.AnalyticsDir()) + require.NoError(err) + assert.True(staleness.NeedsBuild) +} + +func TestRunDaemonSQLQueryWithJobsServesStaleAndCoalescesFresh(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + c, s := openTestDaemonAnalyticsStore(t) + c.Analytics.MinRebuildInterval = 0 + _, err := s.DB().Exec(` + INSERT INTO sources (id, source_type, identifier) VALUES (1, 'gmail', 'user@example.com'); + INSERT INTO conversations (id, source_id, source_conversation_id, conversation_type) + VALUES (1, 1, 'thread-1', 'email_thread'); + INSERT INTO messages (id, source_id, source_message_id, conversation_id, message_type, sent_at) + VALUES (1, 1, 'message-1', 1, 'email', '2024-01-01 00:00:00'); + `) + require.NoError(err) + _, err = buildCache(c.DatabaseDSN(), c.AnalyticsDir(), true) + require.NoError(err) + engine, err := openDaemonDuckDBEngine(c, s) + require.NoError(err) + t.Cleanup(func() { _ = engine.Close() }) + _, err = s.DB().Exec(` + INSERT INTO messages (id, source_id, source_message_id, conversation_id, message_type, sent_at) + VALUES (2, 1, 'message-2', 1, 'email', '2024-01-02 00:00:00') + `) + require.NoError(err) + ctx, cancel := context.WithTimeout(t.Context(), serveLifecycleTestTimeout) + defer cancel() + started := make(chan struct{}) + release := make(chan struct{}) + var releaseOnce sync.Once + releaseBuild := func() { releaseOnce.Do(func() { close(release) }) } + firstBuild := true + buildCacheBeforeMessagesExportHook = func() error { + if !firstBuild { + return nil + } + firstBuild = false + // The first export has already pinned its SQLite snapshot here. + close(started) + select { + case <-release: + return nil + case <-ctx.Done(): + return ctx.Err() + } + } + jobs := newCacheBuildJobs(ctx, nil, func(context.Context, buildCacheMode) error { + _, err := buildCache(c.DatabaseDSN(), c.AnalyticsDir(), true) + return err + }) + t.Cleanup(func() { + releaseBuild() + cancel() + drainCtx, drainCancel := context.WithTimeout(context.Background(), serveLifecycleTestTimeout) + defer drainCancel() + require.True(jobs.waitContext(drainCtx)) + buildCacheBeforeMessagesExportHook = nil + }) + active, err := jobs.accept(buildCacheModeAuto) + require.NoError(err) + select { + case <-started: + case <-ctx.Done(): + require.FailNow("cache job did not reach its export snapshot") + } + result, accepted, err := runDaemonSQLQueryWithJobs(ctx, c, s, engine, "SELECT COUNT(*) FROM messages", daemonSQLQueryOptions{}, jobs) + require.NoError(err, "published rows must remain readable while the builder is held") + assert.Nil(accepted) + require.NotNil(result.Cache) + assert.True(result.Cache.Building) + assert.EqualValues(1, result.Rows[0][0]) + _, err = s.DB().Exec(` + INSERT INTO messages (id, source_id, source_message_id, conversation_id, message_type, sent_at) + VALUES (3, 1, 'message-3', 1, 'email', '2024-01-03 00:00:00') + `) + require.NoError(err) + _, first, err := runDaemonSQLQueryWithJobs(ctx, c, s, engine, "SELECT 1", daemonSQLQueryOptions{fresh: true}, jobs) + require.NoError(err) + require.NotNil(first) + assert.NotEqual(active.JobID, first.JobID) + assert.Equal(api.CacheBuildQueued, first.Status) + _, second, err := runDaemonSQLQueryWithJobs(ctx, c, s, engine, "SELECT 1", daemonSQLQueryOptions{fresh: true}, jobs) + require.NoError(err) + require.NotNil(second) + assert.Equal(first.JobID, second.JobID) + releaseBuild() + require.Eventually(func() bool { + job, ok := jobs.status(first.JobID) + return ok && (job.Status == api.CacheBuildPublished || job.Status == api.CacheBuildFailed) + }, serveLifecycleTestTimeout, 10*time.Millisecond) + job, ok := jobs.status(first.JobID) + require.True(ok) + require.Equal(api.CacheBuildPublished, job.Status, job.Error) + result, accepted, err = runDaemonSQLQueryWithJobs(ctx, c, s, engine, "SELECT COUNT(*) FROM messages", daemonSQLQueryOptions{}, jobs) + require.NoError(err) + assert.Nil(accepted) + assert.EqualValues(3, result.Rows[0][0], "fresh must include the write after the first builder's snapshot") +} + +func TestQueryWithAutomaticCacheBuildsDisabledServesStaleWithoutStartingJob(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + c, s := openTestDaemonAnalyticsStore(t) + c.Analytics.AutoBuildCache = false + c.Analytics.MinRebuildInterval = 0 + _, err := s.DB().Exec(` + INSERT INTO sources (id, source_type, identifier) VALUES (1, 'gmail', 'user@example.com'); + INSERT INTO conversations (id, source_id, source_conversation_id, conversation_type) + VALUES (1, 1, 'thread-1', 'email_thread'); + INSERT INTO messages (id, source_id, source_message_id, conversation_id, message_type, sent_at) + VALUES (1, 1, 'message-1', 1, 'email', '2024-01-01 00:00:00'); + `) + require.NoError(err) + _, err = buildCache(c.DatabaseDSN(), c.AnalyticsDir(), true) + require.NoError(err) + engine, err := openDaemonDuckDBEngine(c, s) + require.NoError(err) + t.Cleanup(func() { _ = engine.Close() }) + _, err = s.DB().Exec(` + INSERT INTO messages (id, source_id, source_message_id, conversation_id, message_type, sent_at) + VALUES (2, 1, 'message-2', 1, 'email', '2024-01-02 00:00:00') + `) + require.NoError(err) + started := make(chan struct{}, 1) + jobs := newCacheBuildJobs(t.Context(), nil, func(context.Context, buildCacheMode) error { + started <- struct{}{} + return nil + }) + result, accepted, err := runDaemonSQLQueryWithJobs(t.Context(), c, s, engine, "SELECT COUNT(*) FROM messages", daemonSQLQueryOptions{}, jobs) + require.NoError(err) + assert.Nil(accepted) + require.NotNil(result.Cache) + assert.NotEmpty(result.Cache.StaleReason) + assert.False(jobs.active()) + select { + case <-started: + assert.Fail("automatic cache build started despite auto_build_cache=false") + default: + } + _, forced, err := runDaemonSQLQueryWithJobs(t.Context(), c, s, engine, "SELECT 1", daemonSQLQueryOptions{fresh: true}, jobs) + require.NoError(err) + require.NotNil(forced) + assert.NotEmpty(forced.JobID) +} + +func TestFreshQueryVerifiesCleanPublicationInBackground(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + c, s := openTestDaemonAnalyticsStore(t) + c.Analytics.MinRebuildInterval = 6 * time.Hour + _, err := s.DB().Exec(` + INSERT INTO sources (id, source_type, identifier) VALUES (1, 'gmail', 'user@example.com'); + INSERT INTO conversations (id, source_id, source_conversation_id, conversation_type) + VALUES (1, 1, 'thread-1', 'email_thread'); + INSERT INTO messages (id, source_id, source_message_id, conversation_id, message_type, sent_at) + VALUES (1, 1, 'message-1', 1, 'email', '2024-01-01 00:00:00'); + `) + require.NoError(err) + _, err = buildCache(c.DatabaseDSN(), c.AnalyticsDir(), true) + require.NoError(err) + engine, err := openDaemonDuckDBEngine(c, s) + require.NoError(err) + t.Cleanup(func() { _ = engine.Close() }) + started := make(chan buildCacheMode, 1) + jobs := newCacheBuildJobs(t.Context(), nil, func(_ context.Context, mode buildCacheMode) error { + started <- mode + return nil + }) + result, accepted, err := runDaemonSQLQueryWithJobs(t.Context(), c, s, engine, "SELECT 1", daemonSQLQueryOptions{fresh: true}, jobs) + require.NoError(err) + assert.Nil(result) + require.NotNil(accepted) + assert.NotEmpty(accepted.JobID) + select { + case mode := <-started: + assert.Equal(buildCacheModeAuto, mode) + case <-time.After(time.Second): + require.FailNow("fresh query did not start cache verification") + } +} + +func TestSQLAnalyticsModeQueryQueuesMissingCache(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + c, s := openTestDaemonAnalyticsStore(t) + c.Analytics.Engine = config.AnalyticsEngineSQL engine := query.NewEngine(s.DB(), false) - defer func() { _ = engine.Close() }() + t.Cleanup(func() { _ = engine.Close() }) + started := make(chan buildCacheMode, 1) + jobs := newCacheBuildJobs(t.Context(), nil, func(_ context.Context, mode buildCacheMode) error { + started <- mode + return nil + }) + result, accepted, err := runDaemonSQLQueryWithJobs(t.Context(), c, s, engine, "SELECT 1", daemonSQLQueryOptions{}, jobs) + require.NoError(err) + assert.Nil(result) + require.NotNil(accepted) + assert.NotEmpty(accepted.JobID) + select { + case mode := <-started: + assert.Equal(buildCacheModeAuto, mode) + case <-time.After(time.Second): + require.FailNow("missing cache did not queue recovery") + } +} - sentinel := errors.New("subprocess sentinel") - var called bool - var gotFullRebuild bool - old := buildCacheSubprocessForRun - buildCacheSubprocessForRun = func(_ context.Context, fullRebuild bool) error { - called = true - gotFullRebuild = fullRebuild - return sentinel +func TestQueryUsesPublishedParquetWhenDaemonEngineIsSQLite(t *testing.T) { + c, s := openPublishedQueryTestStore(t) + sqliteEngine := query.NewEngine(s.DB(), false) + t.Cleanup(func() { _ = sqliteEngine.Close() }) + // Model a SQL-capable fallback without changing the real SQLite engine. + // Cache queries must still go to DuckDB when a publication exists. + engine := &sqlCapableSQLiteFallback{Engine: sqliteEngine} + jobs := newCacheBuildJobs(t.Context(), nil, func(context.Context, buildCacheMode) error { return nil }) + + for _, mode := range []string{config.AnalyticsEngineAuto, config.AnalyticsEngineSQL} { + t.Run(mode, func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + c.Analytics.Engine = mode + result, accepted, err := runDaemonSQLQueryWithJobs(t.Context(), c, s, engine, + "SELECT COUNT(*) FROM v_senders", daemonSQLQueryOptions{}, jobs) + requirements.NoError(err) + assertions.Nil(accepted) + requirements.NotNil(result) + requirements.NotNil(result.Cache) + assertions.EqualValues(0, result.Rows[0][0]) + }) } - t.Cleanup(func() { buildCacheSubprocessForRun = old }) +} - _, err = runDaemonSQLQuery(context.Background(), c, s, engine, "select 1") +type sqlCapableSQLiteFallback struct{ query.Engine } + +func (*sqlCapableSQLiteFallback) QuerySQL(context.Context, string) (*query.QueryResult, error) { + return nil, errors.New("cache query reached SQLite fallback") +} + +func TestQueryQueuesRepairWhenPublishedShardDisappears(t *testing.T) { + for _, fresh := range []bool{false, true} { + t.Run(fmt.Sprintf("fresh-%t", fresh), func(t *testing.T) { + requirements := require.New(t) + assertions := assert.New(t) + c, s := openPublishedQueryTestStore(t) + c.Analytics.AutoBuildCache = true + engine, err := openDaemonDuckDBEngine(c, s) + requirements.NoError(err) + t.Cleanup(func() { _ = engine.Close() }) + shards, err := filepath.Glob(filepath.Join(c.AnalyticsDir(), "messages", "*", "*.parquet")) + requirements.NoError(err) + requirements.NotEmpty(shards) + requirements.NoError(os.Remove(shards[0])) + started := make(chan buildCacheMode, 1) + jobs := newCacheBuildJobs(t.Context(), nil, func(_ context.Context, mode buildCacheMode) error { + started <- mode + return nil + }) - require.Error(err, "query should fail with subprocess sentinel") - require.ErrorIs(err, sentinel, "error") - assert.True(called, "subprocess rebuild should be called") - assert.True(gotFullRebuild, "missing cache should request full rebuild") + result, accepted, err := runDaemonSQLQueryWithJobs(t.Context(), c, s, engine, + "SELECT COUNT(*) FROM v_senders", daemonSQLQueryOptions{fresh: fresh}, jobs) + requirements.NoError(err) + assertions.Nil(result) + requirements.NotNil(accepted) + assertions.NotEmpty(accepted.JobID) + assertions.Nil(accepted.Cache, "damaged publication must not be advertised as queryable") + select { + case mode := <-started: + assertions.Equal(buildCacheModeAuto, mode) + case <-time.After(time.Second): + requirements.FailNow("cache repair did not start") + } + }) + } +} + +func openPublishedQueryTestStore(t *testing.T) (*config.Config, *store.Store) { + t.Helper() + requirements := require.New(t) + c, s := openTestDaemonAnalyticsStore(t) + _, err := s.DB().Exec(` + INSERT INTO sources (id, source_type, identifier) VALUES (1, 'gmail', 'user@example.com'); + INSERT INTO conversations (id, source_id, source_conversation_id, conversation_type) + VALUES (1, 1, 'thread-1', 'email_thread'); + INSERT INTO messages (id, source_id, source_message_id, conversation_id, message_type, sent_at) + VALUES (1, 1, 'message-1', 1, 'email', '2024-01-01 00:00:00'); + `) + requirements.NoError(err) + _, err = buildCache(c.DatabaseDSN(), c.AnalyticsDir(), true) + requirements.NoError(err) + return c, s +} + +func TestManualSyncRefreshQueuesVerificationInsideInterval(t *testing.T) { + requirements := require.New(t) + c, s := openTestDaemonAnalyticsStore(t) + c.Analytics.AutoBuildCache = true + c.Analytics.MinRebuildInterval = 6 * time.Hour + _, err := s.DB().Exec(` + INSERT INTO sources (id, source_type, identifier) VALUES (1, 'gmail', 'user@example.com'); + INSERT INTO conversations (id, source_id, source_conversation_id, conversation_type) + VALUES (1, 1, 'thread-1', 'email_thread'); + INSERT INTO messages (id, source_id, source_message_id, conversation_id, message_type, sent_at) + VALUES (1, 1, 'message-1', 1, 'email', '2024-01-01 00:00:00'); + `) + requirements.NoError(err) + _, err = buildCache(c.DatabaseDSN(), c.AnalyticsDir(), true) + requirements.NoError(err) + _, err = s.DB().Exec(` + INSERT INTO messages (id, source_id, source_message_id, conversation_id, message_type, sent_at) + VALUES (2, 1, 'message-2', 1, 'email', '2024-01-02 00:00:00') + `) + requirements.NoError(err) + for _, test := range []struct { + name string + args []string + interval time.Duration + shutdown bool + wantQueued bool + wantMode buildCacheMode + }{ + {name: "default inside interval", args: []string{"sync-slack"}, interval: 6 * time.Hour, wantQueued: true, wantMode: buildCacheModeScheduledAuto}, + {name: "forced inside interval", args: []string{"sync-teams", "--build-cache"}, interval: 6 * time.Hour, wantQueued: true, wantMode: buildCacheModeAuto}, + {name: "default when due", args: []string{"sync-calendar"}, wantQueued: true, wantMode: buildCacheModeScheduledAuto}, + {name: "skip when due", args: []string{"sync-slack", "--no-build-cache"}}, + {name: "shutdown skips forced refresh", args: []string{"sync-teams", "--build-cache=true"}, interval: 6 * time.Hour, shutdown: true}, + } { + t.Run(test.name, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + c.Analytics.MinRebuildInterval = test.interval + daemonCtx, cancel := context.WithCancel(t.Context()) + started := make(chan buildCacheMode, 1) + jobs := newCacheBuildJobs(daemonCtx, nil, func(ctx context.Context, mode buildCacheMode) error { + started <- mode + <-ctx.Done() + return ctx.Err() + }) + t.Cleanup(func() { + cancel() + cleanupCtx, stop := context.WithTimeout(context.Background(), 10*time.Second) + defer stop() + require.True(jobs.waitContext(cleanupCtx), "cache worker stopped") + }) + adapter := &storeAPIAdapter{store: s, config: c, cacheJobs: jobs} + runnerCalled := false + err := adapter.runCLICommandWithRunner(t.Context(), api.CLIRunRequest{Args: test.args}, nil, + func(_ context.Context, args []string, _ map[string]string, _ string, _ func(string, string) error) error { + runnerCalled = true + assert.Equal(test.args, args) + assert.False(jobs.active(), "cache work starts after the sync child returns") + if test.shutdown { + cancel() + } + return nil + }) + require.NoError(err) + assert.True(runnerCalled) + assert.Equal(test.wantQueued, jobs.active()) + if test.wantQueued { + select { + case mode := <-started: + assert.Equal(test.wantMode, mode) + case <-time.After(10 * time.Second): + require.FailNow("cache job did not start") + } + } + }) + } +} + +func TestRunDaemonSQLQueryReturnsCancellationWhenCachePublicationIsLocked(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + c, s := openTestDaemonAnalyticsStore(t) + engine := query.NewEngine(s.DB(), false) + defer func() { _ = engine.Close() }() + + publicationLock, err := cacheBuildFileLock(c.AnalyticsDir()) + require.NoError(err) + require.NoError(publicationLock.Lock()) + t.Cleanup(func() { _ = publicationLock.Unlock() }) + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + jobs := newCacheBuildJobs(t.Context(), nil, func(context.Context, buildCacheMode) error { + return errors.New("canceled query must not start a rebuild") + }) + _, accepted, err := runDaemonSQLQueryWithJobs(ctx, c, s, engine, "select 1", daemonSQLQueryOptions{}, jobs) + + require.ErrorIs(err, context.Canceled) + assert.Nil(accepted) + assert.False(jobs.active(), "a canceled query must stop before starting a cache rebuild") } func TestOpenDaemonAnalyticsEngineForceSQLSkipsCacheBuild(t *testing.T) { @@ -1003,6 +1429,31 @@ func TestOpenDaemonAnalyticsEngineSkipsCacheBuildWhenDisabled(t *testing.T) { assert.Equal(startupCacheBuildOutcomeNone, outcome, "no explicit intent has no outcome") } +func TestOpenDaemonAnalyticsEngineReturnsCancellationWhenCacheBuilderIsLocked(t *testing.T) { + require := require.New(t) + c, s := openTestDaemonAnalyticsStore(t) + c.Analytics.Engine = config.AnalyticsEngineAuto + c.Analytics.AutoBuildCache = false + + buildLock, err := cacheBuilderFileLock(c.AnalyticsDir()) + require.NoError(err) + require.NoError(buildLock.Lock()) + t.Cleanup(func() { _ = buildLock.Unlock() }) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + engine, _, _, err := openDaemonAnalyticsEngine( + ctx, c, s, startupCacheBuildIntentNone, + ) + if engine != nil { + _ = engine.Close() + } + + require.ErrorIs(err, context.Canceled) + require.Nil(engine, "canceled startup must not open an analytics engine") +} + func TestOpenDaemonAnalyticsEngineWarnsWhenDuckDBRefreshDisabled(t *testing.T) { require := require.New(t) assert := assert.New(t) @@ -1014,12 +1465,14 @@ func TestOpenDaemonAnalyticsEngineWarnsWhenDuckDBRefreshDisabled(t *testing.T) { staleness := cacheNeedsBuild(c.DatabaseDSN(), c.AnalyticsDir()) require.False(staleness.NeedsBuild, "test cache must be ready: %+v", staleness) var logs bytes.Buffer - oldLogger := logger - logger = slog.New(slog.NewTextHandler(&logs, nil)) - t.Cleanup(func() { logger = oldLogger }) + analyticsCtx := testInvocationContext(context.Background(), c, invocationOptions{}) + inv := invocationFromContext(analyticsCtx) + oldLogger := inv.logger + inv.logger = slog.New(slog.NewTextHandler(&logs, nil)) + t.Cleanup(func() { inv.logger = oldLogger }) engine, mode, outcome, err := openDaemonAnalyticsEngine( - context.Background(), c, s, startupCacheBuildIntentNone, + analyticsCtx, c, s, startupCacheBuildIntentNone, ) require.NoError(err, "openDaemonAnalyticsEngine") defer func() { _ = engine.Close() }() @@ -1117,15 +1570,17 @@ func TestOpenDaemonAnalyticsEngineAutoFallsBackWhenStartupBuildFails(t *testing. c.Analytics.Engine = config.AnalyticsEngineAuto c.Analytics.AutoBuildCache = true var logs bytes.Buffer - oldLogger := logger - logger = slog.New(slog.NewTextHandler(&logs, nil)) - t.Cleanup(func() { logger = oldLogger }) + analyticsCtx := testInvocationContext(context.Background(), c, invocationOptions{}) + inv := invocationFromContext(analyticsCtx) + oldLogger := inv.logger + inv.logger = slog.New(slog.NewTextHandler(&logs, nil)) + t.Cleanup(func() { inv.logger = oldLogger }) stubBuildCacheSubprocess(t, func(context.Context, bool) error { return errors.New("simulated build failure") }) engine, mode, outcome, err := openDaemonAnalyticsEngine( - context.Background(), c, s, startupCacheBuildIntentNone, + analyticsCtx, c, s, startupCacheBuildIntentNone, ) require.NoError(err, "a failed auto-mode build must not fail daemon startup") defer func() { _ = engine.Close() }() @@ -1138,6 +1593,158 @@ func TestOpenDaemonAnalyticsEngineAutoFallsBackWhenStartupBuildFails(t *testing. assert.Contains(logs.String(), "step=build_analytics_cache") } +// publishStaleTestCache builds a cache for the fixture archive, then adds a +// message so the publication is usable but stale. +func publishStaleTestCache(t *testing.T, c *config.Config, s *store.Store) { + t.Helper() + _, err := s.DB().Exec(` + INSERT INTO sources (id, source_type, identifier) VALUES (1, 'gmail', 'user@example.com'); + INSERT INTO conversations (id, source_id, source_conversation_id, conversation_type, title) + VALUES (1, 1, 'thread1', 'email_thread', 'Hello'); + INSERT INTO messages (id, conversation_id, source_id, source_message_id, message_type, sent_at, subject, snippet) + VALUES (1, 1, 1, 'msg1', 'email', '2024-01-15 10:00:00', 'Hello', 'Preview'); + `) + require.NoError(t, err, "insert published data") + _, err = buildCache(c.DatabaseDSN(), c.AnalyticsDir(), true) + require.NoError(t, err, "publish cache") + _, err = s.DB().Exec(` + INSERT INTO messages (id, conversation_id, source_id, source_message_id, message_type, sent_at, subject, snippet) + VALUES (2, 1, 1, 'msg2', 'email', '2024-01-16 10:00:00', 'Later', 'Arrived after publication'); + `) + require.NoError(t, err, "insert unpublished message") +} + +func TestOpenDaemonAnalyticsEngineDefersThrottledStartupBuild(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + c, s := openTestDaemonAnalyticsStore(t) + c.Analytics.Engine = config.AnalyticsEngineAuto + c.Analytics.AutoBuildCache = true + c.Analytics.MinRebuildInterval = 6 * time.Hour + publishStaleTestCache(t, c, s) + var logs bytes.Buffer + state := testInvocationWithConfig(c) + state.logger = slog.New(slog.NewTextHandler(&logs, nil)) + testCtx := withInvocation(t.Context(), state) + builds := 0 + stubBuildCacheSubprocess(t, func(context.Context, bool) error { + builds++ + return errors.New("startup must not build within the rebuild interval") + }) + + engine, mode, _, err := openDaemonAnalyticsEngine( + testCtx, c, s, startupCacheBuildIntentNone, + ) + require.NoError(err) + defer func() { _ = engine.Close() }() + + assert.Zero(builds, "a recent usable publication defers the startup build") + assert.Equal(api.AnalyticsModeDuckDB, mode, "the existing publication is served") + assert.Contains(logs.String(), "rebuild deferred by min_rebuild_interval") +} + +func TestOpenDaemonAnalyticsEngineServesPartialPublication(t *testing.T) { + for _, mode := range []string{config.AnalyticsEngineAuto, config.AnalyticsEngineDuckDB} { + for _, interval := range []time.Duration{0, 6 * time.Hour} { + t.Run(fmt.Sprintf("%s/%s", mode, interval), func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + c, s := openTestDaemonAnalyticsStore(t) + c.Analytics.Engine = mode + c.Analytics.AutoBuildCache = true + c.Analytics.MinRebuildInterval = interval + publishStaleTestCache(t, c, s) + state, err := query.ReadCacheSyncState(c.AnalyticsDir()) + require.NoError(err) + state.FullRebuildRequired = true + stateData, err := json.Marshal(state) + require.NoError(err) + require.NoError(os.WriteFile(query.CacheStatePath(c.AnalyticsDir()), stateData, 0o600)) + + builds := 0 + stubBuildCacheSubprocess(t, func(_ context.Context, full bool) error { + builds++ + assert.True(full, "the next build repairs the partial snapshot in full") + return errors.New("simulated repair failure") + }) + engine, gotMode, _, err := openDaemonAnalyticsEngine( + context.Background(), c, s, startupCacheBuildIntentNone, + ) + require.NoError(err, "a usable partial publication keeps startup available") + defer func() { _ = engine.Close() }() + assert.Equal(api.AnalyticsModeDuckDB, gotMode) + stats, err := engine.GetTotalStats(context.Background(), query.StatsOptions{}) + require.NoError(err) + assert.Equal(int64(1), stats.MessageCount, "analytics still query the published snapshot") + if interval > 0 { + assert.Zero(builds, "a recent partial publication honors the rebuild interval") + } else { + assert.Equal(1, builds, "a due publication still attempts its full repair") + } + }) + } + } +} + +func TestOpenDaemonAnalyticsEngineServesUsablePublicationWhenStartupBuildFails(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + c, s := openTestDaemonAnalyticsStore(t) + c.Analytics.Engine = config.AnalyticsEngineAuto + c.Analytics.AutoBuildCache = true + publishStaleTestCache(t, c, s) + builds := 0 + stubBuildCacheSubprocess(t, func(context.Context, bool) error { + builds++ + return errors.New("simulated build failure") + }) + + engine, mode, _, err := openDaemonAnalyticsEngine( + context.Background(), c, s, startupCacheBuildIntentNone, + ) + require.NoError(err) + defer func() { _ = engine.Close() }() + + assert.Equal(1, builds, "without a rebuild interval the startup build runs") + assert.Equal(api.AnalyticsModeDuckDB, mode, + "a failed build keeps serving the last usable publication, not live SQL") +} + +func TestOpenDaemonAnalyticsEngineDuckDBServesUsablePublicationWhenStartupBuildFails(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + c, s := openTestDaemonAnalyticsStore(t) + c.Analytics.Engine = config.AnalyticsEngineDuckDB + c.Analytics.AutoBuildCache = true + publishStaleTestCache(t, c, s) + stubBuildCacheSubprocess(t, func(context.Context, bool) error { + return errors.New("simulated build failure") + }) + + engine, mode, _, err := openDaemonAnalyticsEngine( + context.Background(), c, s, startupCacheBuildIntentNone, + ) + require.NoError(err, "a usable publication keeps engine=duckdb startable after a failed build") + defer func() { _ = engine.Close() }() + assert.Equal(api.AnalyticsModeDuckDB, mode) +} + +func TestOpenDaemonAnalyticsEngineDuckDBServesStalePublicationWithoutAutoBuild(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + c, s := openTestDaemonAnalyticsStore(t) + c.Analytics.Engine = config.AnalyticsEngineDuckDB + c.Analytics.AutoBuildCache = false + publishStaleTestCache(t, c, s) + + engine, mode, _, err := openDaemonAnalyticsEngine( + context.Background(), c, s, startupCacheBuildIntentNone, + ) + require.NoError(err, "engine=duckdb serves a usable publication even when stale") + defer func() { _ = engine.Close() }() + assert.Equal(api.AnalyticsModeDuckDB, mode) +} + func TestOpenDaemonAnalyticsEngineDuckDBRequiresCacheBuild(t *testing.T) { require := require.New(t) c, s := openTestDaemonAnalyticsStore(t) @@ -1428,8 +2035,6 @@ func TestStoreAPIAdapterServesSourceStatus(t *testing.T) { require.NotNil(got.LastSuccessfulSync, "LastSuccessfulSync") assert.Equal(completedID, got.LastSuccessfulSync.ID, "LastSuccessfulSync.ID") assert.Equal(store.SyncStatusCompleted, got.LastSuccessfulSync.Status, "LastSuccessfulSync.Status") - require.NotNil(got.LastSuccessfulSync.CursorAfter, "LastSuccessfulSync.CursorAfter") - assert.Equal("history-2", *got.LastSuccessfulSync.CursorAfter, "LastSuccessfulSync.CursorAfter") } func TestStoreAPIAdapterRunCLISyncPacksOnlyAfterSubprocessSuccess(t *testing.T) { @@ -1522,6 +2127,11 @@ func TestCLISyncSubprocessArgsIncludesExactSourceID(t *testing.T) { } func TestDaemonCLIRunCannotUseServerRemoteDeleteConfigOrEnvironment(t *testing.T) { + cfg := testConfigValue() + cfgFile := "" + homeDir := "" + useLocal := false + require := require.New(t) assert := assert.New(t) @@ -1579,7 +2189,7 @@ remote_enabled = true daemon := api.NewServerWithOptions(api.ServerOptions{ Config: serverCfg, - Store: &storeAPIAdapter{store: st}, + Store: &storeAPIAdapter{store: st, config: serverCfg, options: invocationOptions{cfgFile: configPath}}, Logger: slog.New(slog.DiscardHandler), }) body, err := json.Marshal(api.CLIRunRequest{ @@ -1854,7 +2464,7 @@ func TestStoreAPIAdapterInterceptsExplicitRepackInDaemonParent(t *testing.T) { ) require.NoError(err) - has, err := f.store.HasPackRecord(oldPackID) + has, err := f.store.HasPackRecordContext(t.Context(), oldPackID) require.NoError(err) assert.False(has) require.Len(events, 1) @@ -1899,7 +2509,7 @@ func TestStoreAPIAdapterExplicitRepackAcceptsLoggingPassthroughFlags(t *testing. ) require.NoError(err) - has, err := f.store.HasPackRecord(oldPackID) + has, err := f.store.HasPackRecordContext(t.Context(), oldPackID) require.NoError(err) assert.False(has) } @@ -1964,7 +2574,7 @@ func TestStoreAPIAdapterRepackAfterSuccessfulRemovalOnly(t *testing.T) { require.NoError(err) } assert.Equal(1, runnerCalls) - has, hasErr := f.store.HasPackRecord(oldPackID) + has, hasErr := f.store.HasPackRecordContext(t.Context(), oldPackID) require.NoError(hasErr) assert.Equal(!tt.wantRemoved, has) }) @@ -2018,7 +2628,7 @@ func TestStoreAPIAdapterPostRemovalRepackCancellationPreservesSuccess(t *testing require.NoError(err, "maintenance cancellation cannot erase committed removal success") assert.Empty(events, "cancellation is informational, not a streamed warning") - has, err := f.store.HasPackRecord(oldPackID) + has, err := f.store.HasPackRecordContext(t.Context(), oldPackID) require.NoError(err) assert.True(has, "canceled cleanup remains inventoried for retry") assert.Contains(f.logs.String(), "automatic attachment repack canceled") @@ -2042,7 +2652,7 @@ func TestStoreAPIAdapterExplicitRepackCancellationFailsFast(t *testing.T) { ) require.ErrorIs(err, context.Canceled) - has, getErr := f.store.HasPackRecord(oldPackID) + has, getErr := f.store.HasPackRecordContext(t.Context(), oldPackID) require.NoError(getErr) assert.True(has, "fail-fast cancellation leaves physical inventory untouched") } @@ -2203,17 +2813,24 @@ func TestStoreAPIAdapterServesCLIDeleteDeduped(t *testing.T) { // cfg.Vector.Enabled is false, setupVectorFeatures returns (nil, nil) // regardless of build tag. Runs under both tagged and untagged builds. func TestSetupVectorFeatures_Disabled(t *testing.T) { + cfg := testConfigValue() + savedCfg := cfg defer func() { cfg = savedCfg }() cfg = &config.Config{} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Vector.Enabled = false - vf, err := setupVectorFeatures(context.Background(), nil, "", false) + vf, err := setupVectorFeatures(testCtx, nil, "", false) require.NoError(t, err, "setupVectorFeatures") assert.Nil(t, vf, "setupVectorFeatures should be nil when disabled") } func TestRunScheduledGmailSync_ReauthGuidance(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + for _, tc := range []struct { name, scope, flags string }{ @@ -2225,12 +2842,15 @@ func TestRunScheduledGmailSync_ReauthGuidance(t *testing.T) { require := require.New(t) // An expired token without a refresh token fails locally, without // contacting Google or opening an authorization flow. - _, restore := seedTokenEnv(t, fmt.Sprintf(`{"access_token":"expired","expiry":"2000-01-01T00:00:00Z","scopes":[%q]}`, tc.scope)) + tokenPath, restore := seedTokenEnv(t, fmt.Sprintf(`{"access_token":"expired","expiry":"2000-01-01T00:00:00Z","scopes":[%q]}`, tc.scope)) defer restore() + cfg = testConfigValue() + cfg.OAuth.ClientSecrets = filepath.Join(filepath.Dir(filepath.Dir(tokenPath)), "client_secret.json") mgr, err := oauth.NewManager(cfg.OAuth.ClientSecrets, cfg.TokensDir(), logger) require.NoError(err) - _, err = runScheduledGmailSync(t.Context(), scopeEscalationAccount, nil, nil, - func(string) (*oauth.Manager, error) { return mgr, nil }) + scheduledCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _, err = runScheduledGmailSync(scheduledCtx, scopeEscalationAccount, nil, nil, + func(string) (*oauth.Manager, error) { return mgr, nil }, invocationFromContext(scheduledCtx)) require.Error(err) assert.Contains(err.Error(), "msgvault add-account user@example.com"+tc.flags+" --force") assert.Contains(err.Error(), "msgvault add-account user@example.com"+tc.flags+" --headless") @@ -2238,17 +2858,133 @@ func TestRunScheduledGmailSync_ReauthGuidance(t *testing.T) { } } +func TestDaemonGmailClientUsesSourceCredentialsWithoutScopeUpgrade(t *testing.T) { + // Credential fixtures set process environment; keep these cases sequential. + t.Run("named OAuth app", func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + tokenPath, restore := seedTokenEnv(t, fmt.Sprintf(`{"access_token":"synthetic-access","expiry":"2099-01-01T00:00:00Z","scopes":[%q]}`, oauth.ScopeGmailReadonly)) + defer restore() + cfg := testConfigValue() + logger := testLoggerValue() + cfg.OAuth.ClientSecrets = filepath.Join(filepath.Dir(filepath.Dir(tokenPath)), "client_secret.json") + ctx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + before, err := os.ReadFile(tokenPath) + require.NoError(err) + cfg.OAuth.Apps = map[string]config.OAuthApp{"archive": {ClientSecrets: cfg.OAuth.ClientSecrets}} + source := &store.Source{SourceType: "gmail", Identifier: scopeEscalationAccount, OAuthApp: sql.NullString{String: "archive", Valid: true}} + var selected string + client, _, err := newDaemonGmailClient(ctx, source.Identifier, source, func(app string) (*oauth.Manager, error) { + selected = app + return oauth.NewManager(cfg.OAuth.Apps[app].ClientSecrets, cfg.TokensDir(), logger) + }, invocationFromContext(ctx)) + require.NoError(err) + t.Cleanup(func() { _ = client.Close() }) + assert.Equal("archive", selected) + after, err := os.ReadFile(tokenPath) + require.NoError(err) + assert.Equal(before, after, "reading a profile must not upgrade the stored grant") + }) + t.Run("named service account", func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + cfg := testConfigValue() + ctx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + // A missing source-bound key fails locally. It must never fall back to + // the otherwise configured interactive OAuth app or open authorization. + cfg.OAuth.Apps = map[string]config.OAuthApp{"delegated": {ServiceAccountKey: filepath.Join(t.TempDir(), "missing-key.json")}} + source := &store.Source{SourceType: "gmail", Identifier: scopeEscalationAccount, OAuthApp: sql.NullString{String: "delegated", Valid: true}} + called := false + client, _, err := newDaemonGmailClient(ctx, source.Identifier, source, func(string) (*oauth.Manager, error) { + called = true + return nil, errors.New("unexpected OAuth fallback") + }, invocationFromContext(ctx)) + require.ErrorContains(err, "service account") + credentialErr, ok := errors.AsType[*provideridentity.GmailCredentialError](err) + require.True(ok) + assert.Contains(credentialErr.Remediation(), "service_account_key") + assert.Nil(client) + assert.False(called) + }) +} + +func TestDaemonGmailClientCredentialFailures(t *testing.T) { + // Token fixtures set process environment; keep these cases sequential. + for _, tc := range []struct { + name, providerCode, remediation string + removeToken bool + }{ + {"missing token", "", "msgvault add-account", true}, + {"revoked token", "invalid_grant", "expired or been revoked", false}, + {"unexpected provider error", "server_error", "", false}, + } { + t.Run(tc.name, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + tokenPath, restore := seedTokenEnv(t, `{"access_token":"expired","refresh_token":"synthetic-refresh","expiry":"2000-01-01T00:00:00Z"}`) + defer restore() + cfg := testConfigValue() + cfg.OAuth.ClientSecrets = filepath.Join(filepath.Dir(filepath.Dir(tokenPath)), "client_secret.json") + provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusBadRequest) + _, _ = fmt.Fprintf(w, `{"error":%q,"error_description":"synthetic-secret-token"}`, tc.providerCode) + })) + t.Cleanup(provider.Close) + var secrets map[string]map[string]any + require.NoError(json.Unmarshal([]byte(fakeClientSecrets), &secrets)) + secrets["installed"]["token_uri"] = provider.URL + encoded, err := json.Marshal(secrets) + require.NoError(err) + require.NoError(os.WriteFile(cfg.OAuth.ClientSecrets, encoded, 0600)) + if tc.removeToken { + require.NoError(os.Remove(tokenPath)) + } + ctx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + source := &store.Source{SourceType: "gmail", Identifier: scopeEscalationAccount} + client, _, err := newDaemonGmailClient(ctx, source.Identifier, source, oauthManagerCache(invocationFromContext(ctx)), invocationFromContext(ctx)) + require.Error(err) + assert.Nil(client) + credentialErr, ok := errors.AsType[*provideridentity.GmailCredentialError](err) + if tc.remediation == "" { + assert.False(ok, "unexpected provider errors must remain internal") + return + } + require.True(ok) + assert.Contains(credentialErr.Remediation(), tc.remediation) + assert.NotContains(credentialErr.Remediation(), "synthetic-secret-token") + }) + } + t.Run("missing OAuth configuration", func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + cfg := &config.Config{} + ctx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + source := &store.Source{SourceType: "gmail", Identifier: "owner@example.test"} + client, _, err := newDaemonGmailClient(ctx, source.Identifier, source, oauthManagerCache(invocationFromContext(ctx)), invocationFromContext(ctx)) + require.Error(err) + assert.Nil(client) + credentialErr, ok := errors.AsType[*provideridentity.GmailCredentialError](err) + require.True(ok) + assert.Contains(credentialErr.Remediation(), "OAuth app and client-secrets file") + }) +} + // TestRunScheduledIMAPSync_NoCredentials verifies that the IMAP path // in runScheduledSync is reachable — i.e. an IMAP source row makes the // dispatcher build an IMAP client and surface a credentials error, // rather than the misleading "oauth2: token expired and refresh token // is not set" message reported in #329. func TestRunScheduledIMAPSync_NoCredentials(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) savedCfg := cfg defer func() { cfg = savedCfg }() cfg = &config.Config{} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Data.DataDir = t.TempDir() s, err := store.Open(filepath.Join(cfg.Data.DataDir, "msgvault.db")) @@ -2269,7 +3005,7 @@ func TestRunScheduledIMAPSync_NoCredentials(t *testing.T) { return nil, nil //nolint:nilnil // unreachable guard, see comment above } - err = runScheduledSync(context.Background(), imapID, s, getOAuthMgr) + err = runScheduledSync(testCtx, imapID, s, getOAuthMgr, invocationFromContext(testCtx)) require.Error(err, "runScheduledSync(imap, no creds) want credentials error") msg := err.Error() assert.False(strings.Contains(msg, "refresh token") || strings.Contains(msg, "token may be expired"), @@ -2285,11 +3021,15 @@ func TestRunScheduledIMAPSync_NoCredentials(t *testing.T) { // matched against identifier, so config-driven scheduled syncs fell // through to the Gmail OAuth path (#329). func TestRunScheduledIMAPSync_DispatchByDisplayName(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) savedCfg := cfg defer func() { cfg = savedCfg }() cfg = &config.Config{} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Data.DataDir = t.TempDir() s, err := store.Open(filepath.Join(cfg.Data.DataDir, "msgvault.db")) @@ -2315,7 +3055,7 @@ func TestRunScheduledIMAPSync_DispatchByDisplayName(t *testing.T) { // Pass the email (as config.toml `email = "..."` would supply it), // not the imaps:// identifier. Dispatch must still land on the // IMAP path; absence of credentials produces an IMAP-shaped error. - err = runScheduledSync(context.Background(), imapEmail, s, getOAuthMgr) + err = runScheduledSync(testCtx, imapEmail, s, getOAuthMgr, invocationFromContext(testCtx)) require.Error(err, "runScheduledSync(email, no creds) want IMAP credentials error") msg := err.Error() assert.False(strings.Contains(msg, "refresh token") || strings.Contains(msg, "token may be expired"), @@ -2330,11 +3070,15 @@ func TestRunScheduledIMAPSync_DispatchByDisplayName(t *testing.T) { // would inject e.g. "imaps://user@host:993" into account_identities // when the user had cleared their identities. func TestRunScheduledIMAPSync_DefaultIdentityIsDisplayName(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) assert := assert.New(t) savedCfg := cfg defer func() { cfg = savedCfg }() cfg = &config.Config{} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Data.DataDir = t.TempDir() s, err := store.Open(filepath.Join(cfg.Data.DataDir, "msgvault.db")) @@ -2366,7 +3110,7 @@ func TestRunScheduledIMAPSync_DefaultIdentityIsDisplayName(t *testing.T) { // Expected to fail at the IMAP connection; what matters is that // confirmDefaultIdentity ran first with the display_name. - _ = runScheduledSync(context.Background(), imapID, s, getOAuthMgr) + _ = runScheduledSync(testCtx, imapID, s, getOAuthMgr, invocationFromContext(testCtx)) identities, err := s.ListAccountIdentities(src.ID) require.NoError(err, "ListAccountIdentities") @@ -2469,6 +3213,8 @@ func TestFindScheduledSyncSources(t *testing.T) { } func TestScheduledTeamsImportOptionsApplyMediaPolicy(t *testing.T) { + cfg := testConfigValue() + oldConfig := cfg t.Cleanup(func() { cfg = oldConfig }) enabled := true @@ -2481,8 +3227,10 @@ func TestScheduledTeamsImportOptionsApplyMediaPolicy(t *testing.T) { }, }, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx - opts := scheduledTeamsImportOptions("user@example.com") + opts := scheduledTeamsImportOptions("user@example.com", cfg) assert.Equal(t, cfg.Teams.MediaPolicy("user@example.com"), opts.MediaPolicy) assert.Equal(t, cfg.AttachmentsDir(), opts.AttachmentsDir) assert.True(t, opts.IncludeChannels) @@ -2491,6 +3239,8 @@ func TestScheduledTeamsImportOptionsApplyMediaPolicy(t *testing.T) { func TestRunScheduledSyncUsesSharedDiscordImporterAndRebuildsOnce(t *testing.T) { require := require.New(t) assert := assert.New(t) + cfg := config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) st := storetest.New(t) source, err := st.Store.GetOrCreateSource(sourceTypeDiscord, "113456789012345678") require.NoError(err) @@ -2521,29 +3271,135 @@ func TestRunScheduledSyncUsesSharedDiscordImporterAndRebuildsOnce(t *testing.T) return nil } - err = runScheduledSync(context.Background(), source.Identifier, st.Store, func(string) (*oauth.Manager, error) { + err = runScheduledSync(testCtx, source.Identifier, st.Store, func(string) (*oauth.Manager, error) { require.FailNow("Discord scheduled sync must not resolve Gmail OAuth") return nil, errors.New("unreachable Gmail OAuth resolution") - }) + }, invocationFromContext(testCtx)) require.ErrorContains(err, "synthetic Discord import failure") assert.Equal([]int64{source.ID}, imported) assert.Equal(1, rebuilds) } +func TestRunScheduledSyncStopsAfterYield(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := storetest.New(t) + source, err := st.Store.GetOrCreateSource(sourceTypeDiscord, "113456789012345678") + require.NoError(err) + + originalImport := importDiscordSourceForScheduledRun + originalRebuild := rebuildCacheAfterScheduledSourceRun + t.Cleanup(func() { + importDiscordSourceForScheduledRun = originalImport + rebuildCacheAfterScheduledSourceRun = originalRebuild + }) + + ctx, cancel := context.WithCancelCause(context.Background()) + cancel(scheduler.ErrYieldedToWaiter) + imports, rebuilds := 0, 0 + importDiscordSourceForScheduledRun = func(ctx context.Context, _ *store.Store, _ *store.Source, + _ discordCommandDeps, _ bool, _ time.Time, _ func(string), + ) (*discord.ImportSummary, error) { + imports++ + return nil, ctx.Err() + } + rebuildCacheAfterScheduledSourceRun = func(context.Context, string) error { + rebuilds++ + return nil + } + + err = runScheduledSync(ctx, source.Identifier, st.Store, func(string) (*oauth.Manager, error) { + return nil, errors.New("unexpected Gmail OAuth resolution") + }, testInvocationWithConfig(testConfigValue())) + require.ErrorIs(err, scheduler.ErrYieldedToWaiter) + assert.Equal(1, imports) + assert.Zero(rebuilds, "a yielded run releases the gate before cache rebuilding") +} + +func TestScheduledSyncYieldResultPreservesCollectedErrors(t *testing.T) { + ctx, cancel := context.WithCancelCause(context.Background()) + cancel(scheduler.ErrYieldedToWaiter) + + sourceErr := errors.New("synthetic earlier source failure") + err := scheduledSyncYieldResult(ctx, sourceErr) + + require.ErrorIs(t, err, sourceErr) + require.ErrorIs(t, err, scheduler.ErrYieldedToWaiter) +} + +func TestRunScheduledSyncCooperativePreemptionPreservesSourceResult(t *testing.T) { + for _, tt := range []struct { + name string + sourceErr error + }{ + {name: "successful source"}, + {name: "source error", sourceErr: errors.New("synthetic Discord import failure")}, + } { + t.Run(tt.name, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := storetest.New(t) + source, err := st.Store.GetOrCreateSource(sourceTypeDiscord, "113456789012345678") + require.NoError(err) + + originalImport := importDiscordSourceForScheduledRun + originalRebuild := rebuildCacheAfterScheduledSourceRun + t.Cleanup(func() { + importDiscordSourceForScheduledRun = originalImport + rebuildCacheAfterScheduledSourceRun = originalRebuild + }) + importDiscordSourceForScheduledRun = func( + context.Context, *store.Store, *store.Source, + discordCommandDeps, bool, time.Time, func(string), + ) (*discord.ImportSummary, error) { + return &discord.ImportSummary{}, tt.sourceErr + } + rebuilds := 0 + rebuildCacheAfterScheduledSourceRun = func(context.Context, string) error { + rebuilds++ + return nil + } + + ctx, requestPreemption := jobctx.WithPreemption(context.Background()) + requestPreemption() + err = runScheduledSync(ctx, source.Identifier, st.Store, func(string) (*oauth.Manager, error) { + require.FailNow("Discord scheduled sync must not resolve Gmail OAuth") + return nil, errors.New("unreachable Gmail OAuth resolution") + }, testInvocationWithConfig(testConfigValue())) + + if tt.sourceErr == nil { + require.NoError(err, "successful cooperative yield is not a callback error") + } else { + require.ErrorIs(err, tt.sourceErr, "cooperative yield preserves the source error") + } + assert.Zero(rebuilds, "a cooperative yield releases the gate before cache rebuilding") + }) + } +} + +func TestScheduledSyncYieldedForCooperativePreemption(t *testing.T) { + ctx, requestPreemption := jobctx.WithPreemption(context.Background()) + requestPreemption() + require.True(t, scheduledSyncYielded(ctx)) +} + func TestRunScheduledSyncLogsDiscordImportIssues(t *testing.T) { require := require.New(t) assert := assert.New(t) + cfg := config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + inv := invocationFromContext(testCtx) st := storetest.New(t) source, err := st.Store.GetOrCreateSource(sourceTypeDiscord, "113456789012345678") require.NoError(err) originalImport := importDiscordSourceForScheduledRun originalRebuild := rebuildCacheAfterScheduledSourceRun - originalLogger := logger + originalLogger := inv.logger t.Cleanup(func() { importDiscordSourceForScheduledRun = originalImport rebuildCacheAfterScheduledSourceRun = originalRebuild - logger = originalLogger + inv.logger = originalLogger }) importDiscordSourceForScheduledRun = func( context.Context, *store.Store, *store.Source, @@ -2564,14 +3420,15 @@ func TestRunScheduledSyncLogsDiscordImportIssues(t *testing.T) { } rebuildCacheAfterScheduledSourceRun = func(context.Context, string) error { return nil } var logs bytes.Buffer - logger = slog.New(slog.NewTextHandler(&logs, nil)) + inv.logger = slog.New(slog.NewTextHandler(&logs, nil)) require.NoError(runScheduledSync( - context.Background(), source.Identifier, st.Store, + testCtx, source.Identifier, st.Store, func(string) (*oauth.Manager, error) { require.FailNow("Discord scheduled sync must not resolve Gmail OAuth") return nil, errors.New("unreachable") }, + inv, )) output := logs.String() assert.Contains(output, "discord catalog issue") @@ -2586,9 +3443,24 @@ func TestRunScheduledSyncLogsDiscordImportIssues(t *testing.T) { assert.NotContains(output, "private-response-secret") } +func TestScheduledSyncPreemptibleRequiresResumableSources(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := storetest.New(t) + gmail, err := st.Store.GetOrCreateSource(sourceTypeGmail, "reader@example.test") + require.NoError(err) + imap, err := st.Store.GetOrCreateSource(sourceTypeIMAP, "imaps://reader@example.test:993") + require.NoError(err) + + assert.True(scheduledSyncPreemptible(st.Store, gmail.Identifier, testDiscardLogger())) + assert.False(scheduledSyncPreemptible(st.Store, imap.Identifier, testDiscardLogger()), "IMAP uses a full pass with non-resumable offsets") + assert.True(scheduledSyncPreemptible(st.Store, "new-reader@example.test", testDiscardLogger()), "missing source rows use the resumable Gmail fallback") +} + func TestScheduledDiscordGuildFailureDoesNotBlockLaterGuild(t *testing.T) { require := require.New(t) assert := assert.New(t) + cfg := config.NewDefaultConfig() st := storetest.New(t) first, err := st.Store.GetOrCreateSource(sourceTypeDiscord, "113456789012345678") require.NoError(err) @@ -2621,9 +3493,10 @@ func TestScheduledDiscordGuildFailureDoesNotBlockLaterGuild(t *testing.T) { completed := make(chan string, 2) sched := scheduler.New(func(ctx context.Context, identifier string) error { - err := runScheduledSync(ctx, identifier, st.Store, func(string) (*oauth.Manager, error) { + scheduledCtx := testInvocationContext(ctx, cfg, invocationOptions{}) + err := runScheduledSync(scheduledCtx, identifier, st.Store, func(string) (*oauth.Manager, error) { return nil, errors.New("unreachable Gmail OAuth resolution") - }) + }, invocationFromContext(scheduledCtx)) completed <- identifier return err }) diff --git a/cmd/msgvault/cmd/serve_vector.go b/cmd/msgvault/cmd/serve_vector.go index 69b57aab3..ee8a484a3 100644 --- a/cmd/msgvault/cmd/serve_vector.go +++ b/cmd/msgvault/cmd/serve_vector.go @@ -7,7 +7,6 @@ import ( "database/sql" "errors" "fmt" - "go.kenn.io/docbank/document/voyage" "log/slog" "net/http" "path/filepath" @@ -16,6 +15,9 @@ import ( "strings" "time" + "go.kenn.io/docbank/document/voyage" + + "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/providercredentials" "go.kenn.io/msgvault/internal/scheduler" "go.kenn.io/msgvault/internal/store" @@ -249,14 +251,7 @@ func newEmbeddingRuntime(vectorCfg vector.Config, deps embeddingRuntimeDeps) (*e apiKey := deps.APIKey switch vectorCfg.Embeddings.EffectiveAPIFormat() { case vector.APIFormatOpenAI: - clientConfig := embed.Config{ - Endpoint: vectorCfg.Embeddings.Endpoint, APIKey: apiKey, - Model: vectorCfg.Embeddings.Model, Dimension: vectorCfg.Embeddings.Dimension, - Timeout: vectorCfg.Embeddings.Timeout, MaxRetries: vectorCfg.Embeddings.MaxRetries, - DocumentPrefix: vectorCfg.Embeddings.DocumentPrefix, - QueryPrefix: vectorCfg.Embeddings.QueryPrefix, - RejectRedirects: true, - } + clientConfig := openAIEmbedConfig(vectorCfg, apiKey) messageClient := embed.NewClient(clientConfig) documentClientConfig := clientConfig documentClientConfig.BeforeRequest = deps.DocumentGate @@ -290,24 +285,15 @@ func newEmbeddingRuntime(vectorCfg vector.Config, deps embeddingRuntimeDeps) (*e QuerySemanticClient: queryClient, }, nil case vector.APIFormatVoyageContextual: - if vectorCfg.Embeddings.Model != "voyage-context-4" { - return nil, fmt.Errorf("vector.embeddings.model: api_format=%q requires %q, got %q", - vector.APIFormatVoyageContextual, "voyage-context-4", vectorCfg.Embeddings.Model) + clientConfig, err := voyageContextualEmbedConfig(vectorCfg, apiKey) + if err != nil { + return nil, err } publisher, ok := deps.Backend.(vector.DocumentPublisher) if !ok { return nil, errors.New("voyage contextual embeddings require a document publisher backend") } - clientConfig := embed.VoyageConfig{ - Endpoint: vectorCfg.Embeddings.Endpoint, APIKey: apiKey, - Model: vectorCfg.Embeddings.Model, Dimension: vectorCfg.Embeddings.Dimension, - Timeout: vectorCfg.Embeddings.Timeout, MaxRetries: vectorCfg.Embeddings.MaxRetries, - DocumentPrefix: vectorCfg.Embeddings.DocumentPrefix, - QueryPrefix: vectorCfg.Embeddings.QueryPrefix, - RejectRedirects: true, - Limits: embed.RequestLimits{MaxDocuments: vectorCfg.Embeddings.BatchSize, - MaxChunks: 16_000, MaxUTF8Bytes: contextualDocumentUTF8Limit}, - } + messageClient := embed.NewVoyageClient(clientConfig) documentClientConfig := clientConfig documentClientConfig.BeforeRequest = deps.DocumentGate @@ -351,6 +337,62 @@ func newEmbeddingRuntime(vectorCfg vector.Config, deps embeddingRuntimeDeps) (*e } } +// openAIEmbedConfig keeps indexing and query clients on the same settings. +func openAIEmbedConfig(vectorCfg vector.Config, apiKey string) embed.Config { + return embed.Config{ + Endpoint: vectorCfg.Embeddings.Endpoint, APIKey: apiKey, + Model: vectorCfg.Embeddings.Model, Dimension: vectorCfg.Embeddings.Dimension, + Timeout: vectorCfg.Embeddings.Timeout, MaxRetries: vectorCfg.Embeddings.MaxRetries, + DocumentPrefix: vectorCfg.Embeddings.DocumentPrefix, + QueryPrefix: vectorCfg.Embeddings.QueryPrefix, + RejectRedirects: true, + } +} + +// voyageContextualEmbedConfig keeps indexing and query clients on the same settings. +func voyageContextualEmbedConfig(vectorCfg vector.Config, apiKey string) (embed.VoyageConfig, error) { + if vectorCfg.Embeddings.Model != "voyage-context-4" { + return embed.VoyageConfig{}, fmt.Errorf("vector.embeddings.model: api_format=%q requires %q, got %q", + vector.APIFormatVoyageContextual, "voyage-context-4", vectorCfg.Embeddings.Model) + } + return embed.VoyageConfig{ + Endpoint: vectorCfg.Embeddings.Endpoint, APIKey: apiKey, + Model: vectorCfg.Embeddings.Model, Dimension: vectorCfg.Embeddings.Dimension, + Timeout: vectorCfg.Embeddings.Timeout, MaxRetries: vectorCfg.Embeddings.MaxRetries, + DocumentPrefix: vectorCfg.Embeddings.DocumentPrefix, + QueryPrefix: vectorCfg.Embeddings.QueryPrefix, + RejectRedirects: true, + Limits: embed.RequestLimits{MaxDocuments: vectorCfg.Embeddings.BatchSize, + MaxChunks: 16_000, MaxUTF8Bytes: contextualDocumentUTF8Limit}, + }, nil +} + +// newQueryEmbeddingClient selects the query-time embedding client for the +// configured vector.embeddings.api_format, using the same constructors +// newEmbeddingRuntime uses for the indexing side. +// +// Query-only callers (search, eval) need this rather than newEmbeddingRuntime: +// they never embed a document, so they must not require the document +// publisher backend or build an embed worker. Constructing embed.NewClient +// unconditionally here would send an OpenAI-compatible request body to +// Voyage's /contextualizedembeddings endpoint under a config that indexed with +// the contextual one — the wrong endpoint, the wrong request shape, and no +// input_type=query role. +func newQueryEmbeddingClient(vectorCfg vector.Config, apiKey string) (hybrid.EmbeddingClient, error) { + switch vectorCfg.Embeddings.EffectiveAPIFormat() { + case vector.APIFormatOpenAI: + return embed.NewClient(openAIEmbedConfig(vectorCfg, apiKey)), nil + case vector.APIFormatVoyageContextual: + clientConfig, err := voyageContextualEmbedConfig(vectorCfg, apiKey) + if err != nil { + return nil, err + } + return embed.NewVoyageClient(clientConfig), nil + default: + return nil, fmt.Errorf("unsupported embedding api format %q", vectorCfg.Embeddings.APIFormat) + } +} + func newConvergenceChecker( vectorCfg vector.Config, mainStore *store.Store, @@ -387,7 +429,10 @@ func newConvergenceChecker( // pgvector tag, a SQLite path needs the sqlite_vec tag. Without this, // setupVectorFeatures would only discover the gap later inside the // background init goroutine. -func precheckVectorFeatures(mainPath string) error { +func precheckVectorFeatures(mainPath string, cfg *config.Config) error { + if cfg == nil { + return errors.New("configuration is unavailable") + } if !cfg.Vector.AnyLaneEnabled() { return nil } @@ -443,6 +488,12 @@ func precheckVectorFeatures(mainPath string) error { // those writes); Migrate still runs there because it only touches the // separate vectors.db, which is read-write regardless. func setupVectorFeatures(ctx context.Context, mainStore *store.Store, mainPath string, readOnly bool, openers ...visual.StreamOpener) (*vectorFeatures, error) { + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil { + return nil, errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := loggerFromContext(ctx) if !cfg.Vector.AnyLaneEnabled() { return nil, nil //nolint:nilnil // vector disabled: callers nil-check vf; (nil, nil) means "no features, no error" } @@ -543,11 +594,14 @@ func setupVectorFeatures(ctx context.Context, mainStore *store.Store, mainPath s vecPath = filepath.Join(cfg.Data.DataDir, "vectors.db") } sb, err := sqlitevec.Open(ctx, sqlitevec.Options{ - Path: vecPath, - MainPath: mainPath, - Dimension: vecCfg.Embeddings.Dimension, - MainDB: mainDB, - BuildScope: vecCfg.Embed.Scope.BuildScope(), + Path: vecPath, + MainPath: mainPath, + Dimension: vecCfg.Embeddings.Dimension, + MainDB: mainDB, + BuildScope: vecCfg.Embed.Scope.BuildScope(), + ANNOversample: vecCfg.Search.ANNOversample, + ANNNProbe: vecCfg.Search.ANNNProbe, + AcceleratorMode: vecCfg.Search.SQLiteAccelerator, // Honor the read-only signal on SQLite too: when mainDB is a // query-only handle (MCP), skip the embed_gen upgrade backfill, // which would write through it. Migrate still runs (vectors.db @@ -568,7 +622,7 @@ func setupVectorFeatures(ctx context.Context, mainStore *store.Store, mainPath s } if vecCfg.Enabled { personGate := vector.NewPinnedExactSemanticPersonEmbeddingGate( - vecCfg, currentSemanticPersonVectorConfigSource(), mainStore, + vecCfg, currentSemanticPersonVectorConfigSource(state), mainStore, ) runtime, err := newEmbeddingRuntime(vecCfg, embeddingRuntimeDeps{ Backend: backend, VectorsDB: vectorsDB, MainDB: mainDB, Store: mainStore, diff --git a/cmd/msgvault/cmd/serve_vector_init.go b/cmd/msgvault/cmd/serve_vector_init.go index da616bd7f..c8bfd43bc 100644 --- a/cmd/msgvault/cmd/serve_vector_init.go +++ b/cmd/msgvault/cmd/serve_vector_init.go @@ -4,10 +4,12 @@ import ( "context" "errors" "fmt" + "log/slog" "sync" "time" "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/operations" "go.kenn.io/msgvault/internal/scheduler" "go.kenn.io/msgvault/internal/store" @@ -69,14 +71,14 @@ func (h *vectorInitHandle) WaitTimeout(d time.Duration) bool { // CloseFeatures closes the vector backend if the init goroutine opened one. // Only call after WaitTimeout reports the goroutine finished. -func (h *vectorInitHandle) CloseFeatures() { +func (h *vectorInitHandle) CloseFeatures(logger *slog.Logger) { h.mu.Lock() vf := h.vf h.vf = nil h.mu.Unlock() if vf != nil && vf.Close != nil { if err := vf.Close(); err != nil { - logger.Warn("closing vectors.db failed", "error", err) + repairLogger(logger).Warn("closing vectors.db failed", "error", err) } } } @@ -99,6 +101,13 @@ func startVectorInit( openers ...visual.StreamOpener, ) *vectorInitHandle { h := &vectorInitHandle{done: make(chan struct{})} + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil || state.logger == nil { + close(h.done) + return h + } + cfg := state.cfg + logger := state.logger if !cfg.Vector.AnyLaneEnabled() { close(h.done) return h @@ -148,9 +157,9 @@ func startVectorInit( // even on daemons whose embed job never runs (empty cron, // run_after_sync=false). The embed job's own per-run check remains // the detection path for scheduled embeds. - apiServer.SetVectorScopeCheck(embedScopeDriftCheck(s, vf.Cfg.Embed.Scope.BuildScope())) - checkVectorIndexFreshness(ctx, apiServer, vf) - if err := registerEmbedJob(sched, vf, s, apiServer); err != nil { + apiServer.SetVectorScopeCheck(embedScopeDriftCheck(s, vf.Cfg.Embed.Scope.BuildScope(), state)) + checkVectorIndexFreshness(ctx, apiServer, vf, logger) + if err := registerEmbedJob(sched, vf, s, apiServer, cfg, logger); err != nil { // Cron was validated in precheckVectorFeatures, so this is an // invariant violation, not user error; vector search still works. logger.Error("register embed job failed", "error", err) @@ -222,11 +231,11 @@ func startVectorInit( } return vf.Visual.Backend.DeleteTokens(retireCtx, visualTokens) }) - if err := registerVisualJob(sched, vf.Visual); err != nil { + if err := registerVisualJob(sched, vf.Visual, cfg); err != nil { logger.Error("register multimodal job failed", "error", err) } } - if err := registerDocumentVectorJob(sched, vf, s); err != nil { + if err := registerDocumentVectorJob(sched, vf, s, state); err != nil { logger.Error("register document vector job failed", "error", err) } logger.Info("daemon startup step complete", "step", "init_vector_backend") @@ -430,7 +439,10 @@ func cleanupObsoleteVisualVectors(ctx context.Context, vf *visualFeatures) error } } -func registerVisualJob(sched *scheduler.Scheduler, vf *visualFeatures) error { +func registerVisualJob(sched *scheduler.Scheduler, vf *visualFeatures, cfg *config.Config) error { + if cfg == nil { + return errors.New("configuration is unavailable") + } runScheduled := func(ctx context.Context) error { generation, err := vf.Archive.GetVisualGeneration(ctx, vf.Generation.ID) if err != nil || @@ -466,13 +478,13 @@ func registerVisualJob(sched *scheduler.Scheduler, vf *visualFeatures) error { // backend errors leave the freshly-installed "ready" status untouched, since // those are not the "index does not match the configured embedding settings" failure this // status exists to expose. -func checkVectorIndexFreshness(ctx context.Context, apiServer *api.Server, vf *vectorFeatures) { +func checkVectorIndexFreshness(ctx context.Context, apiServer *api.Server, vf *vectorFeatures, logger *slog.Logger) { _, err := resolveActiveGeneration(ctx, vf.Backend, vf.Cfg.GenerationFingerprint()) if !errors.Is(err, vector.ErrIndexStale) { return } detail := err.Error() + "; if this is a one-off account-scoped generation, set matching [vector.embed.scope] accounts and restart the daemon; otherwise run `msgvault embeddings build --full-rebuild` to rebuild" - logger.Warn("vector index does not match configured embedding settings; vector search unavailable", + repairLogger(logger).Warn("vector index does not match configured embedding settings; vector search unavailable", "detail", detail) apiServer.SetVectorStale(detail) } @@ -491,9 +503,9 @@ func embedScopeDriftDetail(resolved, initialized vector.BuildScope) string { // deterministically unresolvable (a configured account removed or // ambiguous). Transient resolution failures (a busy database) pass through // as errors so the preflight logs and retries them instead of latching. -func embedScopeDriftCheck(s *store.Store, initialized vector.BuildScope) func(context.Context) (string, error) { - return func(context.Context) (string, error) { - resolved, err := configuredEmbedBuildScope(s) +func embedScopeDriftCheck(s *store.Store, initialized vector.BuildScope, state *invocation) func(context.Context) (string, error) { + return func(ctx context.Context) (string, error) { + resolved, err := configuredEmbedBuildScope(s, state) if errors.Is(err, vector.ErrScopeUnresolvable) { return err.Error() + "; fix [vector.embed.scope] accounts and restart the daemon", nil } @@ -518,8 +530,12 @@ type documentVectorJobRegistrar interface { SetDocumentVectorJob(job func(context.Context) error, schedule string, runAfterSync bool) error } -func registerDocumentVectorJob(sched documentVectorJobRegistrar, vf *vectorFeatures, st *store.Store) error { - if cfg == nil || !cfg.Attachments.Documents.Index.Embeddings.Enabled || vf == nil || vf.DocumentBackend == nil { +func registerDocumentVectorJob(sched documentVectorJobRegistrar, vf *vectorFeatures, st *store.Store, state *invocation) error { + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + if !cfg.Attachments.Documents.Index.Embeddings.Enabled || vf == nil || vf.DocumentBackend == nil { return nil } limit := vf.Cfg.Embeddings.BatchSize @@ -529,21 +545,25 @@ func registerDocumentVectorJob(sched documentVectorJobRegistrar, vf *vectorFeatu if limit > 1000 { limit = 1000 } - job := func(ctx context.Context) error { + job := invocationBoundJobRun(state, func(ctx context.Context) error { return runScheduledDocumentVectorGeneration(ctx, st, vf, limit) - } + }) if err := sched.SetDocumentVectorJob(job, cfg.Vector.Embed.Schedule.Cron, cfg.Vector.Embed.Schedule.RunAfterSync); err != nil { return fmt.Errorf("register document vector job: %w", err) } - logger.Info("document vectors scheduled", "cron", cfg.Vector.Embed.Schedule.Cron, + repairLogger(state.logger).Info("document vectors scheduled", "cron", cfg.Vector.Embed.Schedule.Cron, "run_after_sync", cfg.Vector.Embed.Schedule.RunAfterSync) return nil } -func registerEmbedJob(sched embedJobRegistrar, vf *vectorFeatures, s *store.Store, apiServer *api.Server) error { - embedJob := newSchedulerEmbedJob(vf, s) +func registerEmbedJob(sched embedJobRegistrar, vf *vectorFeatures, s *store.Store, apiServer *api.Server, cfg *config.Config, logger *slog.Logger) error { + if cfg == nil { + return errors.New("configuration is unavailable") + } + embedJob := newSchedulerEmbedJob(vf, s, logger) + scopeState := &invocation{cfg: cfg} embedJob.ResolveBuildScope = func() (vector.BuildScope, error) { - return configuredEmbedBuildScope(s) + return configuredEmbedBuildScope(s, scopeState) } // Scope drift also has to reach searchers, not just the log: the // installed components still match the active generation's @@ -557,14 +577,14 @@ func registerEmbedJob(sched embedJobRegistrar, vf *vectorFeatures, s *store.Stor if err := sched.SetEmbedJob(embedJob, schedule, cfg.Vector.Embed.Schedule.RunAfterSync); err != nil { return fmt.Errorf("register embed job: %w", err) } - logger.Info("embed scheduled", + repairLogger(logger).Info("embed scheduled", "cron", schedule, "run_after_sync", cfg.Vector.Embed.Schedule.RunAfterSync, ) return nil } -func newSchedulerEmbedJob(vf *vectorFeatures, s *store.Store) *scheduler.EmbedJob { +func newSchedulerEmbedJob(vf *vectorFeatures, s *store.Store, logger *slog.Logger) *scheduler.EmbedJob { return &scheduler.EmbedJob{ Worker: vf.Runner, Backend: vf.Backend, @@ -575,6 +595,6 @@ func newSchedulerEmbedJob(vf *vectorFeatures, s *store.Store) *scheduler.EmbedJo Fingerprint: vf.Cfg.GenerationFingerprint(), BackstopInterval: vf.Cfg.Embed.BackstopInterval, BuildScope: vf.Cfg.Embed.Scope.BuildScope(), - Log: logger, + Log: repairLogger(logger), } } diff --git a/cmd/msgvault/cmd/serve_vector_init_test.go b/cmd/msgvault/cmd/serve_vector_init_test.go index 67842fb82..4abf8987f 100644 --- a/cmd/msgvault/cmd/serve_vector_init_test.go +++ b/cmd/msgvault/cmd/serve_vector_init_test.go @@ -10,7 +10,9 @@ import ( "net/http" "net/http/httptest" "strings" + "sync" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -107,21 +109,6 @@ func overrideSetupVectorFeatures(t *testing.T, fn func(context.Context, *store.S t.Cleanup(func() { setupVectorFeaturesForRun = prev }) } -func waitForVectorStatus(t *testing.T, srv *api.Server, want api.VectorStatus) string { - t.Helper() - deadline := time.Now().Add(5 * time.Second) - for time.Now().Before(deadline) { - status, msg := srv.VectorStatus() - if status == want { - return msg - } - time.Sleep(5 * time.Millisecond) - } - status, _ := srv.VectorStatus() - require.Equal(t, want, status, "vector status never reached %s", want) - return "" -} - func TestVectorInitHandleWaitContextReturnsTrueWhenFinished(t *testing.T) { h := &vectorInitHandle{done: make(chan struct{})} close(h.done) @@ -156,9 +143,10 @@ func TestVectorInitHandleWaitContextPrefersDoneWhenBothReady(t *testing.T) { func TestStartVectorInitDisabledFinishesImmediately(t *testing.T) { c := config.NewDefaultConfig() c.Vector.Enabled = false - withTestConfig(t, c) + testCtx := withTestConfig(t, c) + _ = testCtx - h := startVectorInit(context.Background(), nil, "", nil, nil, nil) + h := startVectorInit(testCtx, nil, "", nil, nil, nil) assert.True(t, h.WaitTimeout(time.Second)) } @@ -166,7 +154,8 @@ func TestStartVectorInitRunsForIndependentMultimodalLane(t *testing.T) { c := config.NewDefaultConfig() c.Vector.Enabled = false c.Vector.Multimodal.Enabled = true - withTestConfig(t, c) + testCtx := withTestConfig(t, c) + _ = testCtx called := false prev := setupVectorFeaturesForRun @@ -176,7 +165,7 @@ func TestStartVectorInitRunsForIndependentMultimodalLane(t *testing.T) { } t.Cleanup(func() { setupVectorFeaturesForRun = prev }) - h := startVectorInit(context.Background(), nil, "/tmp/msgvault.db", nil, + h := startVectorInit(testCtx, nil, "/tmp/msgvault.db", nil, newVectorInitTestServer(t), scheduler.New(nil)) require.True(t, h.WaitTimeout(5*time.Second)) assert.True(t, called, "multimodal-only enablement must initialize vector infrastructure") @@ -187,7 +176,8 @@ func TestStartVectorInitInstallsFeaturesOnSuccess(t *testing.T) { requirements := require.New(t) c := config.NewDefaultConfig() c.Vector.Enabled = true - withTestConfig(t, c) + testCtx := withTestConfig(t, c) + _ = testCtx closed := false backend := &vectorInitPersonBackend{fakeCmdVectorBackend: &fakeCmdVectorBackend{ @@ -206,10 +196,11 @@ func TestStartVectorInitInstallsFeaturesOnSuccess(t *testing.T) { srv := newVectorInitTestServer(t) sched := scheduler.New(nil) - h := startVectorInit(context.Background(), nil, "/tmp/msgvault.db", nil, srv, sched) + h := startVectorInit(testCtx, nil, "/tmp/msgvault.db", nil, srv, sched) requirements.True(h.WaitTimeout(5 * time.Second)) - waitForVectorStatus(t, srv, api.VectorStatusReady) + status, _ := srv.VectorStatus() + requirements.Equal(api.VectorStatusReady, status) request := httptest.NewRequest(http.MethodPost, "/api/v1/people/search", strings.NewReader(`{"query":"synthetic"}`)) request.Header.Set("Content-Type", "application/json") @@ -220,7 +211,7 @@ func TestStartVectorInitInstallsFeaturesOnSuccess(t *testing.T) { "ready status must publish the person engine in the same installation") assertions.NotContains(response.Body.String(), "vector_not_enabled", "ready status must never precede person engine installation") - h.CloseFeatures() + h.CloseFeatures(testDiscardLogger()) assertions.True(closed, "CloseFeatures must close the opened backend") } @@ -303,16 +294,18 @@ func TestRegisterDocumentVectorJobRequiresDocumentEmbeddingsAndBackend(t *testin c := config.NewDefaultConfig() c.Vector.Embed.Schedule.Cron = "*/7 * * * *" c.Vector.Embed.Schedule.RunAfterSync = true - withTestConfig(t, c) + testCtx := withTestConfig(t, c) + state := invocationFromContext(testCtx) + state.logger = testDiscardLogger() available := &vectorFeatures{DocumentBackend: startupDocumentBackend{}, Cfg: c.Vector} disabled := ®isteredDocumentVectorJobCapture{} - requirements.NoError(registerDocumentVectorJob(disabled, available, nil)) + requirements.NoError(registerDocumentVectorJob(disabled, available, nil, state)) assertions.Zero(disabled.calls) c.Attachments.Documents.Index.Embeddings.Enabled = true capture := ®isteredDocumentVectorJobCapture{} - requirements.NoError(registerDocumentVectorJob(capture, available, nil)) + requirements.NoError(registerDocumentVectorJob(capture, available, nil, state)) assertions.Equal(1, capture.calls) assertions.NotNil(capture.job) assertions.Equal("*/7 * * * *", capture.schedule) @@ -324,7 +317,9 @@ func TestRegisterDocumentVectorJobRequiresDocumentEmbeddingsAndBackend(t *testin } { t.Run(name, func(t *testing.T) { unregistered := ®isteredDocumentVectorJobCapture{} - require.NoError(t, registerDocumentVectorJob(unregistered, features, nil)) + testState := invocationFromContext(withTestConfig(t, c)) + testState.logger = testDiscardLogger() + require.NoError(t, registerDocumentVectorJob(unregistered, features, nil, testState)) assert.Zero(t, unregistered.calls) }) } @@ -345,7 +340,7 @@ func TestStartVectorInitInstallsOnlyConsentedDocumentSearch(t *testing.T) { t.Run(test.name, func(t *testing.T) { c := config.NewDefaultConfig() c.Vector.Enabled = true - withTestConfig(t, c) + testCtx := withTestConfig(t, c) overrideSetupVectorFeatures(t, func(context.Context, *store.Store, string, bool) (*vectorFeatures, error) { return &vectorFeatures{ Backend: &fakeCmdVectorBackend{}, DocumentSearch: test.service, @@ -358,7 +353,7 @@ func TestStartVectorInitInstallsOnlyConsentedDocumentSearch(t *testing.T) { Config: c, Store: &storeAPIAdapter{store: mainStore}, Logger: slog.New(slog.DiscardHandler), VectorStatus: api.VectorStatusInitializing, }) - h := startVectorInit(t.Context(), mainStore, "/tmp/msgvault.db", nil, srv, scheduler.New(nil)) + h := startVectorInit(testCtx, mainStore, "/tmp/msgvault.db", nil, srv, scheduler.New(nil)) require.True(t, h.WaitTimeout(5*time.Second)) request := httptest.NewRequest(http.MethodGet, "/api/v1/documents/search?q=bounded&mode=semantic&candidate_limit=10", nil) @@ -366,7 +361,7 @@ func TestStartVectorInitInstallsOnlyConsentedDocumentSearch(t *testing.T) { srv.Router().ServeHTTP(response, request) assert.Equal(t, test.wantStatus, response.Code, response.Body.String()) assert.Contains(t, response.Body.String(), test.wantPayload) - h.CloseFeatures() + h.CloseFeatures(testDiscardLogger()) }) } } @@ -374,7 +369,8 @@ func TestStartVectorInitInstallsOnlyConsentedDocumentSearch(t *testing.T) { func TestStartVectorInitFlagsStaleIndex(t *testing.T) { c := config.NewDefaultConfig() c.Vector.Enabled = true - withTestConfig(t, c) + testCtx := withTestConfig(t, c) + _ = testCtx // Active generation's fingerprint differs from the configured one, so // the same check the query path runs (ResolveActiveForFingerprint) @@ -390,10 +386,11 @@ func TestStartVectorInitFlagsStaleIndex(t *testing.T) { }) srv := newVectorInitTestServer(t) - h := startVectorInit(context.Background(), nil, "/tmp/msgvault.db", nil, srv, scheduler.New(nil)) + h := startVectorInit(testCtx, nil, "/tmp/msgvault.db", nil, srv, scheduler.New(nil)) require.True(t, h.WaitTimeout(5*time.Second)) - detail := waitForVectorStatus(t, srv, api.VectorStatusStale) + status, detail := srv.VectorStatus() + require.Equal(t, api.VectorStatusStale, status) assert := assert.New(t) assert.Contains(detail, "old-model:384:c6000:e1", "detail names the stored fingerprint") assert.Contains(detail, c.Vector.GenerationFingerprint(), "detail names the configured fingerprint") @@ -403,60 +400,80 @@ func TestStartVectorInitFlagsStaleIndex(t *testing.T) { func TestStartVectorInitReportsError(t *testing.T) { c := config.NewDefaultConfig() c.Vector.Enabled = true - withTestConfig(t, c) + testCtx := withTestConfig(t, c) + _ = testCtx overrideSetupVectorFeatures(t, func(context.Context, *store.Store, string, bool) (*vectorFeatures, error) { return nil, errors.New("migration exploded") }) srv := newVectorInitTestServer(t) - h := startVectorInit(context.Background(), nil, "/tmp/msgvault.db", nil, srv, scheduler.New(nil)) + h := startVectorInit(testCtx, nil, "/tmp/msgvault.db", nil, srv, scheduler.New(nil)) require.True(t, h.WaitTimeout(5*time.Second)) - msg := waitForVectorStatus(t, srv, api.VectorStatusError) + status, msg := srv.VectorStatus() + require.Equal(t, api.VectorStatusError, status) assert.Contains(t, msg, "migration exploded") } func TestStartVectorInitHoldsWorkTracker(t *testing.T) { - c := config.NewDefaultConfig() - c.Vector.Enabled = true - withTestConfig(t, c) - - gate := api.NewSerialOperationGate() - release := make(chan struct{}) - overrideSetupVectorFeatures(t, func(ctx context.Context, _ *store.Store, _ string, _ bool) (*vectorFeatures, error) { - <-release - return nil, ctx.Err() - }) + synctest.Test(t, func(t *testing.T) { + require := require.New(t) + c := config.NewDefaultConfig() + c.Vector.Enabled = true + withTestConfig(t, c) + + gate := api.NewSerialOperationGate() + release := make(chan struct{}) + var releaseOnce sync.Once + releaseWork := func() { releaseOnce.Do(func() { close(release) }) } + overrideSetupVectorFeatures(t, func(ctx context.Context, _ *store.Store, _ string, _ bool) (*vectorFeatures, error) { + <-release + return nil, ctx.Err() + }) - srv := newVectorInitTestServer(t) - h := startVectorInit(context.Background(), nil, "/tmp/msgvault.db", gate, srv, scheduler.New(nil)) + srv := newVectorInitTestServer(t) + testCtx := withTestConfig(t, c) + h := startVectorInit(testCtx, nil, "/tmp/msgvault.db", gate, srv, scheduler.New(nil)) + t.Cleanup(func() { + releaseWork() + require.True(h.WaitTimeout(5 * time.Second)) + require.NoError(srv.Shutdown(context.Background()), "shutdown") + synctest.Wait() + }) - // While init runs, the gate must be held: BeginWorkContext with an - // already-cancelled context must fail rather than acquire. - assert.Eventually(t, func() bool { - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + synctest.Wait() + _, _, held := gate.Holder() + require.True(held, "gate should be held during init") + probeCtx, cancel := context.WithTimeout(context.Background(), 5*time.Millisecond) defer cancel() - done, ok := gate.BeginWorkContext(ctx) - if ok { - done() - } - return !ok - }, 2*time.Second, 10*time.Millisecond, "gate should be held during init") - - close(release) - require.True(t, h.WaitTimeout(5*time.Second)) - done, ok := gate.BeginWork() - require.True(t, ok, "gate must be released after init") - done() + probeDone := make(chan bool, 1) + go func() { + done, ok := gate.BeginWorkContext(probeCtx) + if ok { + done() + } + probeDone <- ok + }() + synctest.Sleep(6 * time.Millisecond) + synctest.Wait() + assert.False(t, <-probeDone, "live gate probe must time out while init holds the gate") + + releaseWork() + require.True(h.WaitTimeout(5 * time.Second)) + done, ok := gate.BeginWork() + require.True(ok, "gate must be released after init") + done() + }) } func TestStartVectorInitAbortsQuietlyOnCancel(t *testing.T) { c := config.NewDefaultConfig() c.Vector.Enabled = true - withTestConfig(t, c) + testCtx := withTestConfig(t, c) + _ = testCtx - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(testCtx) overrideSetupVectorFeatures(t, func(ctx context.Context, _ *store.Store, _ string, _ bool) (*vectorFeatures, error) { <-ctx.Done() return nil, ctx.Err() @@ -480,7 +497,7 @@ func TestNewSchedulerEmbedJobThreadsGenerationRunnerAndConvergenceChecker(t *tes Cfg: config.NewDefaultConfig().Vector, } - job := newSchedulerEmbedJob(vf, nil) + job := newSchedulerEmbedJob(vf, nil, testDiscardLogger()) assert.Same(t, runner, job.Worker) assert.Same(t, checker, job.Convergence) } @@ -577,7 +594,8 @@ func runRegisteredContextJob( vectorCfg := contextualSchedulerConfig() testCfg := config.NewDefaultConfig() testCfg.Vector = vectorCfg - withTestConfig(t, testCfg) + testCtx := withTestConfig(t, testCfg) + _ = testCtx if buildingFingerprint == "" { buildingFingerprint = vectorCfg.GenerationFingerprint() } @@ -591,9 +609,9 @@ func runRegisteredContextJob( Backend: backend, Runner: runner, Convergence: registeredConvergenceChecker{state: state}, Cfg: vectorCfg, } - require.NoError(t, registerEmbedJob(capture, vf, nil, nil)) + require.NoError(t, registerEmbedJob(capture, vf, nil, nil, testCfg, testDiscardLogger())) require.NotNil(t, capture.job) - capture.job.Run(t.Context()) + capture.job.Run(testCtx) return backend, runner } @@ -809,9 +827,10 @@ func TestVisualHTTPProductionRegistrationRecordsBuildResumeAndRetryPasses(t *tes c.Vector.Multimodal.Enabled = true c.Vector.Multimodal.Schedule.Cron = "" c.Vector.Multimodal.Schedule.RunAfterSync = false - withTestConfig(t, c) + testCtx := withTestConfig(t, c) + _ = testCtx st := testutil.NewSQLiteTestStore(t) - generation, err := st.EnsureVisualGeneration(t.Context(), store.VisualGenerationSpec{ + generation, err := st.EnsureVisualGeneration(testCtx, store.VisualGenerationSpec{ Fingerprint: "visual-http-production-registration", Model: "visual-test", Dimension: 1024, }) require.NoError(err) @@ -834,9 +853,9 @@ func TestVisualHTTPProductionRegistrationRecordsBuildResumeAndRetryPasses(t *tes Config: c, Store: &storeAPIAdapter{store: st}, Logger: slog.New(slog.DiscardHandler), VectorStatus: api.VectorStatusInitializing, }) - handle := startVectorInit(t.Context(), st, "/tmp/msgvault.db", nil, srv, scheduler.New(nil)) + handle := startVectorInit(testCtx, st, "/tmp/msgvault.db", nil, srv, scheduler.New(nil)) require.True(handle.WaitTimeout(5 * time.Second)) - t.Cleanup(handle.CloseFeatures) + t.Cleanup(func() { handle.CloseFeatures(testDiscardLogger()) }) router := srv.Router() privateRequestID := "private-http-operation-request-owner" privateBlobHash := strings.Repeat("ab", 32) @@ -889,9 +908,10 @@ func TestRegisterVisualJobSkipsUnconsentedScheduledPassWithoutRow(t *testing.T) c := config.NewDefaultConfig() c.Vector.Multimodal.Schedule.Cron = "0 0 1 1 *" c.Vector.Multimodal.Schedule.RunAfterSync = false - withTestConfig(t, c) + testCtx := withTestConfig(t, c) + _ = testCtx st := testutil.NewSQLiteTestStore(t) - generation, err := st.EnsureVisualGeneration(t.Context(), store.VisualGenerationSpec{ + generation, err := st.EnsureVisualGeneration(testCtx, store.VisualGenerationSpec{ Fingerprint: "visual-scheduled-unconsented", Model: "visual-test", Dimension: 1024, }) require.NoError(err) @@ -900,12 +920,12 @@ func TestRegisterVisualJobSkipsUnconsentedScheduledPassWithoutRow(t *testing.T) Reconciler: newVectorInitVisualReconciler(t, st, generation.ID, "visual-test/scheduled-unconsented"), } sched := scheduler.New(nil) - require.NoError(registerVisualJob(sched, vf)) + require.NoError(registerVisualJob(sched, vf, c)) require.NoError(sched.TriggerJob("multimodal-attachments")) assert.Empty(operationRunsForKind(t, st, operations.KindVisualEmbedding), "the outer consent gate skips before runVisualPass owns a row") - _, err = st.GetAttachmentChangeConsumer(t.Context(), "visual-test/scheduled-unconsented") + _, err = st.GetAttachmentChangeConsumer(testCtx, "visual-test/scheduled-unconsented") assert.ErrorIs(err, store.ErrAttachmentChangeConsumerMissing, "the skipped callback must not enter reconciliation") } @@ -957,7 +977,7 @@ func TestRegisterVisualJobGenerationStateGateSkipsBeforePass(t *testing.T) { Reconciler: newVectorInitVisualReconciler(t, st, generationID, consumerKey), } sched := scheduler.New(nil) - require.NoError(registerVisualJob(sched, vf)) + require.NoError(registerVisualJob(sched, vf, c)) triggerErr := sched.TriggerJob("multimodal-attachments") if test.wantTriggerErr { @@ -984,29 +1004,30 @@ func TestRegisterVisualJobRecordsLaterPostActivationMaintenancePass(t *testing.T c := config.NewDefaultConfig() c.Vector.Multimodal.Schedule.Cron = "0 0 1 1 *" c.Vector.Multimodal.Schedule.RunAfterSync = false - withTestConfig(t, c) + testCtx := withTestConfig(t, c) + _ = testCtx st := testutil.NewSQLiteTestStore(t) - generation, err := st.EnsureVisualGeneration(t.Context(), store.VisualGenerationSpec{ + generation, err := st.EnsureVisualGeneration(testCtx, store.VisualGenerationSpec{ Fingerprint: "visual-scheduled-post-activation", Model: "visual-test", Dimension: 1024, }) require.NoError(err) - require.NoError(st.ConsentVisualGeneration(t.Context(), generation.ID, "private-policy-fingerprint")) + require.NoError(st.ConsentVisualGeneration(testCtx, generation.ID, "private-policy-fingerprint")) vf := &visualFeatures{ Archive: st, Generation: generation, PolicyFingerprint: "private-policy-fingerprint", Reconciler: newVectorInitVisualReconciler(t, st, generation.ID, "visual-test/post-activation"), } sched := scheduler.New(nil) - require.NoError(registerVisualJob(sched, vf)) + require.NoError(registerVisualJob(sched, vf, c)) require.NoError(sched.TriggerJob("multimodal-attachments")) - activated, err := st.GetVisualGeneration(t.Context(), generation.ID) + activated, err := st.GetVisualGeneration(testCtx, generation.ID) require.NoError(err) require.Equal(store.VisualGenerationActive, activated.State) firstRuns := operationRunsForKind(t, st, operations.KindVisualEmbedding) require.Len(firstRuns, 1) require.NoError(sched.TriggerJob("multimodal-attachments")) - afterMaintenance, err := st.GetVisualGeneration(t.Context(), generation.ID) + afterMaintenance, err := st.GetVisualGeneration(testCtx, generation.ID) require.NoError(err) assert.Equal(store.VisualGenerationActive, afterMaintenance.State, "post-activation maintenance must preserve the active generation") diff --git a/cmd/msgvault/cmd/serve_vector_integration_test.go b/cmd/msgvault/cmd/serve_vector_integration_test.go index 722bfc833..762a73201 100644 --- a/cmd/msgvault/cmd/serve_vector_integration_test.go +++ b/cmd/msgvault/cmd/serve_vector_integration_test.go @@ -23,6 +23,8 @@ import ( // seam is overridden to block until daemon shutdown, so a passing test // proves the API listener comes up independently of vector maintenance. func TestRunServeServesHealthWhileVectorInitBlocked(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) oldCfg := cfg @@ -49,7 +51,7 @@ func TestRunServeServesHealthWhileVectorInitBlocked(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() cmd := &cobra.Command{Use: "serve"} - cmd.SetContext(ctx) + cmd.SetContext(testInvocationContext(ctx, cfg, invocationOptions{})) errCh := make(chan error, 1) go func() { errCh <- runServe(cmd, nil) @@ -92,6 +94,8 @@ func TestRunServeServesHealthWhileVectorInitBlocked(t *testing.T) { } func TestRunServeStartsVectorWhileAnalyticsInitializationBlocked(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) oldCfg := cfg dataDir := t.TempDir() @@ -124,7 +128,7 @@ func TestRunServeStartsVectorWhileAnalyticsInitializationBlocked(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() cmd := &cobra.Command{Use: "serve"} - cmd.SetContext(ctx) + cmd.SetContext(testInvocationContext(ctx, cfg, invocationOptions{})) errCh := make(chan error, 1) go func() { errCh <- runServe(cmd, nil) }() diff --git a/cmd/msgvault/cmd/serve_vector_nopg_test.go b/cmd/msgvault/cmd/serve_vector_nopg_test.go index 18ee5877c..d0d20f4e9 100644 --- a/cmd/msgvault/cmd/serve_vector_nopg_test.go +++ b/cmd/msgvault/cmd/serve_vector_nopg_test.go @@ -3,7 +3,6 @@ package cmd import ( - "context" "testing" "github.com/stretchr/testify/assert" @@ -20,6 +19,8 @@ import ( // gained real PG vector support; this pins that no remaining code path // emits it under this tag combo. func TestSetupVectorFeatures_PostgresWithoutPgvectorTag(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) @@ -27,6 +28,8 @@ func TestSetupVectorFeatures_PostgresWithoutPgvectorTag(t *testing.T) { defer func() { cfg = savedCfg }() cfg = &config.Config{} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Vector.Enabled = true cfg.Vector.Backend = "sqlite-vec" cfg.Vector.Embeddings.Endpoint = "http://localhost:11434/v1/embeddings" @@ -43,7 +46,7 @@ func TestSetupVectorFeatures_PostgresWithoutPgvectorTag(t *testing.T) { t.Cleanup(func() { _ = st.Close() }) - _, err = setupVectorFeatures(context.Background(), st, "postgres://user@host/db", false) + _, err = setupVectorFeatures(testCtx, st, "postgres://user@host/db", false) require.Error(err, "setupVectorFeatures with postgres DSN and no pgvector tag") assert. // Must come from the stub, not the removed up-front refusal. Contains(err.Error(), "pgvector support not compiled in", @@ -69,9 +72,10 @@ func TestPrecheckVectorFeatures_PostgresWithoutPgvectorTag(t *testing.T) { c.Vector.Embeddings.Endpoint = "http://localhost:11434/v1/embeddings" c.Vector.Embeddings.Model = "test-model" c.Vector.Embeddings.Dimension = 768 - withTestConfig(t, c) + testCtx := withTestConfig(t, c) + _ = testCtx - err := precheckVectorFeatures("postgres://user@host/db") + err := precheckVectorFeatures("postgres://user@host/db", c) require.Error(err, "precheck must fail fast for postgres mainPath without pgvector tag") assert.Contains(err.Error(), "pgvector", "error should point at the missing pgvector build tag") diff --git a/cmd/msgvault/cmd/serve_vector_nosqlitevec_test.go b/cmd/msgvault/cmd/serve_vector_nosqlitevec_test.go index 6d972e20b..df23832e9 100644 --- a/cmd/msgvault/cmd/serve_vector_nosqlitevec_test.go +++ b/cmd/msgvault/cmd/serve_vector_nosqlitevec_test.go @@ -28,9 +28,10 @@ func TestPrecheckVectorFeatures_SQLiteWithoutSqliteVecTag(t *testing.T) { c.Vector.Embeddings.Endpoint = "http://localhost:11434/v1/embeddings" c.Vector.Embeddings.Model = "test-model" c.Vector.Embeddings.Dimension = 768 - withTestConfig(t, c) + testCtx := withTestConfig(t, c) + _ = testCtx - err := precheckVectorFeatures("msgvault.db") + err := precheckVectorFeatures("msgvault.db", c) require.Error(err, "precheck must fail fast for sqlite mainPath without sqlite_vec tag") assert.Contains(err.Error(), "sqlite-vec", "error should point at the missing sqlite_vec build tag") diff --git a/cmd/msgvault/cmd/serve_vector_pg_test.go b/cmd/msgvault/cmd/serve_vector_pg_test.go index 27c535c7c..f3a22e96a 100644 --- a/cmd/msgvault/cmd/serve_vector_pg_test.go +++ b/cmd/msgvault/cmd/serve_vector_pg_test.go @@ -3,7 +3,6 @@ package cmd import ( - "context" "crypto/rand" "database/sql" "encoding/hex" @@ -67,6 +66,8 @@ func openServePGSchema(t *testing.T) (*sql.DB, string) { // must succeed against a postgres:// DSN and wire up the backend, hybrid // engine, and worker. Runs only with a live PG (MSGVAULT_TEST_DB). func TestSetupVectorFeatures_SucceedsOnPostgres(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) @@ -88,6 +89,8 @@ func TestSetupVectorFeatures_SucceedsOnPostgres(t *testing.T) { db := st.DB() cfg = &config.Config{} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Vector.Enabled = true cfg.Vector.Backend = "sqlite-vec" // Validate's backend gate; PG is selected from the DSN cfg.Vector.Embeddings.Endpoint = "http://localhost:11434/v1/embeddings" @@ -95,7 +98,7 @@ func TestSetupVectorFeatures_SucceedsOnPostgres(t *testing.T) { cfg.Vector.Embeddings.Dimension = 768 cfg.Vector.Embeddings.BatchSize = 32 - vf, err := setupVectorFeatures(context.Background(), st, dsn, false) + vf, err := setupVectorFeatures(testCtx, st, dsn, false) require.NoError( err, "setupVectorFeatures on postgres DSN must succeed with pgvector built in") diff --git a/cmd/msgvault/cmd/serve_vector_precheck_test.go b/cmd/msgvault/cmd/serve_vector_precheck_test.go index c5a0c6736..24b8031f6 100644 --- a/cmd/msgvault/cmd/serve_vector_precheck_test.go +++ b/cmd/msgvault/cmd/serve_vector_precheck_test.go @@ -16,9 +16,10 @@ import ( func TestPrecheckVectorFeaturesDisabled(t *testing.T) { c := config.NewDefaultConfig() c.Vector.Enabled = false - withTestConfig(t, c) + testCtx := withTestConfig(t, c) + _ = testCtx - assert.NoError(t, precheckVectorFeatures(precheckTestMainPath)) + assert.NoError(t, precheckVectorFeatures(precheckTestMainPath, c)) } // TestPrecheckVectorFeaturesRejectsBadCron verifies the precheck validates @@ -31,9 +32,10 @@ func TestPrecheckVectorFeaturesRejectsBadCron(t *testing.T) { c.Vector.Embeddings.Model = "test-model" c.Vector.Embeddings.Dimension = 768 c.Vector.Embed.Schedule.Cron = "not a cron" - withTestConfig(t, c) + testCtx := withTestConfig(t, c) + _ = testCtx - err := precheckVectorFeatures(precheckTestMainPath) + err := precheckVectorFeatures(precheckTestMainPath, c) require.Error(t, err) assert.Contains(t, err.Error(), "cron") } @@ -46,9 +48,10 @@ func TestPrecheckVectorFeaturesRejectsInvalidConfig(t *testing.T) { c.Vector.Enabled = true // Leave required embeddings fields (endpoint, model, dimension) empty // so Validate() fails. - withTestConfig(t, c) + testCtx := withTestConfig(t, c) + _ = testCtx - assert.Error(t, precheckVectorFeatures(precheckTestMainPath)) + assert.Error(t, precheckVectorFeatures(precheckTestMainPath, c)) } // TestPrecheckVectorFeaturesAcceptsValidConfig verifies the precheck @@ -59,7 +62,8 @@ func TestPrecheckVectorFeaturesAcceptsValidConfig(t *testing.T) { c.Vector.Embeddings.Endpoint = "http://localhost:11434/v1/embeddings" c.Vector.Embeddings.Model = "test-model" c.Vector.Embeddings.Dimension = 768 - withTestConfig(t, c) + testCtx := withTestConfig(t, c) + _ = testCtx - assert.NoError(t, precheckVectorFeatures(precheckTestMainPath)) + assert.NoError(t, precheckVectorFeatures(precheckTestMainPath, c)) } diff --git a/cmd/msgvault/cmd/serve_vector_query_client_test.go b/cmd/msgvault/cmd/serve_vector_query_client_test.go new file mode 100644 index 000000000..ef882ab68 --- /dev/null +++ b/cmd/msgvault/cmd/serve_vector_query_client_test.go @@ -0,0 +1,134 @@ +//go:build sqlite_vec || pgvector + +package cmd + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/msgvault/internal/vector" +) + +// embedRequestRecorder captures the one request a query-time embedding call +// makes, so a test can assert the protocol on the wire rather than the Go +// type of the client. +type embedRequestRecorder struct { + mu sync.Mutex + path string + body map[string]any +} + +func (r *embedRequestRecorder) record(req *http.Request) { + r.mu.Lock() + defer r.mu.Unlock() + r.path = req.URL.Path + r.body = map[string]any{} + _ = json.NewDecoder(req.Body).Decode(&r.body) +} + +func (r *embedRequestRecorder) seen() (string, map[string]any) { + r.mu.Lock() + defer r.mu.Unlock() + return r.path, r.body +} + +// embedTestServer serves one canned embedding response and records the +// request that asked for it. +func embedTestServer(t *testing.T, response string) (*embedRequestRecorder, string) { + t.Helper() + rec := &embedRequestRecorder{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rec.record(r) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(response)) + })) + t.Cleanup(srv.Close) + return rec, srv.URL + "/v1" +} + +// queryClientTestConfig is a minimal vector config for the query-side client, +// defaults applied the way config load applies them. +func queryClientTestConfig(endpoint string, format vector.EmbeddingAPIFormat, model string) vector.Config { + var c vector.Config + c.Enabled = true + c.Embeddings.Endpoint = endpoint + c.Embeddings.Model = model + c.Embeddings.APIFormat = format + c.Embeddings.Dimension = 3 + c.ApplyDefaults() + return c +} + +// TestNewQueryEmbeddingClient_VoyageContextualUsesContextualQueryRole pins the +// contract a Voyage-contextual eval or search run depends on: the query goes to +// the contextual endpoint, in the nested request shape, tagged with the query +// role. Constructing the OpenAI-compatible client for this config instead — +// which is what the eval command used to do unconditionally — posts a flat +// {"input": [...]} body to /v1/embeddings with no input_type, so the query +// vector would come from a different endpoint and a different role than the +// documents it is compared against, if the request succeeded at all. +func TestNewQueryEmbeddingClient_VoyageContextualUsesContextualQueryRole(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + rec, endpoint := embedTestServer(t, `{"data":[{"index":0,"data":[{"index":0,"embedding":[0.25,0.5,0.75]}]}]}`) + client, err := newQueryEmbeddingClient( + queryClientTestConfig(endpoint, vector.APIFormatVoyageContextual, "voyage-context-4"), "") + require.NoError(err) + + vec, err := client.EmbedQuery(context.Background(), "who signed the lease?") + require.NoError(err) + assert.Equal([]float32{0.25, 0.5, 0.75}, vec) + + path, body := rec.seen() + assert.Equal("/v1/contextualizedembeddings", path, "the contextual endpoint, not /embeddings") + assert.Equal("query", body["input_type"], "a query must be embedded in the query role, not the document role") + assert.Equal([]any{[]any{"who signed the lease?"}}, body["inputs"], "the contextual request nests chunks per document") +} + +// TestNewQueryEmbeddingClient_DefaultFormatStaysOpenAICompatible pins the other +// half: an omitted api_format is still the OpenAI-compatible path, unchanged. +func TestNewQueryEmbeddingClient_DefaultFormatStaysOpenAICompatible(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + rec, endpoint := embedTestServer(t, `{"data":[{"index":0,"embedding":[0.25,0.5,0.75]}]}`) + client, err := newQueryEmbeddingClient(queryClientTestConfig(endpoint, "", "bge-m3"), "") + require.NoError(err) + + vec, err := client.EmbedQuery(context.Background(), "who signed the lease?") + require.NoError(err) + assert.Equal([]float32{0.25, 0.5, 0.75}, vec) + + path, body := rec.seen() + assert.Equal("/v1/embeddings", path) + assert.Equal([]any{"who signed the lease?"}, body["input"]) + assert.NotContains(body, "input_type", "the OpenAI-compatible body carries no role") +} + +// TestNewQueryEmbeddingClient_RejectsFormatsItCannotBuild proves the selector +// fails loudly on a format it has no client for, naming the offending value, +// rather than silently falling back to the OpenAI-compatible client and +// scoring a protocol mismatch as retrieval quality. +func TestNewQueryEmbeddingClient_RejectsFormatsItCannotBuild(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + _, err := newQueryEmbeddingClient(queryClientTestConfig("http://127.0.0.1:1/v1", "voyage", "voyage-context-4"), "") + require.Error(err, "an api_format with no client must not fall back") + assert.Contains(err.Error(), `"voyage"`, "the error names the unsupported value") + + // A contextual format with a model the contextual endpoint does not serve + // is the same class of silent mismatch, and fails the same way. + _, err = newQueryEmbeddingClient( + queryClientTestConfig("http://127.0.0.1:1/v1", vector.APIFormatVoyageContextual, "voyage-large-4"), "") + require.Error(err) + assert.Contains(err.Error(), "voyage-large-4") +} diff --git a/cmd/msgvault/cmd/serve_vector_stub.go b/cmd/msgvault/cmd/serve_vector_stub.go index 8e398ffdc..735c494b3 100644 --- a/cmd/msgvault/cmd/serve_vector_stub.go +++ b/cmd/msgvault/cmd/serve_vector_stub.go @@ -6,6 +6,7 @@ import ( "context" "errors" + "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/scheduler" "go.kenn.io/msgvault/internal/store" "go.kenn.io/msgvault/internal/vector" @@ -15,15 +16,15 @@ import ( func newConvergenceChecker( vector.Config, *store.Store, vector.Backend, vector.SemanticPersonEmbeddingGate, ) (scheduler.ConvergenceChecker, error) { - return nil, errVectorBuildUnsupported(cfg.DatabaseDSN()) + return nil, errVectorBuildUnsupported("") } func convergenceError(vector.GenerationID, scheduler.ConvergenceResult) error { - return errVectorBuildUnsupported(cfg.DatabaseDSN()) + return errVectorBuildUnsupported("") } func manualConvergenceError(vector.GenerationID, scheduler.ConvergenceResult) error { - return errVectorBuildUnsupported(cfg.DatabaseDSN()) + return errVectorBuildUnsupported("") } // errVectorBuildUnsupported reports that vector search is enabled in @@ -47,7 +48,18 @@ func errVectorBuildUnsupported(mainPath string) error { // (nil, nil) when vector search is disabled, and a descriptive error // when the user enabled vector search in config but built the binary // without -tags sqlite_vec. -func setupVectorFeatures(_ context.Context, _ *store.Store, mainPath string, _ bool, _ ...visual.StreamOpener) (*vectorFeatures, error) { +func setupVectorFeatures(ctx context.Context, _ *store.Store, mainPath string, _ bool, _ ...visual.StreamOpener) (*vectorFeatures, error) { + state := invocationFromContext(ctx) + if state == nil { + return nil, errors.New("configuration is unavailable") + } + return setupVectorFeaturesWithConfig(mainPath, state.cfg) +} + +func setupVectorFeaturesWithConfig(mainPath string, cfg *config.Config) (*vectorFeatures, error) { + if cfg == nil { + return nil, errors.New("configuration is unavailable") + } if !cfg.Vector.AnyLaneEnabled() { return nil, nil //nolint:nilnil // vector disabled: callers nil-check vf; (nil, nil) means "no features, no error" } @@ -58,7 +70,10 @@ func setupVectorFeatures(_ context.Context, _ *store.Store, mainPath string, _ b // It mirrors setupVectorFeatures's enabled/disabled gate without the // backend construction, since the stub build never has a backend to // build. -func precheckVectorFeatures(mainPath string) error { +func precheckVectorFeatures(mainPath string, cfg *config.Config) error { + if cfg == nil { + return errors.New("configuration is unavailable") + } if !cfg.Vector.AnyLaneEnabled() { return nil } diff --git a/cmd/msgvault/cmd/serve_vector_stub_test.go b/cmd/msgvault/cmd/serve_vector_stub_test.go index c2e2409b4..95e644d86 100644 --- a/cmd/msgvault/cmd/serve_vector_stub_test.go +++ b/cmd/msgvault/cmd/serve_vector_stub_test.go @@ -3,7 +3,6 @@ package cmd import ( - "context" "testing" "github.com/stretchr/testify/assert" @@ -16,13 +15,17 @@ import ( // config but builds the binary without -tags sqlite_vec. Runs only // under the untagged build, which is where this error path exists. func TestSetupVectorFeatures_EnabledWithoutTag(t *testing.T) { + cfg := testConfigValue() + prev := cfg t.Cleanup(func() { cfg = prev }) cfg = &config.Config{} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Vector.Enabled = true - vf, err := setupVectorFeatures(context.Background(), nil, "", false) + vf, err := setupVectorFeatures(testCtx, nil, "", false) require.Error(t, err, "setupVectorFeatures with Enabled=true but no tag") assert.Nil(t, vf, "vf should be nil when error is returned") msg := err.Error() @@ -37,12 +40,13 @@ func TestSetupVectorFeatures_EnabledWithoutTag(t *testing.T) { func TestPrecheckVectorFeatures_Stub(t *testing.T) { c := &config.Config{} c.Vector.Enabled = false - withTestConfig(t, c) + testCtx := withTestConfig(t, c) + _ = testCtx - assert.NoError(t, precheckVectorFeatures("/tmp/x.db"), "disabled: precheck should be a no-op") + assert.NoError(t, precheckVectorFeatures("/tmp/x.db", c), "disabled: precheck should be a no-op") c.Vector.Enabled = true - err := precheckVectorFeatures("/tmp/x.db") + err := precheckVectorFeatures("/tmp/x.db", c) require.Error(t, err, "enabled without vector build tags") assert.Contains(t, err.Error(), "sqlite_vec") } diff --git a/cmd/msgvault/cmd/serve_vector_visual_credentials_test.go b/cmd/msgvault/cmd/serve_vector_visual_credentials_test.go index 298c32f8c..58e70680f 100644 --- a/cmd/msgvault/cmd/serve_vector_visual_credentials_test.go +++ b/cmd/msgvault/cmd/serve_vector_visual_credentials_test.go @@ -169,6 +169,8 @@ func TestNewVisualRuntimeUsesStoredCredentialSnapshotAndRejectsRedirectReplay(t require.NoError(t, writeVisualCapabilityManifest(manifestPath, manifest)) cfg := config.NewDefaultConfig() + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx cfg.Data.DataDir = dir cfg.Vector.Multimodal.Enabled = true cfg.Vector.Multimodal.CapabilitiesFile = manifestPath @@ -212,11 +214,11 @@ func TestNewVisualRuntimeUsesStoredCredentialSnapshotAndRejectsRedirectReplay(t )} runtime, err := newVisualRuntime( - t.Context(), cfg.Vector, mainStore, backend, unavailableVisualCredentialOpener{}, + testCtx, cfg.Vector, mainStore, backend, unavailableVisualCredentialOpener{}, visualRuntimeCredential{APIKey: apiKey, HTTPClient: httpClient}, ) require.NoError(t, err) - _, _, err = runtime.Provider.EmbedQuery(t.Context(), visual.QueryInput{Text: "private query"}) + _, _, err = runtime.Provider.EmbedQuery(testCtx, visual.QueryInput{Text: "private query"}) require.Error(t, err) assert.Equal(t, "Bearer stored-at-startup", authorization) assert.Empty(t, redirectedAuthorization) diff --git a/cmd/msgvault/cmd/setup.go b/cmd/msgvault/cmd/setup.go index de7a6e47a..04c494c5c 100644 --- a/cmd/msgvault/cmd/setup.go +++ b/cmd/msgvault/cmd/setup.go @@ -7,13 +7,17 @@ import ( "errors" "fmt" "io" + "maps" "net" "os" "path/filepath" + "slices" "strconv" "strings" "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/oauth" ) var setupCmd = &cobra.Command{ @@ -22,9 +26,10 @@ var setupCmd = &cobra.Command{ Long: `Interactive setup wizard to configure msgvault for first use. This command helps you: - 1. Locate or configure Google OAuth credentials - 2. Create the config.toml file - 3. Optionally configure a remote NAS server for token export + 1. Optionally configure Google OAuth credentials (Gmail and Google + Calendar only; press Enter to skip) + 2. Optionally configure a remote NAS server for token export + 3. Create the config.toml file Run this once after installing msgvault to get started quickly. Then run "msgvault setup providers" to turn on search, attachment, and people lanes @@ -42,7 +47,12 @@ func init() { } func runSetup(cmd *cobra.Command, args []string) error { - reader := bufio.NewReader(os.Stdin) + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + reader := bufio.NewReader(cmd.InOrStdin()) fmt.Println("Welcome to msgvault setup!") fmt.Println() @@ -53,61 +63,108 @@ func runSetup(cmd *cobra.Command, args []string) error { } // Step 1: Find or prompt for OAuth credentials - secretsPath, err := setupOAuthSecrets(reader) + secretsPath, err := setupOAuthSecrets(reader, cfg) if err != nil { return err } // Step 2: Optionally configure remote NAS - remoteURL, remoteAPIKey, err := setupRemoteServer(reader, secretsPath) + remoteURL, remoteAPIKey, err := setupRemoteServer(reader, secretsPath, cfg) if err != nil { return err } - // Step 3: Update config + // Step 3: Persist choices made in the wizard, including values that match + // environment overrides. Keeping existing settings does not persist them. + var edits []config.Edit if secretsPath != "" { cfg.OAuth.ClientSecrets = secretsPath + edits = append(edits, config.Edit{Key: "oauth.client_secrets", Value: secretsPath}) } if remoteURL != "" { cfg.Remote.URL = remoteURL - cfg.Remote.APIKey = remoteAPIKey - // Auto-set for HTTP: target is Tailscale/LAN, not public internet. - if strings.HasPrefix(remoteURL, "http://") { - cfg.Remote.AllowInsecure = true - } + edits = append(edits, + config.Edit{Key: "remote.url", Value: remoteURL}, + config.Edit{Key: "remote.api_key", Value: remoteAPIKey}, + config.Edit{Key: "remote.allow_insecure", Value: strings.HasPrefix(remoteURL, "http://")}, + ) } // Only save if we configured something - if secretsPath != "" || remoteURL != "" { - if err := cfg.Save(); err != nil { + if len(edits) > 0 { + snapshot, err := config.ReadConfigFile(cfg.ConfigFilePath()) + if err == nil { + _, err = config.EditConfigFilePrivate(cfg.ConfigFilePath(), snapshot.ETag, edits) + } + if err != nil { return fmt.Errorf("save config: %w", err) } fmt.Printf("\nConfiguration saved to %s\n", cfg.ConfigFilePath()) } - // Print next steps - fmt.Println() - fmt.Println("Setup complete! Next steps:") - fmt.Println() - fmt.Println(" 1. Add a Gmail account:") - fmt.Println(" msgvault add-account you@gmail.com") - fmt.Println() - fmt.Println(" 2. Sync your emails:") - fmt.Println(" msgvault sync-full you@gmail.com") - fmt.Println() - if remoteURL != "" { - fmt.Println(" 3. Export token to your NAS (after add-account):") - fmt.Println(" msgvault export-token you@gmail.com") - fmt.Println() + printSetupNextSteps(cmd.OutOrStdout(), setupAddAccountCommand(&cfg.OAuth), cfg.Remote.URL != "", + cfg.OAuth.ClientSecrets != "" && cfg.OAuth.ServiceAccountKey == "") + return nil +} + +// setupAddAccountCommand returns the add-account invocation that works +// with the configured Google credentials, or "" when there are none. +// Without a default credential, add-account needs --oauth-app to pick a +// named app. +func setupAddAccountCommand(o *config.OAuthConfig) string { + const base = "msgvault add-account you@gmail.com" + if o.ClientSecrets != "" || o.ServiceAccountKey != "" { + return base } - fmt.Println("For more help: msgvault --help") + names := slices.Sorted(maps.Keys(o.Apps)) + for _, name := range names { + if app := o.Apps[name]; app.ClientSecrets != "" || app.ServiceAccountKey != "" { + return base + " --oauth-app " + oauth.ShellQuote(name) + } + } + return "" +} - return nil +// printSetupNextSteps prints the closing steps. bundleHasSecrets reports +// whether the NAS bundle carries the credential add-account uses. The +// bundle copies only the default [oauth] client_secrets, and add-account +// prefers a service account key, which leaves no token to export. +func printSetupNextSteps(w io.Writer, addAccountCmd string, hasRemote, bundleHasSecrets bool) { + var b strings.Builder + b.WriteString("\nSetup complete! Next steps:\n\n") + if addAccountCmd != "" { + b.WriteString(" 1. Add a Gmail account:\n") + b.WriteString(" " + addAccountCmd + "\n\n") + b.WriteString(" 2. Sync your emails:\n") + b.WriteString(" msgvault sync-full you@gmail.com\n\n") + switch { + case hasRemote && bundleHasSecrets: + b.WriteString(" 3. Export token to your NAS (after add-account):\n") + b.WriteString(" msgvault export-token you@gmail.com\n\n") + case hasRemote: + b.WriteString(" This account cannot be exported to the NAS: export-token needs a\n") + b.WriteString(" token from the default [oauth] client_secrets, the only credential\n") + b.WriteString(" the NAS bundle carries.\n\n") + } + } else { + b.WriteString(" Add a source, for example:\n") + b.WriteString(" msgvault add-imap --host imap.example.com --username you@example.com\n") + b.WriteString(" msgvault add-o365 you@example.com\n") + // A configured remote would receive this local file path. + if !hasRemote { + b.WriteString(" msgvault import-mbox you@example.com /path/to/export.mbox\n") + } + b.WriteString("\n") + b.WriteString(" Gmail needs a Google OAuth credential: run msgvault setup again when you have one.\n") + b.WriteString(" Setup guide: https://msgvault.io/docs/setup/\n\n") + } + b.WriteString("For more help: msgvault --help\n") + _, _ = io.WriteString(w, b.String()) } -func setupOAuthSecrets(reader *bufio.Reader) (string, error) { - fmt.Println("Step 1: OAuth Credentials") - fmt.Println("--------------------------") +func setupOAuthSecrets(reader *bufio.Reader, cfg *config.Config) (string, error) { + fmt.Println("Step 1: Google OAuth Credentials (Optional)") + fmt.Println("--------------------------------------------") // Check if already configured if cfg.OAuth.ClientSecrets != "" { @@ -118,7 +175,8 @@ func setupOAuthSecrets(reader *bufio.Reader) (string, error) { } fmt.Println() - fmt.Println("You need a Google Cloud OAuth credential (client_secret.json).") + fmt.Println("Gmail and Google Calendar need a Google Cloud OAuth credential") + fmt.Println("(client_secret.json). Other sources do not. Press Enter to skip.") fmt.Println() fmt.Println("To get one:") fmt.Println(" 1. Go to https://console.cloud.google.com/apis/credentials") @@ -132,7 +190,8 @@ func setupOAuthSecrets(reader *bufio.Reader) (string, error) { path = strings.TrimSpace(path) if path == "" { - return "", errors.New("OAuth credentials path is required") + fmt.Println("Skipping Google OAuth credentials.") + return "", nil } // Expand ~ in path @@ -150,7 +209,7 @@ func setupOAuthSecrets(reader *bufio.Reader) (string, error) { return path, nil } -func setupRemoteServer(reader *bufio.Reader, oauthSecretsPath string) (string, string, error) { +func setupRemoteServer(reader *bufio.Reader, oauthSecretsPath string, cfg *config.Config) (string, string, error) { fmt.Println() fmt.Println("Step 2: Remote NAS Server (Optional)") fmt.Println("-------------------------------------") @@ -161,7 +220,7 @@ func setupRemoteServer(reader *bufio.Reader, oauthSecretsPath string) (string, s if cfg.Remote.URL != "" { fmt.Printf("Remote server already configured: %s\n", cfg.Remote.URL) if promptYesNo(reader, "Keep existing configuration?") { - return cfg.Remote.URL, cfg.Remote.APIKey, nil + return "", "", nil } } @@ -250,15 +309,16 @@ func createNASBundle(bundleDir, apiKey, oauthSecretsPath string, port int) error } // Create NAS config.toml + oauthBlock := "" + if oauthSecretsPath != "" { + oauthBlock = "[oauth]\nclient_secrets = \"/data/client_secret.json\"\n\n" + } nasConfig := fmt.Sprintf(`[server] bind_addr = "0.0.0.0" api_port = 8080 api_key = %q -[oauth] -client_secrets = "/data/client_secret.json" - -[sync] +%s[sync] rate_limit_qps = 5 # Accounts will be added automatically when you export tokens. @@ -267,19 +327,22 @@ rate_limit_qps = 5 # email = "you@gmail.com" # schedule = "0 2 * * *" # enabled = true -`, apiKey) +`, apiKey, oauthBlock) configPath := filepath.Join(bundleDir, "config.toml") if err := os.WriteFile(configPath, []byte(nasConfig), 0600); err != nil { return fmt.Errorf("write config.toml: %w", err) } - // Copy client_secret.json if available + // Copy client_secret.json if available. Otherwise remove a copy left + // by an earlier run, so the bundle ships no credential it does not use. + destPath := filepath.Join(bundleDir, "client_secret.json") if oauthSecretsPath != "" { - destPath := filepath.Join(bundleDir, "client_secret.json") if err := copyFile(oauthSecretsPath, destPath); err != nil { return fmt.Errorf("copy client_secret.json: %w", err) } + } else if err := os.Remove(destPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("remove stale client_secret.json: %w", err) } // Create docker-compose.yml diff --git a/cmd/msgvault/cmd/setup_lanes.go b/cmd/msgvault/cmd/setup_lanes.go index e2470d96b..ea81448e0 100644 --- a/cmd/msgvault/cmd/setup_lanes.go +++ b/cmd/msgvault/cmd/setup_lanes.go @@ -47,25 +47,22 @@ const ( // Recommended provider defaults. These are the values setup writes when // nothing is configured; every one of them remains settable per lane. - setupVoyageKeyEnv = "VOYAGE_API_KEY" // #nosec G101 -- environment variable name, not a credential. - setupOpenAIKeyEnv = "OPENAI_API_KEY" // #nosec G101 -- environment variable name, not a credential. - setupVoyageEndpoint = "https://api.voyageai.com/v1" - setupVoyageTextModel = "voyage-context-4" - setupVoyageTextDim = 1024 - setupOpenAIEndpoint = "https://api.openai.com/v1" - setupOpenAITextModel = "text-embedding-3-small" - setupOpenAITextDim = 1536 - setupOllamaTextModel = "nomic-embed-text" - setupOllamaTextDim = 768 - setupOllamaDocPrefix = "search_document: " - setupOllamaQueryPrefix = "search_query: " - setupOllamaMaxInput = 2000 - setupEmbedCron = "*/15 * * * *" - setupInferenceModel = "gpt-5.6-luna" - setupInferenceReasoning = "medium" - setupInferenceProfile = "openai" - setupOllamaProfile = "ollama" - setupPostureDeclared = "provider-declared" + setupVoyageKeyEnv = "VOYAGE_API_KEY" // #nosec G101 -- environment variable name, not a credential. + setupOpenAIKeyEnv = "OPENAI_API_KEY" // #nosec G101 -- environment variable name, not a credential. + setupVoyageEndpoint = "https://api.voyageai.com/v1" + setupVoyageTextModel = "voyage-context-4" + setupVoyageTextDim = 1024 + setupOpenAIEndpoint = "https://api.openai.com/v1" + setupOpenAITextModel = "text-embedding-3-small" + setupOpenAITextDim = 1536 + setupOllamaTextModel = "nomic-embed-text" + setupOllamaTextDim = 768 + setupOllamaDocPrefix = "search_document: " + setupOllamaQueryPrefix = "search_query: " + setupOllamaMaxInput = 2000 + setupEmbedCron = "*/15 * * * *" + setupOllamaProfile = "ollama" + setupPostureDeclared = "provider-declared" setupVoyageManifestName = "voyage-capabilities.json" setupMistralManifestName = "mistral-capabilities.json" @@ -620,14 +617,25 @@ func peopleInferenceLane(cfg *config.Config, env setupEnvironment) laneStatus { lane.State = laneStateOff if env.hasEnv(setupOpenAIKeyEnv) { lane.State = laneStatePending - lane.Reason = setupOpenAIKeyEnv + " present; setup can onboard the " + setupInferenceModel + " profile" - lane.Next = []string{"msgvault setup providers"} + lane.Reason = setupOpenAIKeyEnv + " is available; select an HTTP people inference provider with --provider and --model, or enroll Codex through the daemon" + lane.Next = []string{peopleInferencePresetSetupCommand("openai", setupOpenAIKeyEnv), peopleInferenceCodexEnrollCommand()} return lane } - lane.Reason = "needs " + setupOpenAIKeyEnv + " or a reachable local Ollama server, then `msgvault setup providers`" + lane.Reason = "choose an HTTP people inference provider with --provider and --model, enroll Codex through the daemon, or use a reachable local Ollama server" + lane.Next = []string{peopleInferencePresetSetupCommand("", ""), peopleInferenceCodexEnrollCommand()} return lane } +func peopleInferencePresetSetupCommand(provider, credentialEnv string) string { + return "msgvault setup providers --provider " + provider + " --model --credential-env " + credentialEnv + + " --retention-posture --training-posture --allow-sensitive= --yes" +} + +func peopleInferenceCodexEnrollCommand() string { + return "msgvault person provider enroll-codex --source conversation_text --source-since " + + " --retention-posture --training-posture --allow-sensitive=" +} + func activityLane(cfg *config.Config) laneStatus { lane := laneStatus{Lane: laneActivity, Label: "Contact activity (last contacted, cadence)"} if cfg.Activity.Schedule == "" { diff --git a/cmd/msgvault/cmd/setup_providers.go b/cmd/msgvault/cmd/setup_providers.go index b7a6e4e49..45da03bf1 100644 --- a/cmd/msgvault/cmd/setup_providers.go +++ b/cmd/msgvault/cmd/setup_providers.go @@ -34,9 +34,11 @@ const ( planActionOnboard = "onboard" // Consent gates: one explicit answer per hosted provider. - gateVoyage = "voyage" - gateMistral = "mistral" - gateOpenAI = "openai" + gateVoyage = "voyage" + gateMistral = "mistral" + gateOpenAI = "openai" + gateOpenRouter = "openrouter" + gateVenice = "venice" ollamaProbeTimeout = 2 * time.Second ollamaProbeMaxBody = 1 << 20 @@ -56,6 +58,10 @@ type setupProvidersOptions struct { trainingPosture string personRetentionPosture string personTrainingPosture string + providerID string + model string + credentialEnv string + apiKeyStdin bool } // ollamaProbeResult is what a local Ollama server reports about itself. @@ -68,6 +74,7 @@ type ollamaProbeResult struct { // environment, the config file, the archive, the daemon, and the people // provider onboarding machinery are all injectable. type setupProvidersDeps struct { + bind func(context.Context) setupProvidersDeps lookupEnv func(string) (string, bool) fileExists func(string) bool readConfigFile func() (config.ConfigFile, error) @@ -83,39 +90,67 @@ type setupProvidersDeps struct { now func() time.Time } -func defaultSetupProvidersDeps() setupProvidersDeps { +func defaultSetupProvidersDeps(contexts ...context.Context) setupProvidersDeps { + if len(contexts) > 0 { + deps := defaultSetupProvidersDeps() + if deps.bind != nil { + return deps.bind(contexts[0]) + } + } return setupProvidersDeps{ + bind: func(ctx context.Context) setupProvidersDeps { + deps := defaultSetupProvidersDeps() + state := invocationFromContext(ctx) + var currentCfg *config.Config + if state != nil && state.cfg != nil { + currentCfg = state.cfg + } + deps.readConfigFile = func() (config.ConfigFile, error) { + if currentCfg == nil { + return config.ConfigFile{}, errors.New("configuration is unavailable") + } + return config.ReadConfigFile(currentCfg.ConfigFilePath()) + } + deps.editConfigTables = func(ifMatch string, edits []config.TableEdit) (config.ConfigFile, error) { + if currentCfg == nil { + return config.ConfigFile{}, errors.New("configuration is unavailable") + } + return config.EditConfigTables(currentCfg.ConfigFilePath(), ifMatch, edits) + } + deps.loadConfig = func(snapshot config.ConfigFile) (*config.Config, error) { + if currentCfg == nil { + return nil, errors.New("configuration is unavailable") + } + return loadSetupConfig(snapshot, currentCfg.HomeDir) + } + deps.remoteConfigured = func() bool { return isRemoteModeFor(state) } + deps.personProvider = func() personProviderCommandDeps { + return defaultPersonProviderCommandDepsForContext(ctx) + } + return deps + }, lookupEnv: os.LookupEnv, fileExists: defaultFileExists, readConfigFile: func() (config.ConfigFile, error) { - if cfg == nil { - return config.ConfigFile{}, errors.New("configuration is unavailable") - } - return config.ReadConfigFile(cfg.ConfigFilePath()) + return config.ConfigFile{}, errors.New("configuration is unavailable") }, editConfigTables: func(ifMatch string, edits []config.TableEdit) (config.ConfigFile, error) { - if cfg == nil { - return config.ConfigFile{}, errors.New("configuration is unavailable") - } - return config.EditConfigTables(cfg.ConfigFilePath(), ifMatch, edits) + return config.ConfigFile{}, errors.New("configuration is unavailable") }, restoreConfigFile: func(published, before config.ConfigFile) (config.ConfigFile, error) { return config.RestoreConfigFile(before.LogicalPath, published, before) }, loadConfig: func(snapshot config.ConfigFile) (*config.Config, error) { - if cfg == nil { - return nil, errors.New("configuration is unavailable") - } - return loadSetupConfig(snapshot, cfg.HomeDir) + return nil, errors.New("configuration is unavailable") }, - remoteConfigured: IsRemoteMode, + remoteConfigured: func() bool { return false }, isTerminal: commandStdinIsTerminal, probeOllama: probeOllamaServer, consentState: readSetupConsentState, daemonAlive: func(ctx context.Context, loaded *config.Config) bool { return findAnyDaemonRuntimeContext(ctx, loaded.Data.DataDir) != nil }, - personProvider: defaultPersonProviderCommandDeps, + personProvider: func() personProviderCommandDeps { return defaultPersonProviderCommandDeps() }, now: time.Now, } } @@ -304,7 +339,7 @@ func (p *setupProvidersPlan) gates() []string { } } ordered := []string{} - for _, gate := range []string{gateVoyage, gateMistral, gateOpenAI} { + for _, gate := range []string{gateVoyage, gateMistral, gateOpenAI, gateOpenRouter, gateVenice} { if seen[gate] { ordered = append(ordered, gate) } @@ -379,6 +414,20 @@ func (p *setupProvidersPlan) mergedEdits() []config.TableEdit { } func validateSetupProvidersOptions(options setupProvidersOptions) error { + if options.providerID == "" && (options.model != "" || options.credentialEnv != "" || options.apiKeyStdin) { + return errors.New("--model, --credential-env, and --api-key-stdin require --provider") + } + if options.providerID != "" { + if _, err := peoplesweep.PresetProviderConfig(options.providerID, "validation-model"); err != nil { + return err + } + if strings.TrimSpace(options.model) == "" { + return errors.New("--model is required with --provider") + } + if options.apiKeyStdin && options.credentialEnv != "" { + return errors.New("--api-key-stdin and --credential-env are mutually exclusive") + } + } if options.documentRetention != documentindex.RetentionStandard && options.documentRetention != documentindex.RetentionZDR { return fmt.Errorf("--document-retention must be %q or %q", documentindex.RetentionStandard, documentindex.RetentionZDR) } @@ -683,8 +732,8 @@ func planPeopleInference( lane.Reason = "already enabled" return lane, nil } - if !options.allowSensitive && (detection.openAIKey || - (detection.ollama.Reachable && detection.ollamaLoopback && detection.ollama.hasModel(loaded.Chat.Model))) { + if options.providerID == "" && !options.allowSensitive && + detection.ollama.Reachable && detection.ollamaLoopback && detection.ollama.hasModel(loaded.Chat.Model) { lane.Action = planActionPending lane.Reason = "people sweep requires --allow-sensitive: sensitive archive excerpts may be sent to the selected inference provider and used to infer sensitive personal attributes" lane.next = []string{"msgvault setup providers --allow-sensitive"} @@ -701,22 +750,35 @@ func planPeopleInference( allowedSources: sources, sourceSince: since, allowSensitive: options.allowSensitive, requestTimeout: time.Minute, confirmed: true, } - switch { - case detection.openAIKey: - if _, exists := loaded.People.Sweep.Providers[setupInferenceProfile]; exists { - lane.Action = planActionSkip - lane.Provider = setupInferenceProfile - lane.Reason = "profile exists but the sweep is off; run `msgvault person provider consent " + - setupInferenceProfile + " --yes` and `msgvault person provider use " + setupInferenceProfile + "`" + if options.providerID != "" { + preset, err := peoplesweep.PresetProviderConfig(options.providerID, options.model) + if err != nil { + lane.Action, lane.Reason = planActionSkip, err.Error() + return lane, nil + } + base.custom = false + base.presetID = options.providerID + base.protocol, base.endpoint, base.auth = string(preset.Protocol), preset.Endpoint, string(preset.Auth) + base.model, base.credentialEnv, base.apiKeyStdin = options.model, options.credentialEnv, options.apiKeyStdin + gate := options.providerID + if _, exists := loaded.People.Sweep.Providers[gate]; exists { + lane.Action, lane.Provider = planActionSkip, gate + lane.Reason = "profile exists but the sweep is off; review and select it with `msgvault person provider use " + gate + "`" return lane, nil } - base.endpoint, base.model, base.auth = setupOpenAIEndpoint, setupInferenceModel, string(peoplesweep.AuthBearer) - base.credentialEnv, base.reasoningEffort = setupOpenAIKeyEnv, setupInferenceReasoning - lane.Action, lane.Provider, lane.Model, lane.Gate = planActionOnboard, setupInferenceProfile, setupInferenceModel, gateOpenAI - lane.Reason = fmt.Sprintf("openai_chat profile %q at %s reasoning; sensitive archive excerpts from %s since %s may be sent to OpenAI and used to infer sensitive personal attributes; extraction runs for tracked people only", - setupInferenceProfile, setupInferenceReasoning, strings.Join(sources, ", "), since) + lane.Action, lane.Provider, lane.Model, lane.Gate = planActionOnboard, gate, options.model, gate + lane.Reason = "selected " + gate + " people inference profile; synthetic check and separate disclosure consent are required" lane.next = []string{"msgvault person track "} - return lane, &setupInferencePlan{name: setupInferenceProfile, options: base, gate: gateOpenAI} + return lane, &setupInferencePlan{name: gate, options: base, gate: gate} + } + switch { + case detection.openAIKey: + lane.Action = planActionSkip + command := peopleInferencePresetSetupCommand("openai", setupOpenAIKeyEnv) + codexCommand := peopleInferenceCodexEnrollCommand() + lane.Reason = setupOpenAIKeyEnv + " is available for embeddings; select people inference explicitly with `" + command + "`, or enroll Codex with `" + codexCommand + "` (requires a terminal)" + lane.next = []string{command, codexCommand} + return lane, nil case detection.ollama.Reachable && detection.ollamaLoopback && detection.ollama.hasModel(loaded.Chat.Model): if _, exists := loaded.People.Sweep.Providers[setupOllamaProfile]; exists { lane.Action = planActionSkip @@ -743,7 +805,10 @@ func planPeopleInference( "` or set [chat].model to an available chat model, then re-run setup" default: lane.Action = planActionSkip - lane.Reason = "needs " + setupOpenAIKeyEnv + " or a local Ollama server" + command := peopleInferencePresetSetupCommand("", "") + codexCommand := peopleInferenceCodexEnrollCommand() + lane.Reason = "choose an HTTP people inference provider with `" + command + "`, enroll Codex with `" + codexCommand + "` (requires a terminal), or run local Ollama" + lane.next = []string{command, codexCommand} } return lane, nil } @@ -786,10 +851,29 @@ func gateDisclosure(gate string, plan *setupProvidersPlan) string { " - document search query text, after `msgvault documents vectors consent --purpose queries --yes`") } if plan.inference != nil && plan.inference.gate == gateOpenAI { + choice := plan.inference.options lines = append(lines, " - bounded evidence packets of "+ - strings.Join(plan.inference.options.allowedSources, ", ")+ - " for tracked people ("+setupInferenceModel+"); a synthetic check request is sent now", - " - --allow-sensitive authorizes sending sensitive archive excerpts to OpenAI and inferring sensitive personal attributes") + strings.Join(choice.allowedSources, ", ")+ + " for tracked people ("+choice.model+") since "+choice.sourceSince+"; a synthetic check request is sent now", + " - recorded assertions: retention="+choice.retentionPosture+", training="+choice.trainingPosture) + if choice.allowSensitive { + lines = append(lines, " - --allow-sensitive authorizes sending sensitive archive excerpts to OpenAI and inferring sensitive personal attributes") + } else { + lines = append(lines, " - sensitive archive excerpts are excluded from provider packets") + } + } + case gateOpenRouter, gateVenice: + if plan.inference != nil && plan.inference.gate == gate { + choice := plan.inference.options + lines = append(lines, gate+" ("+choice.endpoint+") receives:", + " - a synthetic check request now; after consent, bounded evidence packets of "+ + strings.Join(choice.allowedSources, ", ")+" for tracked people ("+choice.model+") since "+choice.sourceSince, + " - recorded assertions: retention="+choice.retentionPosture+", training="+choice.trainingPosture) + if choice.allowSensitive { + lines = append(lines, " - sensitive archive excerpts and inferred personal attributes are allowed") + } else { + lines = append(lines, " - sensitive archive excerpts are excluded from provider packets") + } } } return strings.Join(lines, "\n") @@ -840,7 +924,7 @@ func newSetupProvidersCommand(deps setupProvidersDeps) *cobra.Command { var options setupProvidersOptions command := &cobra.Command{ Use: "providers", - Short: "Turn on the retrieval and people lanes the available API keys support, with recommended defaults", + Short: "Configure retrieval and people inference providers", Long: `Read the environment and configure every lane that is still unset: ` + setupVoyageKeyEnv + ` text search with Voyage contextual embeddings (conversation @@ -848,23 +932,33 @@ func newSetupProvidersCommand(deps setupProvidersDeps) *cobra.Command { people search, and the visual attachment lane once its probe manifest exists ` + setupOpenAIKeyEnv + ` text search on the OpenAI-compatible path when no Voyage key - is present, and the people sweep on ` + setupInferenceModel + ` + is present MISTRAL_API_KEY document attachment extraction, plus document vectors when a text lane is on (no keys) a local Ollama server at [chat].server when it is reachable -Hosted lanes never turn on from a key alone: setup asks once per provider, -writes the recommended values to config.toml, runs the people-provider -check and consent, and prints what is on, what is off, and why. Lanes that -are already configured are left alone, so re-running after adding a key -upgrades only that lane. Probe manifests are expected at +Choose a hosted people inference provider with --provider openai, openrouter, +or venice. Supply --model, a credential source, retention and training +assertions, and --allow-sensitive=true or --allow-sensitive=false. Setup checks +the named provider with a synthetic request before consent and selection. +For a Codex subscription, run msgvault person provider enroll-codex with a +profile name and explicit source scope and policy flags. Codex enrollment +requires a terminal and uses daemon device login; setup never starts the login +flow. + +Hosted lanes require confirmation. Setup prints the planned changes and asks +before writing them. Configured lanes keep their existing settings. Probe +manifests are expected at /` + setupVoyageManifestName + ` and /` + setupMistralManifestName + `. -The people sweep also requires --allow-sensitive: archive excerpts may contain -sensitive details and may be used to infer sensitive personal attributes. ---yes accepts provider prompts but does not grant this separate opt-in.`, +The people sweep can send archive excerpts and infer personal attributes. +--allow-sensitive controls whether sensitive excerpts may be included. +--yes accepts provider prompts; it does not choose a provider or a model.`, Args: cobra.NoArgs, RunE: func(command *cobra.Command, _ []string) error { + if invocationFromContext(command.Context()) != nil && deps.bind != nil { + deps = deps.bind(command.Context()) + } return runSetupProviders(command, deps, options) }, } @@ -882,6 +976,10 @@ sensitive details and may be used to infer sensitive personal attributes. "Retention assertion recorded for embedding and inference providers") flags.StringVar(&options.trainingPosture, "training-posture", setupPostureDeclared, "Training assertion recorded for embedding and inference providers") + flags.StringVar(&options.providerID, "provider", "", "People inference provider preset: openai, openrouter, or venice") + flags.StringVar(&options.model, "model", "", "Explicit people inference model ID") + flags.StringVar(&options.credentialEnv, "credential-env", "", "Read only this environment variable for people inference") + flags.BoolVar(&options.apiKeyStdin, "api-key-stdin", false, "Read the people inference API key from standard input") return command } @@ -889,6 +987,16 @@ func runSetupProviders(command *cobra.Command, deps setupProvidersDeps, options if err := validateSetupProvidersOptions(options); err != nil { return err } + if options.providerID != "" { + if !command.Flags().Changed("retention-posture") || !command.Flags().Changed("training-posture") || + !command.Flags().Changed("allow-sensitive") { + return errors.New("--provider requires explicit --retention-posture, --training-posture, and --allow-sensitive=true|false") + } + if options.credentialEnv == "" && !options.apiKeyStdin && + (deps.isTerminal == nil || !deps.isTerminal(command)) { + return errors.New("--provider requires --credential-env or --api-key-stdin outside a terminal") + } + } if deps.remoteConfigured != nil && deps.remoteConfigured() { return errors.New("setup providers cannot run against a configured remote daemon: it edits this machine's config.toml, which the remote daemon never reads; run it on the daemon host, or pass --local to configure a daemon on this machine") } @@ -904,6 +1012,9 @@ func runSetupProviders(command *cobra.Command, deps setupProvidersDeps, options if err != nil { return err } + if options.providerID != "" && loaded.People.Sweep.Enabled { + return errors.New("people sweep is already enabled; use person provider add and person provider use to change providers") + } // Read saved assertions before config defaults turn an absent document // posture into "unknown". Only the corresponding explicit flag replaces // an existing assertion; inference still uses its own command defaults. diff --git a/cmd/msgvault/cmd/setup_providers_test.go b/cmd/msgvault/cmd/setup_providers_test.go index 35258823b..a7a286c05 100644 --- a/cmd/msgvault/cmd/setup_providers_test.go +++ b/cmd/msgvault/cmd/setup_providers_test.go @@ -31,6 +31,15 @@ import ( const setupProvidersTestKey = "setup-providers-test-key" +func explicitOpenAISetupFlags() []string { + return []string{ + "--provider", "openai", "--model", "gpt-5.6-luna", + "--credential-env", setupOpenAIKeyEnv, + "--retention-posture", setupPostureDeclared, + "--training-posture", setupPostureDeclared, + } +} + // setupProvidersFixture is one operator machine: a real config file, a real // archive store, a fixed environment, and a fixed filesystem view for the // probe manifests. @@ -48,11 +57,12 @@ type setupProvidersFixture struct { func newSetupProvidersFixture(t *testing.T, content string) *setupProvidersFixture { t.Helper() + require := require.New(t) dir := t.TempDir() path := filepath.Join(dir, "config.toml") if content != "" { content = strings.ReplaceAll(content, "{{DIR}}", filepath.ToSlash(dir)) - require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) + require.NoError(os.WriteFile(path, []byte(content), 0o600)) } return &setupProvidersFixture{ dir: dir, @@ -69,10 +79,11 @@ func newSetupProvidersFixture(t *testing.T, content string) *setupProvidersFixtu func (f *setupProvidersFixture) load(t *testing.T) *config.Config { t.Helper() + require := require.New(t) snapshot, err := config.ReadConfigFile(f.path) - require.NoError(t, err) + require.NoError(err) loaded, err := loadSetupConfig(snapshot, f.dir) - require.NoError(t, err) + require.NoError(err) return loaded } @@ -230,6 +241,8 @@ func TestSetupProvidersResolvesDisabledVisualCredential(t *testing.T) { } func TestSetupProvidersPreservesMissingCustomVisualManifest(t *testing.T) { + require := require.New(t) + assert := assert.New(t) fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig+` [vector.multimodal] @@ -238,7 +251,7 @@ capabilities_file = "{{DIR}}/custom-voyage.json" fixture.env[setupVoyageKeyEnv] = setupProvidersTestKey fixture.writeVisualManifest(t, filepath.Join(fixture.dir, setupVoyageManifestName), voyage.CapabilityQueryText) output, err := fixture.run(t, "providers", "--yes") - require.NoError(t, err, output) + require.NoError(err, output) loaded := fixture.load(t) custom := filepath.ToSlash(filepath.Join(fixture.dir, "custom-voyage.json")) assert.False(loaded.Vector.Multimodal.Enabled) @@ -312,6 +325,8 @@ func TestSetupDocumentConsentMatchesCompletePolicy(t *testing.T) { assert.True(t, setupConsentFromStore(t.Context(), c, st).Documents) for _, field := range []string{"provider", "model", "region", "retention", "training", "spool limit", "scope", "normalization", "missing manifest"} { t.Run(field, func(t *testing.T) { + assert := assert.New(t) + changed := *c documents := &changed.Attachments.Documents switch field { @@ -335,10 +350,10 @@ func TestSetupDocumentConsentMatchesCompletePolicy(t *testing.T) { changed.HomeDir = t.TempDir() } consent := setupConsentFromStore(t.Context(), &changed, st) - assert.False(t, consent.Documents) + assert.False(consent.Documents) lane := documentsLane(&changed, setupEnvironment{consent: consent, lookupEnv: func(string) (string, bool) { return setupProvidersTestKey, true }}) - assert.Equal(t, laneStatePending, lane.State) - assert.Equal(t, consentMissing, lane.Consent) + assert.Equal(laneStatePending, lane.State) + assert.Equal(consentMissing, lane.Consent) }) } _, err = st.RetireDocumentExtractionProfile(t.Context(), profile.ID) @@ -348,6 +363,7 @@ func TestSetupDocumentConsentMatchesCompletePolicy(t *testing.T) { func (f *setupProvidersFixture) personProviderDeps(t *testing.T) personProviderCommandDeps { t.Helper() + assert := assert.New(t) loaded := f.load(t) deps := localPersonProviderDeps(loaded.People.Sweep, f.store, f.checker) deps.readConfigFile = func() (config.ConfigFile, error) { return config.ReadConfigFile(f.path) } @@ -362,7 +378,7 @@ func (f *setupProvidersFixture) personProviderDeps(t *testing.T) personProviderC lookupEnv: f.lookupEnv, negotiate: func(_ context.Context, candidate peoplesweep.ProviderConfig, credential peoplesweep.Credential) (peoplesweep.NegotiatedCapabilities, error) { if credential.Scheme != peoplesweep.AuthNone { - assert.Equal(t, setupProvidersTestKey, credential.Value()) + assert.Equal(setupProvidersTestKey, credential.Value()) } return peoplesweep.NegotiatedCapabilities{ OutputMode: peoplesweep.OutputModeNativeJSONSchema, TokenLimitParameter: "max_completion_tokens", @@ -433,11 +449,12 @@ func (f *setupProvidersFixture) run(t *testing.T, args ...string) (string, error func (f *setupProvidersFixture) readConfig(t *testing.T) string { t.Helper() + require := require.New(t) content, err := os.ReadFile(f.path) if os.IsNotExist(err) { return "" } - require.NoError(t, err) + require.NoError(err) return string(content) } @@ -449,6 +466,8 @@ data_dir = "{{DIR}}/data" func TestSetupProvidersPreservesPosturesUnlessFlagsOverride(t *testing.T) { for name, override := range map[string]bool{"preserve": false, "override": true} { t.Run(name, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig+` [vector.people] @@ -467,7 +486,7 @@ training_posture = "opted-out" peopleRetention, documentTraining = setupPostureDeclared, documentindex.TrainingDefaultOptOut } output, err := fixture.run(t, args...) - require.NoError(t, err, output) + require.NoError(err, output) loaded := fixture.load(t) assert.Equal(peopleRetention, loaded.Vector.People.RetentionPosture) assert.Equal("no_training", loaded.Vector.People.TrainingPosture) @@ -479,6 +498,8 @@ training_posture = "opted-out" } func TestSetupProvidersResolvesUnknownDocumentPostures(t *testing.T) { + cfg := testConfigValue() + for _, enabled := range []bool{false, true} { for _, explicit := range []bool{false, true} { t.Run(fmt.Sprintf("enabled=%t/explicit=%t", enabled, explicit), func(t *testing.T) { @@ -512,7 +533,8 @@ training_posture = %q previous := cfg cfg = loaded t.Cleanup(func() { cfg = previous }) - _, _, _, _, err := configuredDocumentProfile(writeCommandCapabilityManifest(t, loaded.Attachments.Documents.MaxPagesPerDocument)) + testCtx := testInvocationContext(t.Context(), loaded, invocationOptions{}) + _, _, _, _, err := configuredDocumentProfile(writeCommandCapabilityManifest(t, loaded.Attachments.Documents.MaxPagesPerDocument), invocationFromContext(testCtx)) require.NoError(err) } }) @@ -523,6 +545,8 @@ training_posture = %q func TestSetupProvidersCustomHostedEndpointNeedsExplicitConfiguration(t *testing.T) { for _, endpoint := range []string{"https://api.openai.com.example.test/v1", "https://localhost.example.test/v1", "https://embeddings.example.test/v1"} { t.Run(endpoint, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig+fmt.Sprintf(` [vector] @@ -538,7 +562,7 @@ training_posture = "opted-out" `, endpoint)) before := fixture.readConfig(t) output, err := fixture.run(t, "providers", "--yes") - require.NoError(t, err, output) + require.NoError(err, output) assert.Equal(before, fixture.readConfig(t)) assert.False(fixture.load(t).Vector.People.Enabled) assert.False(fixture.load(t).Attachments.Documents.Index.Embeddings.Enabled) @@ -560,11 +584,16 @@ func TestEmbeddingProviderNameUsesURLHost(t *testing.T) { "https://api.openai.com@example.test/v1": "custom", "not a URL": "custom", "": "", } { - t.Run(endpoint, func(t *testing.T) { assert.Equal(t, want, embeddingProviderName(endpoint)) }) + t.Run(endpoint, func(t *testing.T) { + assert := assert.New(t) + assert.Equal(want, embeddingProviderName(endpoint)) + }) } } func TestSetupProvidersPostgresRequiresCompiledBackend(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) require := require.New(t) fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig+`database_url = "postgres://localhost/setup_test"`) @@ -579,7 +608,7 @@ func TestSetupProvidersPostgresRequiresCompiledBackend(t *testing.T) { previous := cfg cfg = loaded t.Cleanup(func() { cfg = previous }) - require.NoError(precheckVectorFeatures(loaded.DatabaseDSN())) + require.NoError(precheckVectorFeatures(loaded.DatabaseDSN(), loaded)) if !pgvector.Available() { var result setupProvidersOutput require.NoError(json.Unmarshal([]byte(output), &result)) @@ -589,12 +618,15 @@ func TestSetupProvidersPostgresRequiresCompiledBackend(t *testing.T) { } func TestSetupStatusConfiguredVectorLanesRequireCompiledBackend(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) fixture.env[setupVoyageKeyEnv] = setupProvidersTestKey fixture.env["MISTRAL_API_KEY"] = setupProvidersTestKey fixture.writeVisualManifest(t, filepath.Join(fixture.dir, setupVoyageManifestName), voyage.CapabilityQueryText) output, err := fixture.run(t, "providers", "--yes") - require.NoError(t, err, output) + require.NoError(err, output) loaded := fixture.load(t) previous := cfg cfg = loaded @@ -607,7 +639,7 @@ func TestSetupStatusConfiguredVectorLanesRequireCompiledBackend(t *testing.T) { lookupEnv: fixture.lookupEnv, fileExists: func(path string) bool { return fixture.files[path] }, consent: &setupConsentState{Documents: true, Visual: true, PersonSemantic: true, DocumentEmbedding: true, QueryEmbedding: true}, }) - startupErr := precheckVectorFeatures(loaded.DatabaseDSN()) + startupErr := precheckVectorFeatures(loaded.DatabaseDSN(), loaded) for _, name := range []string{laneTextSearch, lanePersonSearch, laneVisualSearch, laneDocumentVectors} { lane := findLane(t, report, name) if startupErr != nil { @@ -622,21 +654,20 @@ func TestSetupStatusConfiguredVectorLanesRequireCompiledBackend(t *testing.T) { } } -func TestSetupProvidersRequiresSensitiveOptIn(t *testing.T) { +func TestSetupProvidersRequiresExplicitSensitiveDecision(t *testing.T) { assert := assert.New(t) require := require.New(t) fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) fixture.env[setupOpenAIKeyEnv] = setupProvidersTestKey - output, err := fixture.run(t, "providers", "--yes", "--json") - require.NoError(err, output) + output, err := fixture.run(t, append([]string{"providers", "--yes", "--json"}, explicitOpenAISetupFlags()...)...) + require.ErrorContains(err, "--allow-sensitive") assert.False(fixture.load(t).People.Sweep.Enabled) - assert.NotContains(fixture.load(t).People.Sweep.Providers, setupInferenceProfile) + assert.NotContains(fixture.load(t).People.Sweep.Providers, "openai") assert.Zero(fixture.checker.calls.Load()) assert.Contains(output, "--allow-sensitive") - output, err = fixture.run(t, "providers", "--yes", "--allow-sensitive", "--json") + output, err = fixture.run(t, append([]string{"providers", "--yes", "--allow-sensitive", "--json"}, explicitOpenAISetupFlags()...)...) require.NoError(err, output) - assert.Contains(output, "sensitive archive excerpts") assert.EqualValues(1, fixture.checker.calls.Load()) profile, err := fixture.load(t).People.Sweep.Profile() require.NoError(err) @@ -654,7 +685,7 @@ func TestSetupStatusReportsMissingHostedCredentials(t *testing.T) { fixture.env[key] = setupProvidersTestKey } fixture.writeVisualManifest(t, filepath.Join(fixture.dir, setupVoyageManifestName), voyage.CapabilityQueryText) - _, err := fixture.run(t, "providers", "--yes", "--allow-sensitive") + _, err := fixture.run(t, append([]string{"providers", "--yes", "--allow-sensitive"}, explicitOpenAISetupFlags()...)...) require.NoError(err) for lane, key := range map[string]string{ laneTextSearch: setupVoyageKeyEnv, lanePersonSearch: setupVoyageKeyEnv, laneDocumentVectors: setupVoyageKeyEnv, @@ -828,13 +859,15 @@ func TestSetupReportsCheckStoredPeopleCredential(t *testing.T) { } func TestSetupStatusConsentGatedLanesRequireActiveConsent(t *testing.T) { + require := require.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) for _, key := range []string{setupVoyageKeyEnv, "MISTRAL_API_KEY", setupOpenAIKeyEnv} { fixture.env[key] = setupProvidersTestKey } fixture.writeVisualManifest(t, filepath.Join(fixture.dir, setupVoyageManifestName), voyage.CapabilityQueryText) - output, err := fixture.run(t, "providers", "--yes", "--allow-sensitive") - require.NoError(t, err, output) + output, err := fixture.run(t, append([]string{"providers", "--yes", "--allow-sensitive"}, explicitOpenAISetupFlags()...)...) + require.NoError(err, output) loaded := fixture.load(t) for _, test := range []struct { consent *setupConsentState @@ -880,6 +913,8 @@ func TestSetupProvidersPreservesExplicitSchedules(t *testing.T) { {name: "sync disabled", toml: "run_after_sync = false", cron: setupEmbedCron}, } { t.Run(fmt.Sprintf("%s/manifest=%t", schedule.name, manifest), func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig+ "\n[vector.embed.schedule]\n"+schedule.toml+ @@ -889,7 +924,7 @@ func TestSetupProvidersPreservesExplicitSchedules(t *testing.T) { fixture.writeVisualManifest(t, filepath.Join(fixture.dir, setupVoyageManifestName), voyage.CapabilityQueryText) } output, err := fixture.run(t, "providers", "--yes") - require.NoError(t, err, output) + require.NoError(err, output) loaded := fixture.load(t) assert.Equal(schedule.cron, loaded.Vector.Embed.Schedule.Cron) assert.Equal(schedule.runAfterSync, loaded.Vector.Embed.Schedule.RunAfterSync) @@ -1009,21 +1044,24 @@ func TestSetupProvidersMistralEnablesDocumentsAndVectors(t *testing.T) { } func TestSetupProvidersRejectsUnknownDocumentPostures(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) fixture.env["MISTRAL_API_KEY"] = setupProvidersTestKey _, err := fixture.run(t, "providers", "--yes", "--document-retention", "unknown") - require.ErrorContains(t, err, "--document-retention") - assert.Empty(t, strings.TrimSpace(strings.TrimPrefix(fixture.readConfig(t), strings.ReplaceAll(setupProvidersMinimalConfig, "{{DIR}}", filepath.ToSlash(fixture.dir))))) + require.ErrorContains(err, "--document-retention") + assert.Empty(strings.TrimSpace(strings.TrimPrefix(fixture.readConfig(t), strings.ReplaceAll(setupProvidersMinimalConfig, "{{DIR}}", filepath.ToSlash(fixture.dir))))) } -func TestSetupProvidersOpenAIFallbackOnboardsInference(t *testing.T) { +func TestSetupProvidersExplicitOpenAIOnboardsInference(t *testing.T) { assert := assert.New(t) require := require.New(t) fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) fixture.env[setupOpenAIKeyEnv] = setupProvidersTestKey - output, err := fixture.run(t, "providers", "--yes", "--allow-sensitive") + output, err := fixture.run(t, append([]string{"providers", "--yes", "--allow-sensitive"}, explicitOpenAISetupFlags()...)...) require.NoError(err, output) loaded := fixture.load(t) @@ -1036,12 +1074,13 @@ func TestSetupProvidersOpenAIFallbackOnboardsInference(t *testing.T) { sweep := loaded.People.Sweep require.True(sweep.Enabled) - assert.Equal(setupInferenceProfile, sweep.Provider.Name) - profile := sweep.Providers[setupInferenceProfile] + assert.Equal("openai", sweep.Provider.Name) + profile := sweep.Providers["openai"] assert.Equal(peoplesweep.ProtocolOpenAIChat, profile.Protocol) assert.Equal(setupOpenAIEndpoint, profile.Endpoint) - assert.Equal(setupInferenceModel, profile.Model) - assert.Equal(setupInferenceReasoning, profile.ReasoningEffort) + assert.Equal("gpt-5.6-luna", profile.Model) + assert.Equal("openai", profile.PresetID) + assert.Empty(profile.ReasoningEffort) assert.Equal(peoplesweep.CredentialEnv, profile.Credential) assert.Equal(setupOpenAIKeyEnv, profile.CredentialEnv) assert.Equal("2025-01-01", profile.SourceSince) @@ -1071,7 +1110,195 @@ func TestSetupProvidersOpenAIFallbackOnboardsInference(t *testing.T) { inference := findLane(t, report, lanePeopleInference) assert.Equal(laneStateOn, inference.State) assert.Equal(consentActive, inference.Consent) - assert.Equal(setupInferenceModel, inference.Model) + assert.Equal("gpt-5.6-luna", inference.Model) +} + +func TestSetupProvidersSelectsExplicitHTTPPresets(t *testing.T) { + for _, test := range []struct { + provider, endpoint, credentialEnv string + }{ + {provider: "openrouter", endpoint: "https://openrouter.ai/api/v1", credentialEnv: "OPENROUTER_KEY"}, + {provider: "venice", endpoint: "https://api.venice.ai/api/v1", credentialEnv: "VENICE_KEY"}, + } { + t.Run(test.provider, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) + fixture.env[test.credentialEnv] = setupProvidersTestKey + model := test.provider + "/model" + output, err := fixture.run(t, "providers", "--provider", test.provider, "--model", model, + "--credential-env", test.credentialEnv, "--retention-posture", "operator_asserted", + "--training-posture", "operator_asserted", "--allow-sensitive", "--yes") + require.NoError(err, output) + sweep := fixture.load(t).People.Sweep + require.True(sweep.Enabled) + provider := sweep.Providers[test.provider] + assert.Equal(test.provider, provider.PresetID) + assert.Equal(test.endpoint, provider.Endpoint) + assert.Equal(model, provider.Model) + assert.Equal(test.credentialEnv, provider.CredentialEnv) + assert.Equal("operator_asserted", provider.RetentionPosture) + }) + } +} + +func TestSetupProvidersDoesNotInferPeopleProviderFromEmbeddingKey(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) + fixture.env[setupOpenAIKeyEnv] = setupProvidersTestKey + output, err := fixture.run(t, "providers", "--allow-sensitive", "--yes") + require.NoError(err, output) + loaded := fixture.load(t) + assert.True(loaded.Vector.Enabled) + assert.False(loaded.People.Sweep.Enabled) + assert.NotContains(loaded.People.Sweep.Providers, "openai") + status, err := fixture.run(t, "status", "--json") + require.NoError(err, status) + var report laneReport + require.NoError(json.Unmarshal([]byte(status), &report)) + assert.Contains(findLane(t, report, lanePeopleInference).Reason, "--provider") +} + +func TestSetupStatusPeopleInferenceHintsIncludeRequiredPresetFlags(t *testing.T) { + const codexNext = "msgvault person provider enroll-codex --source conversation_text --source-since --retention-posture --training-posture --allow-sensitive=" + for _, test := range []struct { + name string + openAIKey bool + wantNext string + }{ + {name: "OpenAI key present", openAIKey: true, wantNext: "msgvault setup providers --provider openai --model --credential-env OPENAI_API_KEY --retention-posture --training-posture --allow-sensitive= --yes"}, + {name: "no key", wantNext: "msgvault setup providers --provider --model --credential-env --retention-posture --training-posture --allow-sensitive= --yes"}, + } { + t.Run(test.name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) + if test.openAIKey { + fixture.env[setupOpenAIKeyEnv] = setupProvidersTestKey + } + output, err := fixture.run(t, "status", "--json") + require.NoError(err, output) + var report laneReport + require.NoError(json.Unmarshal([]byte(output), &report)) + lane := findLane(t, report, lanePeopleInference) + assert.Equal([]string{test.wantNext, codexNext}, lane.Next) + human, err := fixture.run(t, "status") + require.NoError(err, human) + assert.Contains(human, "next: "+codexNext) + }) + } +} + +func TestSetupProvidersNoSelectionPlanShowsCompletePresetCommand(t *testing.T) { + const codexNext = "msgvault person provider enroll-codex --source conversation_text --source-since --retention-posture --training-posture --allow-sensitive=" + for _, test := range []struct { + name string + openAIKey bool + wantNext string + }{ + {name: "OpenAI key present", openAIKey: true, wantNext: "msgvault setup providers --provider openai --model --credential-env OPENAI_API_KEY --retention-posture --training-posture --allow-sensitive= --yes"}, + {name: "no key", wantNext: "msgvault setup providers --provider --model --credential-env --retention-posture --training-posture --allow-sensitive= --yes"}, + } { + t.Run(test.name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) + if test.openAIKey { + fixture.env[setupOpenAIKeyEnv] = setupProvidersTestKey + } + output, err := fixture.run(t, "providers", "--dry-run", "--json") + require.NoError(err, output) + var result setupProvidersOutput + require.NoError(json.Unmarshal([]byte(output), &result)) + var inference setupLanePlan + for _, lane := range result.Plan { + if lane.Lane == lanePeopleInference { + inference = lane + break + } + } + require.Equal(lanePeopleInference, inference.Lane) + assert.Contains(inference.Reason, test.wantNext) + assert.Contains(inference.Reason, codexNext) + assert.False(result.Applied) + human, err := fixture.run(t, "providers", "--dry-run") + require.NoError(err, human) + assert.Contains(human, codexNext) + }) + } +} + +func TestSetupProvidersHelpListsCodexEnrollmentWithoutProviderFlag(t *testing.T) { + assert := assert.New(t) + + command := newSetupProvidersCommand(setupProvidersDeps{}) + help := command.Long + assert.Contains(help, "msgvault person provider enroll-codex") + assert.Contains(help, "requires a terminal") + assert.NotContains(help, "--provider codex") +} + +func TestSetupProvidersNamedPresetRequiresModel(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) + fixture.env["VENICE_KEY"] = setupProvidersTestKey + _, err := fixture.run(t, "providers", "--provider", "venice", "--credential-env", "VENICE_KEY", + "--retention-posture", "operator_asserted", "--training-posture", "operator_asserted", "--allow-sensitive", "--yes") + require.ErrorContains(err, "--model") + assert.NotContains(fixture.readConfig(t), "preset_id") + status, err := fixture.run(t, "status", "--json") + require.NoError(err, status) + var report laneReport + require.NoError(json.Unmarshal([]byte(status), &report)) + assert.Contains(findLane(t, report, lanePeopleInference).Reason, "--provider") +} + +func TestSetupProvidersRejectsKeySourceWithoutProviderChoice(t *testing.T) { + require := require.New(t) + + base := setupProvidersOptions{ + documentRetention: documentindex.RetentionStandard, + documentTraining: documentindex.TrainingDefaultOptOut, + retentionPosture: setupPostureDeclared, trainingPosture: setupPostureDeclared, + } + for _, change := range []func(*setupProvidersOptions){ + func(o *setupProvidersOptions) { o.model = "explicit/model" }, + func(o *setupProvidersOptions) { o.credentialEnv = "EXACT_KEY" }, + func(o *setupProvidersOptions) { o.apiKeyStdin = true }, + } { + options := base + change(&options) + require.ErrorContains(validateSetupProvidersOptions(options), "--provider") + } +} + +func TestSetupProvidersExplicitFalseSensitiveDecision(t *testing.T) { + for _, preset := range []string{"openai", "venice"} { + t.Run(preset, func(t *testing.T) { + require := require.New(t) + + assert := assert.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) + args := []string{"providers", "--yes", "--allow-sensitive=false"} + if preset == "openai" { + fixture.env[setupOpenAIKeyEnv] = setupProvidersTestKey + args = append(args, explicitOpenAISetupFlags()...) + } else { + fixture.env["VENICE_KEY"] = setupProvidersTestKey + args = append(args, "--provider", "venice", "--model", "venice/model", + "--credential-env", "VENICE_KEY", "--retention-posture", "operator_asserted", + "--training-posture", "operator_asserted") + } + output, err := fixture.run(t, args...) + require.NoError(err, output) + provider := fixture.load(t).People.Sweep.Providers[preset] + assert.False(provider.AllowSensitive) + assert.Contains(output, "sensitive archive excerpts are excluded") + assert.NotContains(output, "--allow-sensitive authorizes") + }) + } } func TestSetupProvidersLocalOllamaFallback(t *testing.T) { @@ -1162,7 +1389,7 @@ func TestSetupProvidersDisclosureListsInferenceSources(t *testing.T) { fixture.env[setupOpenAIKeyEnv] = setupProvidersTestKey fixture.env["MISTRAL_API_KEY"] = setupProvidersTestKey - output, err := fixture.run(t, "providers", "--dry-run", "--allow-sensitive") + output, err := fixture.run(t, append([]string{"providers", "--dry-run", "--allow-sensitive"}, explicitOpenAISetupFlags()...)...) require.NoError(err, output) assert.Contains(output, "bounded evidence packets of conversation_text, meeting_text, document_text for tracked people") assert.Contains(output, "--allow-sensitive authorizes sending sensitive archive excerpts to OpenAI") @@ -1171,11 +1398,13 @@ func TestSetupProvidersDisclosureListsInferenceSources(t *testing.T) { func TestSetupProvidersDeclinedDocumentsUpdateDependentLanes(t *testing.T) { for _, local := range []bool{false, true} { t.Run(fmt.Sprint("local=", local), func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) fixture.env["MISTRAL_API_KEY"] = setupProvidersTestKey fixture.tty = true - profileName := setupInferenceProfile + profileName := "openai" if local { fixture.ollama = ollamaProbeResult{Reachable: true, Models: []string{"nomic-embed-text:latest", "gpt-oss-128k:latest"}} fixture.input = strings.NewReader("n\n") @@ -1185,8 +1414,12 @@ func TestSetupProvidersDeclinedDocumentsUpdateDependentLanes(t *testing.T) { fixture.input = strings.NewReader("n\ny\n") } - output, err := fixture.run(t, "providers", "--allow-sensitive") - require.NoError(t, err, output) + args := []string{"providers", "--allow-sensitive"} + if !local { + args = append(args, explicitOpenAISetupFlags()...) + } + output, err := fixture.run(t, args...) + require.NoError(err, output) loaded := fixture.load(t) assert.False(loaded.Attachments.Documents.Enabled) assert.False(loaded.Attachments.Documents.Index.Embeddings.Enabled) @@ -1235,7 +1468,7 @@ func TestSetupProvidersFailureRestoresConfig(t *testing.T) { return provider } command := newSetupProvidersCommand(deps) - command.SetArgs([]string{"--yes", "--allow-sensitive"}) + command.SetArgs(append([]string{"--yes", "--allow-sensitive"}, explicitOpenAISetupFlags()...)) command.SetOut(io.Discard) command.SetErr(io.Discard) @@ -1247,7 +1480,7 @@ func TestSetupProvidersFailureRestoresConfig(t *testing.T) { require.ErrorIs(err, os.ErrNotExist) } fixture.checker.err = nil - output, err := fixture.run(t, "providers", "--yes", "--allow-sensitive") + output, err := fixture.run(t, append([]string{"providers", "--yes", "--allow-sensitive"}, explicitOpenAISetupFlags()...)...) require.NoError(err, output) assert.True(fixture.load(t).People.Sweep.Enabled) }) @@ -1256,6 +1489,9 @@ func TestSetupProvidersFailureRestoresConfig(t *testing.T) { } func TestSetupProvidersRollbackPreservesConcurrentConfig(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) fixture.env[setupOpenAIKeyEnv] = setupProvidersTestKey deps := fixture.deps(t) @@ -1263,28 +1499,28 @@ func TestSetupProvidersRollbackPreservesConcurrentConfig(t *testing.T) { var concurrent config.ConfigFile deps.personProvider = func() personProviderCommandDeps { provider := fixture.personProviderDeps(t) - provider.newChecker = func(peoplesweep.Config, personProviderStore) (personProviderChecker, error) { + provider.newChecker = func(peoplesweep.Config, personProviderStore, personProviderSetupDeps) (personProviderChecker, error) { return callbackPersonProviderChecker(func(context.Context) (peoplesweep.StructuredResponse, error) { before, err := config.ReadConfigFile(fixture.path) - require.NoError(t, err) + require.NoError(err) concurrent, err = config.EditConfigTables(fixture.path, before.ETag, []config.TableEdit{{ Path: []string{"activity"}, Values: map[string]any{"schedule": "0 * * * *"}, }}) - require.NoError(t, err) + require.NoError(err) return peoplesweep.StructuredResponse{}, failure }), nil } return provider } command := newSetupProvidersCommand(deps) - command.SetArgs([]string{"--yes", "--allow-sensitive"}) + command.SetArgs(append([]string{"--yes", "--allow-sensitive"}, explicitOpenAISetupFlags()...)) command.SetOut(io.Discard) command.SetErr(io.Discard) err := command.ExecuteContext(t.Context()) - require.ErrorIs(t, err, failure) - require.ErrorIs(t, err, config.ErrConfigConflict) - assert.Equal(t, string(concurrent.Content), fixture.readConfig(t)) + require.ErrorIs(err, failure) + require.ErrorIs(err, config.ErrConfigConflict) + assert.Equal(string(concurrent.Content), fixture.readConfig(t)) } func TestSetupProvidersWithoutProvidersReportsEveryLaneOff(t *testing.T) { @@ -1324,14 +1560,16 @@ func TestSetupProvidersDryRunWritesNothing(t *testing.T) { } func TestSetupProvidersRequiresConsentWithoutTerminal(t *testing.T) { + require := require.New(t) + assert := assert.New(t) fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) fixture.env[setupVoyageKeyEnv] = setupProvidersTestKey before := fixture.readConfig(t) _, err := fixture.run(t, "providers") - require.ErrorContains(t, err, "--yes") - require.ErrorContains(t, err, gateVoyage) + require.ErrorContains(err, "--yes") + require.ErrorContains(err, gateVoyage) assert.Equal(before, fixture.readConfig(t)) } @@ -1375,7 +1613,7 @@ dimension = 3072 fixture.env[setupVoyageKeyEnv] = setupProvidersTestKey fixture.env[setupOpenAIKeyEnv] = setupProvidersTestKey - output, err := fixture.run(t, "providers", "--yes", "--allow-sensitive") + output, err := fixture.run(t, append([]string{"providers", "--yes", "--allow-sensitive"}, explicitOpenAISetupFlags()...)...) require.NoError(err, output) loaded := fixture.load(t) @@ -1389,6 +1627,9 @@ dimension = 3072 } func TestSetupProvidersRefusesConfiguredRemote(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) fixture.env[setupVoyageKeyEnv] = setupProvidersTestKey deps := fixture.deps(t) @@ -1400,8 +1641,8 @@ func TestSetupProvidersRefusesConfiguredRemote(t *testing.T) { root.SetErr(io.Discard) err := root.ExecuteContext(t.Context()) - require.ErrorContains(t, err, "remote daemon") - assert.Equal(t, strings.ReplaceAll(setupProvidersMinimalConfig, "{{DIR}}", filepath.ToSlash(fixture.dir)), fixture.readConfig(t)) + require.ErrorContains(err, "remote daemon") + assert.Equal(strings.ReplaceAll(setupProvidersMinimalConfig, "{{DIR}}", filepath.ToSlash(fixture.dir)), fixture.readConfig(t)) } func TestSetupStatusReportsPendingLanesForPresentKeys(t *testing.T) { @@ -1439,11 +1680,29 @@ func TestSetupStatusReportsPendingLanesForPresentKeys(t *testing.T) { func findLane(t *testing.T, report laneReport, lane string) laneStatus { t.Helper() + require := require.New(t) for _, item := range report.Lanes { if item.Lane == lane { return item } } - require.Failf(t, "lane missing", "lane %q not in report", lane) + require.Failf("lane missing", "lane %q not in report", lane) return laneStatus{} } + +func TestSetupProvidersRejectsExplicitProviderWhenSweepEnabled(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + fixture := newSetupProvidersFixture(t, setupProvidersMinimalConfig) + fixture.env[setupOpenAIKeyEnv] = setupProvidersTestKey + args := append([]string{"providers", "--yes", "--allow-sensitive"}, explicitOpenAISetupFlags()...) + _, err := fixture.run(t, args...) + require.NoError(err) + before, err := os.ReadFile(fixture.path) + require.NoError(err) + _, err = fixture.run(t, "providers", "--provider", "venice", "--model", "venice/model", "--credential-env", "VENICE_KEY", "--retention-posture", "operator_asserted", "--training-posture", "operator_asserted", "--allow-sensitive", "--yes") + require.ErrorContains(err, "people sweep is already enabled") + after, err := os.ReadFile(fixture.path) + require.NoError(err) + assert.Equal(before, after) +} diff --git a/cmd/msgvault/cmd/setup_status.go b/cmd/msgvault/cmd/setup_status.go index fe27947f6..6e374675e 100644 --- a/cmd/msgvault/cmd/setup_status.go +++ b/cmd/msgvault/cmd/setup_status.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "errors" "os" @@ -12,13 +13,33 @@ import ( // setupStatusDeps isolates the report from the process so tests can drive it // from a temp config and a fixed environment. type setupStatusDeps struct { + bind func(context.Context) setupStatusDeps config func() *config.Config environment func(*cobra.Command, *config.Config) setupEnvironment } func defaultSetupStatusDeps() setupStatusDeps { return setupStatusDeps{ - config: func() *config.Config { return cfg }, + bind: func(ctx context.Context) setupStatusDeps { + state := invocationFromContext(ctx) + var currentCfg *config.Config + if state != nil && state.cfg != nil { + currentCfg = state.cfg + } + return setupStatusDeps{ + config: func() *config.Config { return currentCfg }, + environment: func(command *cobra.Command, loaded *config.Config) setupEnvironment { + credentials, _ := providercredentials.Read(loaded.TokensDir()) + return setupEnvironment{ + lookupEnv: os.LookupEnv, + fileExists: defaultFileExists, + consent: readSetupConsentState(command.Context(), loaded), + credentials: credentials, + } + }, + } + }, + config: func() *config.Config { return nil }, environment: func(command *cobra.Command, loaded *config.Config) setupEnvironment { // Read retains any load error in the snapshot so each lane can report it. credentials, _ := providercredentials.Read(loaded.TokensDir()) @@ -47,11 +68,18 @@ The report reads config.toml, the process environment, stored provider credentials, and the local archive's consent records. It never contacts a provider.`, Args: cobra.NoArgs, RunE: func(command *cobra.Command, _ []string) error { - loaded := deps.config() + bound := deps + if invocationFromContext(command.Context()) != nil && bound.bind != nil { + bound = bound.bind(command.Context()) + } + loaded := bound.config() + if inv := invocationFromCommand(command); inv != nil && inv.cfg != nil { + loaded = inv.cfg + } if loaded == nil { return errors.New("configuration is unavailable") } - report := buildLaneReport(loaded, deps.environment(command, loaded)) + report := buildLaneReport(loaded, bound.environment(command, loaded)) return writeLaneReport(command.OutOrStdout(), report, jsonOutput) }, } diff --git a/cmd/msgvault/cmd/setup_test.go b/cmd/msgvault/cmd/setup_test.go index b421bc249..18c4527cf 100644 --- a/cmd/msgvault/cmd/setup_test.go +++ b/cmd/msgvault/cmd/setup_test.go @@ -1,13 +1,18 @@ package cmd import ( + "bytes" "os" "path/filepath" "runtime" + "strings" "testing" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/config" ) func TestCreateNASBundle(t *testing.T) { @@ -84,6 +89,106 @@ func TestCreateNASBundle_NoSecrets(t *testing.T) { // client_secret.json should NOT exist (no source path given) _, err = os.Stat(filepath.Join(bundleDir, "client_secret.json")) assert.True(os.IsNotExist(err), "client_secret.json should not exist when no secrets path given") + + // config.toml must not point at a credential the bundle does not hold + cfgData, err := os.ReadFile(filepath.Join(bundleDir, "config.toml")) + require.NoError(err, "read config.toml") + assert.NotContains(string(cfgData), "[oauth]") +} + +// runSetupForTest runs the wizard against a fresh home directory with +// the given answers on stdin and returns the home directory and output. +func runSetupForTest(t *testing.T, answers string) (string, string) { + t.Helper() + home := t.TempDir() + c := config.NewDefaultConfig() + c.HomeDir = home + c.Data.DataDir = home + testCtx := withTestConfig(t, c) + _ = testCtx + + var out bytes.Buffer + cmd := &cobra.Command{} + cmd.SetContext(testCtx) + cmd.SetIn(strings.NewReader(answers)) + cmd.SetOut(&out) + require.NoError(t, runSetup(cmd, nil)) + return home, out.String() +} + +func TestSetupWithoutGoogleCredentials(t *testing.T) { + // Enter at the credential prompt, then "n" for the remote server. + home, out := runSetupForTest(t, "\nn\n") + + assert.NotContains(t, out, "add-account") + assert.Contains(t, out, "add-imap") + _, err := os.Stat(filepath.Join(home, "nas-bundle")) + assert.True(t, os.IsNotExist(err), "no NAS bundle without a remote") +} + +func TestSetupDoesNotPersistHTTPForRuntimeRemoteURL(t *testing.T) { //nolint:paralleltest // process environment + require := require.New(t) + assert := assert.New(t) + home := t.TempDir() + cfg := config.NewDefaultConfig() + cfg.HomeDir = home + cfg.Data.DataDir = home + cfg.Remote.URL = "https://configured.example.test:8080" + require.NoError(cfg.Save()) + + t.Setenv("MSGVAULT_REMOTE_URL", "http://runtime.example.test:8080") + runtimeConfig, err := config.Load("", home) + require.NoError(err) + cmd := &cobra.Command{} + cmd.SetContext(withTestConfig(t, runtimeConfig)) + cmd.SetIn(strings.NewReader("\ny\n")) + var output bytes.Buffer + cmd.SetOut(&output) + require.NoError(runSetup(cmd, nil)) + + persisted, err := os.ReadFile(runtimeConfig.ConfigFilePath()) + require.NoError(err) + assert.Contains(string(persisted), "https://configured.example.test:8080") + assert.NotContains(string(persisted), "http://runtime.example.test:8080") + assert.NotContains(string(persisted), "allow_insecure = true") +} + +func TestSetupPersistsExplicitRemoteMatchingEnvironment(t *testing.T) { //nolint:paralleltest // process environment + require := require.New(t) + assert := assert.New(t) + home := t.TempDir() + path := filepath.Join(home, "config.toml") + require.NoError(os.WriteFile(path, []byte("[remote]\nurl = 'https://old.example.test'\n"), 0o600)) + t.Setenv("MSGVAULT_REMOTE_URL", "http://archive.example.test:8080") + t.Setenv("MSGVAULT_REMOTE_ALLOW_INSECURE", "true") + cfg, err := config.Load(path, home) + require.NoError(err) + cmd := &cobra.Command{} + cmd.SetContext(withTestConfig(t, cfg)) + cmd.SetIn(strings.NewReader("\nn\ny\narchive.example.test\n8080\n")) + cmd.SetOut(&bytes.Buffer{}) + require.NoError(runSetup(cmd, nil)) + snapshot, err := config.ReadConfigFile(path) + require.NoError(err) + saved, err := config.LoadConfigFile(snapshot, home) + require.NoError(err) + assert.Equal("http://archive.example.test:8080", saved.Remote.URL) + assert.True(saved.Remote.AllowInsecure) + assert.NotEmpty(saved.Remote.APIKey) +} + +func TestSetupWithGoogleCredentialsPrintsGmailSteps(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + secretsPath := filepath.Join(t.TempDir(), "client_secret.json") + require.NoError(os.WriteFile(secretsPath, []byte(`{"installed":{}}`), 0600)) + + home, out := runSetupForTest(t, secretsPath+"\nn\n") + + assert.Contains(out, "add-account") + data, err := os.ReadFile(filepath.Join(home, "config.toml")) + require.NoError(err, "setup should save config.toml") + assert.Contains(string(data), "client_secret.json") } func TestCreateNASBundle_CopiesSecrets(t *testing.T) { @@ -130,3 +235,64 @@ func TestGenerateAPIKey(t *testing.T) { require.NoError(err, "generateAPIKey") assert.NotEqual(key1, key2, "generateAPIKey should return unique keys") } + +func TestSetupAddAccountCommand(t *testing.T) { + tests := []struct { + name string + oauth config.OAuthConfig + want string + }{ + {"none", config.OAuthConfig{}, ""}, + {"default secrets", config.OAuthConfig{ClientSecrets: "/c.json"}, "msgvault add-account you@gmail.com"}, + {"service account", config.OAuthConfig{ServiceAccountKey: "/sa.json"}, "msgvault add-account you@gmail.com"}, + { + "named apps only", + config.OAuthConfig{Apps: map[string]config.OAuthApp{ + "work": {ClientSecrets: "/w.json"}, + "empty": {}, + }}, + "msgvault add-account you@gmail.com --oauth-app 'work'", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, setupAddAccountCommand(&tt.oauth)) + }) + } +} + +func TestPrintSetupNextStepsExportNeedsBundledSecrets(t *testing.T) { + assert := assert.New(t) + const named = "msgvault add-account you@gmail.com --oauth-app 'work'" + + var withSecrets bytes.Buffer + printSetupNextSteps(&withSecrets, "msgvault add-account you@gmail.com", true, true) + assert.Contains(withSecrets.String(), "msgvault export-token") + + var namedOnly bytes.Buffer + printSetupNextSteps(&namedOnly, named, true, false) + assert.NotContains(namedOnly.String(), "msgvault export-token") + assert.Contains(namedOnly.String(), "cannot be exported to the NAS") +} + +func TestCreateNASBundle_RebuildWithoutSecretsRemovesOldCopy(t *testing.T) { + require := require.New(t) + secretsPath := filepath.Join(t.TempDir(), "client_secret.json") + require.NoError(os.WriteFile(secretsPath, []byte(`{"installed":{}}`), 0600)) + bundleDir := filepath.Join(t.TempDir(), "nas-bundle") + + require.NoError(createNASBundle(bundleDir, "key", secretsPath, 8080)) + require.NoError(createNASBundle(bundleDir, "key", "", 8080)) + + _, err := os.Stat(filepath.Join(bundleDir, "client_secret.json")) + assert.True(t, os.IsNotExist(err), "rebuild without secrets should remove the old copy") +} + +func TestPrintSetupNextStepsOmitsLocalImportForRemote(t *testing.T) { + var local, remote bytes.Buffer + printSetupNextSteps(&local, "", false, false) + printSetupNextSteps(&remote, "", true, false) + + assert.Contains(t, local.String(), "import-mbox") + assert.NotContains(t, remote.String(), "import-mbox") +} diff --git a/cmd/msgvault/cmd/show_message_test.go b/cmd/msgvault/cmd/show_message_test.go index 15692acd8..f46a927c1 100644 --- a/cmd/msgvault/cmd/show_message_test.go +++ b/cmd/msgvault/cmd/show_message_test.go @@ -68,6 +68,9 @@ func TestOutputMessageJSONShowsDeletedFromSourceOnlyWhenPresent(t *testing.T) { } func TestShowMessageUsesLocalDaemonHTTPAndPreservesTextOutput(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) dataDir := t.TempDir() @@ -87,11 +90,15 @@ func TestShowMessageUsesLocalDaemonHTTPAndPreservesTextOutput(t *testing.T) { HomeDir: dataDir, Data: config.DataConfig{DataDir: dataDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true showMessageJSON = false done := captureStdout(t) cmd := &cobra.Command{Use: "show-message", RunE: showMessageCmd.RunE, Args: showMessageCmd.Args} + cmd.SetContext(testCtx) cmd.SetArgs([]string{"remote-42"}) err := cmd.Execute() @@ -107,6 +114,9 @@ func TestShowMessageUsesLocalDaemonHTTPAndPreservesTextOutput(t *testing.T) { } func TestShowMessageHTTPNotFoundPreservesCLIError(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) dataDir := t.TempDir() @@ -126,11 +136,15 @@ func TestShowMessageHTTPNotFoundPreservesCLIError(t *testing.T) { HomeDir: dataDir, Data: config.DataConfig{DataDir: dataDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true showMessageJSON = false done := captureStdout(t) cmd := &cobra.Command{Use: "show-message", RunE: showMessageCmd.RunE, Args: showMessageCmd.Args} + cmd.SetContext(testCtx) cmd.SetArgs([]string{"missing"}) err := cmd.Execute() @@ -258,6 +272,9 @@ func TestOutputMessageLabelsSanitizedOnlyForText(t *testing.T) { } func TestShowMessageJSONPreservesRFCMessageIDFromDaemon(t *testing.T) { + cfg := testConfigValue() + useLocal := false + assert, require := assert.New(t), require.New(t) dataDir := t.TempDir() server, _ := messageHTTPDaemon(t) @@ -265,9 +282,12 @@ func TestShowMessageJSONPreservesRFCMessageIDFromDaemon(t *testing.T) { oldCfg, oldLocal, oldJSON := cfg, useLocal, showMessageJSON t.Cleanup(func() { cfg, useLocal, showMessageJSON = oldCfg, oldLocal, oldJSON }) cfg = &config.Config{HomeDir: dataDir, Data: config.DataConfig{DataDir: dataDir}} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal, showMessageJSON = true, true done := captureStdout(t) cmd := &cobra.Command{Use: "show-message", RunE: showMessageCmd.RunE, Args: showMessageCmd.Args} + cmd.SetContext(testCtx) cmd.SetArgs([]string{"remote-42"}) err := cmd.Execute() output := done() diff --git a/cmd/msgvault/cmd/sqlite_maintenance.go b/cmd/msgvault/cmd/sqlite_maintenance.go new file mode 100644 index 000000000..9a66616a5 --- /dev/null +++ b/cmd/msgvault/cmd/sqlite_maintenance.go @@ -0,0 +1,30 @@ +package cmd + +import ( + "context" + + "go.kenn.io/msgvault/internal/scheduler" + "go.kenn.io/msgvault/internal/store" +) + +// The daily SQLite maintenance job refreshes planner statistics and truncates +// the WAL off-peak, instead of after every sync, where a loaded archive keeps +// both from finishing. +const ( + sqliteMaintenanceJob = "sqlite-maintenance" + sqliteMaintenanceCron = "29 4 * * *" +) + +func registerSQLiteMaintenanceJob(sched *scheduler.Scheduler, s *store.Store) error { + if s.IsPostgreSQL() { + return nil + } + return sched.AddJob(scheduler.Job{ + Name: sqliteMaintenanceJob, + Schedule: sqliteMaintenanceCron, + Run: func(ctx context.Context) error { + _, err := s.RunDailyMaintenance(ctx) + return err + }, + }) +} diff --git a/cmd/msgvault/cmd/sqlite_maintenance_test.go b/cmd/msgvault/cmd/sqlite_maintenance_test.go new file mode 100644 index 000000000..466ba29f6 --- /dev/null +++ b/cmd/msgvault/cmd/sqlite_maintenance_test.go @@ -0,0 +1,34 @@ +package cmd + +import ( + "context" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/msgvault/internal/scheduler" + "go.kenn.io/msgvault/internal/store" +) + +func TestRegisterSQLiteMaintenanceJobRunsDailyMaintenance(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + s, err := store.Open(filepath.Join(t.TempDir(), "archive.db")) + require.NoError(err) + t.Cleanup(func() { _ = s.Close() }) + require.NoError(s.InitSchema()) + + sched := scheduler.New(func(context.Context, string) error { return nil }) + t.Cleanup(func() { <-sched.Stop().Done() }) + require.NoError(registerSQLiteMaintenanceJob(sched, s)) + require.True(sched.IsJobScheduled(sqliteMaintenanceJob)) + + require.NoError(sched.TriggerJob(sqliteMaintenanceJob)) + jobs := sched.JobStatus() + require.Len(jobs, 1) + assert.Equal("29 4 * * *", jobs[0].Schedule) + assert.Empty(jobs[0].LastError) + assert.False(jobs[0].LastRun.IsZero(), "maintenance ran") +} diff --git a/cmd/msgvault/cmd/stage_delete.go b/cmd/msgvault/cmd/stage_delete.go index 73ae2b735..a96704e69 100644 --- a/cmd/msgvault/cmd/stage_delete.go +++ b/cmd/msgvault/cmd/stage_delete.go @@ -28,7 +28,7 @@ func newStageDeleteCommand() *cobra.Command { Long: `Stage the deletable messages matching a search query for deletion. The search runs with the same semantics as msgvault search. Matches that no -source supports deleting, such as chats, meetings, and non-Gmail mail, are +source supports deleting, such as chats, meetings, and mail from IMAP or file imports, are reported and skipped rather than rejecting the whole search. Use --dry-run to see the same staged subset and counts without creating a batch. Alternatively, pass a comma-separated --ids list to stage explicit internal @@ -324,7 +324,7 @@ func writeStageDeleteSkipped(w io.Writer, result *generated.StageDeletionRespons matched = *result.MatchedCount } if _, err := fmt.Fprintf(w, - "%d of %d matching item(s) cannot be deleted from their source (chats, meetings, or non-Gmail mail) and were skipped.\n", + "%d of %d matching item(s) cannot be deleted from their source (chats, meetings, or mail from other sources) and were skipped.\n", skipped, matched); err != nil { return fmt.Errorf("write skipped summary: %w", err) } @@ -344,8 +344,8 @@ func stageDeleteDaemonErr(op string, err error) error { "or run 'msgvault build-cache' and rerun stage-delete", op, apiErr.Message) case "selection_not_deletable": return fmt.Errorf("%s: %s; nothing the search matched can be deleted from its "+ - "source. Deletion currently covers Gmail mail only, so widen or retarget the "+ - "search, for example with --source-id ", + "source. Deletion currently covers Gmail and Microsoft Graph mail only, so widen or retarget the "+ + "search, for example with --source-id ", op, apiErr.Message) case "multi_account_selection": return fmt.Errorf("%s: %s; rerun stage-delete once per source with --source-id", diff --git a/cmd/msgvault/cmd/stage_delete_test.go b/cmd/msgvault/cmd/stage_delete_test.go index 8be77f921..3a1098cac 100644 --- a/cmd/msgvault/cmd/stage_delete_test.go +++ b/cmd/msgvault/cmd/stage_delete_test.go @@ -2,6 +2,7 @@ package cmd import ( "bytes" + "context" _ "embed" "encoding/json" "net/http" @@ -41,7 +42,7 @@ func issue769CommandArgs(t *testing.T, dryRun bool) []string { return nil } -func newRegisteredStageDeleteTestRoot(t *testing.T) *cobra.Command { +func newRegisteredStageDeleteTestRoot(t *testing.T, contexts ...context.Context) *cobra.Command { t.Helper() registered, _, err := rootCmd.Find([]string{"stage-delete"}) require.NoError(t, err, "find registered stage-delete command") @@ -54,6 +55,10 @@ func newRegisteredStageDeleteTestRoot(t *testing.T) *cobra.Command { require.NoError(t, registered.Flags().Set("ids", "")) registered.Flags().Lookup("ids").Changed = false root := &cobra.Command{Use: "msgvault"} + if len(contexts) > 0 { + root.SetContext(contexts[0]) + registered.SetContext(contexts[0]) + } root.AddCommand(registered) return root } @@ -123,12 +128,12 @@ func TestStageDeleteCommand(t *testing.T) { wantSource = []int64{tt.wantSource} } server, routes := newStageDeleteTestServer(t, tt.wantQuery, tt.dryRun, tt.status, wantSource...) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) var stdout bytes.Buffer - root := newRegisteredStageDeleteTestRoot(t) + root := newRegisteredStageDeleteTestRoot(t, testCtx) root.SetOut(&stdout) args := append([]string{"stage-delete"}, tt.query...) args = append(args, tt.flags...) @@ -143,12 +148,12 @@ func TestStageDeleteCommand(t *testing.T) { t.Run("configured_remote_routing", func(t *testing.T) { server, routes := newStageDeleteTestServer(t, "from:bob@example.com", false, http.StatusCreated) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) var stdout bytes.Buffer - root := newRegisteredStageDeleteTestRoot(t) + root := newRegisteredStageDeleteTestRoot(t, testCtx) root.SetOut(&stdout) root.SetArgs([]string{"stage-delete", "from:bob@example.com"}) @@ -178,11 +183,11 @@ func TestStageDeleteCommand(t *testing.T) { requests++ })) defer server.Close() - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) - root := newRegisteredStageDeleteTestRoot(t) + root := newRegisteredStageDeleteTestRoot(t, testCtx) root.SetArgs(append([]string{"stage-delete"}, tt.args...)) err := root.Execute() @@ -210,11 +215,11 @@ func TestStageDeleteCommand(t *testing.T) { } })) defer server.Close() - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) - root := newRegisteredStageDeleteTestRoot(t) + root := newRegisteredStageDeleteTestRoot(t, testCtx) root.SetArgs([]string{"stage-delete", "subject:receipt"}) err := root.Execute() @@ -235,14 +240,14 @@ func TestStageDeleteCommand(t *testing.T) { name: "staged", status: http.StatusCreated, wantOutput: "Preflight: 3 matching item(s); 2 message(s) can be staged; 1 item(s) will be skipped.\n" + "Staged 2 message(s) for deletion in batch batch-191.\n" + - "1 of 3 matching item(s) cannot be deleted from their source (chats, meetings, or non-Gmail mail) and were skipped.\n" + + "1 of 3 matching item(s) cannot be deleted from their source (chats, meetings, or mail from other sources) and were skipped.\n" + "Review with 'msgvault show-deletion batch-191', then execute with 'msgvault delete-staged batch-191'.\n", }, { name: "dry_run", dryRun: true, status: http.StatusOK, wantOutput: "Preflight: 3 matching item(s); 2 message(s) can be staged; 1 item(s) will be skipped.\n" + "Dry run: 2 message(s) would be staged; no deletion batch was created.\n" + - "1 of 3 matching item(s) cannot be deleted from their source (chats, meetings, or non-Gmail mail) and were skipped.\n", + "1 of 3 matching item(s) cannot be deleted from their source (chats, meetings, or mail from other sources) and were skipped.\n", }, } { t.Run(tt.name, func(t *testing.T) { @@ -288,12 +293,12 @@ func TestStageDeleteCommand(t *testing.T) { } })) defer server.Close() - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) var stdout bytes.Buffer - root := newRegisteredStageDeleteTestRoot(t) + root := newRegisteredStageDeleteTestRoot(t, testCtx) root.SetOut(&stdout) root.SetArgs(append([]string{"stage-delete", "subject:receipt"}, boolFlag(tt.dryRun, "--dry-run")...)) @@ -347,11 +352,11 @@ func TestStageDeleteCommand(t *testing.T) { } })) defer server.Close() - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) - root := newRegisteredStageDeleteTestRoot(t) + root := newRegisteredStageDeleteTestRoot(t, testCtx) root.SetArgs([]string{"stage-delete", "from:alice@example.com"}) err := root.Execute() @@ -380,11 +385,11 @@ func TestStageDeleteCommand(t *testing.T) { } })) defer server.Close() - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) - root := newRegisteredStageDeleteTestRoot(t) + root := newRegisteredStageDeleteTestRoot(t, testCtx) root.SetArgs([]string{"stage-delete", "subject:receipt"}) err := root.Execute() @@ -418,11 +423,11 @@ func TestStageDeleteCommand(t *testing.T) { } })) defer server.Close() - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) - root := newRegisteredStageDeleteTestRoot(t) + root := newRegisteredStageDeleteTestRoot(t, testCtx) root.SetArgs([]string{"stage-delete", tt.query}) err := root.Execute() @@ -469,11 +474,11 @@ func TestStageDeleteCommand(t *testing.T) { } else { defer server.Close() } - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) - root := newRegisteredStageDeleteTestRoot(t) + root := newRegisteredStageDeleteTestRoot(t, testCtx) root.SetArgs([]string{"stage-delete", "list:announce.example.org"}) err := root.Execute() @@ -487,11 +492,11 @@ func TestStageDeleteCommand(t *testing.T) { t.Run("newer_daemon_and_ordinary_query", func(t *testing.T) { t.Run("newer_daemon", func(t *testing.T) { server, routes, healthRequests := newStageDeleteTestServerWithSchema(t, "list:announce.example.org", false, http.StatusCreated, "2.18.0") - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) - root := newRegisteredStageDeleteTestRoot(t) + root := newRegisteredStageDeleteTestRoot(t, testCtx) root.SetArgs([]string{"stage-delete", "list:announce.example.org"}) require.NoError(t, root.Execute()) assert.Equal(t, []string{"/api/v1/cli/search", "/api/v1/explore", "/api/v1/explore/preflight", "/api/v1/deletions"}, *routes) @@ -501,11 +506,11 @@ func TestStageDeleteCommand(t *testing.T) { t.Run("ordinary_query_checks_contract", func(t *testing.T) { server, routes, healthRequests := newStageDeleteTestServerWithSchema(t, "subject:test", false, http.StatusCreated, "2.18.0") - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) - root := newRegisteredStageDeleteTestRoot(t) + root := newRegisteredStageDeleteTestRoot(t, testCtx) root.SetArgs([]string{"stage-delete", "subject:test"}) require.NoError(t, root.Execute()) assert.Equal(t, []string{"/api/v1/cli/search", "/api/v1/explore", "/api/v1/explore/preflight", "/api/v1/deletions"}, *routes) @@ -698,12 +703,12 @@ func TestStageDeleteCommandByIDs(t *testing.T) { }) })) defer server.Close() - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) var stdout bytes.Buffer - root := newRegisteredStageDeleteTestRoot(t) + root := newRegisteredStageDeleteTestRoot(t, testCtx) root.SetOut(&stdout) root.SetArgs(issue769CommandArgs(t, false)) @@ -736,12 +741,12 @@ func TestStageDeleteCommandByIDs(t *testing.T) { }) })) defer server.Close() - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) var stdout bytes.Buffer - root := newRegisteredStageDeleteTestRoot(t) + root := newRegisteredStageDeleteTestRoot(t, testCtx) root.SetOut(&stdout) root.SetArgs(issue769CommandArgs(t, true)) @@ -768,11 +773,11 @@ func TestStageDeleteCommandByIDs(t *testing.T) { }) })) defer server.Close() - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) - root := newRegisteredStageDeleteTestRoot(t) + root := newRegisteredStageDeleteTestRoot(t, testCtx) root.SetArgs([]string{"stage-delete", "--ids", "123,456"}) err := root.Execute() @@ -806,11 +811,11 @@ func TestStageDeleteCommandByIDs(t *testing.T) { requests++ })) defer server.Close() - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) - root := newRegisteredStageDeleteTestRoot(t) + root := newRegisteredStageDeleteTestRoot(t, testCtx) root.SetArgs(append([]string{"stage-delete"}, tt.args...)) err := root.Execute() diff --git a/cmd/msgvault/cmd/stats.go b/cmd/msgvault/cmd/stats.go index 541aa42d9..5d2a00322 100644 --- a/cmd/msgvault/cmd/stats.go +++ b/cmd/msgvault/cmd/stats.go @@ -1,6 +1,7 @@ package cmd import ( + "errors" "fmt" "io" @@ -25,6 +26,12 @@ Use --local to use the local daemon even when a remote is configured.`, } func runStats(cmd *cobra.Command, _ []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil || state.logger == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger out := cmd.OutOrStdout() scoped := statsAccount != "" || statsCollection != "" diff --git a/cmd/msgvault/cmd/stats_test.go b/cmd/msgvault/cmd/stats_test.go index 2ea198662..5bca183f1 100644 --- a/cmd/msgvault/cmd/stats_test.go +++ b/cmd/msgvault/cmd/stats_test.go @@ -50,6 +50,10 @@ func TestStatsCommand_AccountAndCollectionMutuallyExclusive(t *testing.T) { // SourceIDs() returned an empty slice, and GetStatsForScope treats // an empty slice as unscoped/global. func TestStatsCommand_EmptyCollectionRejected(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + useLocal := false + require := require.New(t) dataDir := t.TempDir() st := testutil.NewTestStore(t) @@ -78,8 +82,11 @@ func TestStatsCommand_EmptyCollectionRejected(t *testing.T) { Data: config.DataConfig{DataDir: dataDir}, Remote: config.RemoteConfig{URL: "http://configured-daemonclient.invalid"}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) useLocal = true + invocationFromContext(testCtx).options.useLocal = true statsCollection = "empty" testCmd := &cobra.Command{Use: "stats", RunE: statsCmd.RunE} @@ -87,6 +94,7 @@ func TestStatsCommand_EmptyCollectionRejected(t *testing.T) { testCmd.Flags().StringVar(&statsCollection, "collection", "empty", "") root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(testCmd) root.SetArgs([]string{"stats", "--collection", "empty"}) @@ -96,6 +104,10 @@ func TestStatsCommand_EmptyCollectionRejected(t *testing.T) { } func TestStatsCommand_ScopedUsesLocalDaemonHTTPAndPreservesLocalOutput(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + useLocal := false + require := require.New(t) assertions := assert.New(t) dataDir := t.TempDir() @@ -122,12 +134,14 @@ func TestStatsCommand_ScopedUsesLocalDaemonHTTPAndPreservesLocalOutput(t *testin cfg = testCfg logger = slog.New(slog.DiscardHandler) useLocal = true + testCtx := testInvocationContext(t.Context(), testCfg, invocationOptions{useLocal: true}) statsAccount = "" statsCollection = "Important" var stdout bytes.Buffer var stderr bytes.Buffer cmd := &cobra.Command{Use: "stats", RunE: runStats} + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetErr(&stderr) @@ -149,6 +163,10 @@ Note: Size is global (not scoped). } func TestStatsCommand_UnscopedUsesLocalDaemonHTTPAndPreservesLocalOutput(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + useLocal := false + require := require.New(t) assertions := assert.New(t) dataDir := t.TempDir() @@ -175,12 +193,14 @@ func TestStatsCommand_UnscopedUsesLocalDaemonHTTPAndPreservesLocalOutput(t *test cfg = testCfg logger = slog.New(slog.DiscardHandler) useLocal = true + testCtx := testInvocationContext(t.Context(), testCfg, invocationOptions{useLocal: true}) statsAccount = "" statsCollection = "" var stdout bytes.Buffer var stderr bytes.Buffer cmd := &cobra.Command{Use: "stats", RunE: runStats} + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetErr(&stderr) diff --git a/cmd/msgvault/cmd/store_resolver.go b/cmd/msgvault/cmd/store_resolver.go index 2ee0ae025..89273f3ee 100644 --- a/cmd/msgvault/cmd/store_resolver.go +++ b/cmd/msgvault/cmd/store_resolver.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "log/slog" "maps" "net/http" "os" @@ -14,7 +15,7 @@ import ( "unicode/utf8" "github.com/gofrs/flock" - "github.com/spf13/pflag" + "github.com/spf13/cobra" "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/daemonclient" "go.kenn.io/msgvault/internal/store" @@ -31,20 +32,17 @@ const ( localDaemonStartupProgressInterval = 10 * time.Second ) -// runStartupMigrations pulls legacy identity addresses from the global config -// and runs the one-time migration. If migration was performed, the notice is -// logged and printed to stderr. If the migration is deferred because no source -// exists yet, it will be retried on a later command after a source has been -// created — and ingest commands that create the first source should call -// runPostSourceCreateMigrations after GetOrCreateSource so the deferred -// migration applies on the same invocation. -// -// Always returns nil unless the migration itself errors. -func runStartupMigrations(s *store.Store) error { - addrs := cfg.Identity.Addresses - res, err := s.RunStartupMigrations(addrs) +// runStartupMigrationsContext uses the invocation's config and logger for the +// migration. Callers with a cancellable command context pass it through. +func runStartupMigrationsContext(ctx context.Context, s *store.Store, state *invocation) error { + currentCfg, currentLogger := invocationConfigLogger(state) + if currentCfg == nil { + return errors.New("configuration is unavailable") + } + addrs := currentCfg.Identity.Addresses + res, err := s.RunStartupMigrationsContext(ctx, addrs) if err != nil { - logger.Warn("startup migration failed", "error", err) + currentLogger.Warn("startup migration failed", "error", err) return err } // Success cases log at Info (the operation succeeded; res.Notice is @@ -52,11 +50,11 @@ func runStartupMigrations(s *store.Store) error { // error path above. switch { case res.Deferred: - logger.Info("legacy [identity] block in config detected (migration deferred until a source exists)", + currentLogger.Info("legacy [identity] block in config detected (migration deferred until a source exists)", "address_count", res.AddressCount, "hint", "run 'msgvault add-account ...' to create a source; the migration will retry on the next command") case res.Applied: - logger.Info("legacy identity migrated", + currentLogger.Info("legacy identity migrated", "addresses", res.AddressCount, "sources", res.SourceCount) } @@ -74,7 +72,7 @@ func runStartupMigrations(s *store.Store) error { // `len(existing) > 0` guard to skip the source's own address (regression // caught upstream at iter20). // -// All ingest paths already invoke runPostSourceCreateMigrations after +// All ingest paths already invoke runPostSourceCreateMigrationsForInvocation after // confirmDefaultIdentity, which handles the legacy migration correctly // in the deferred (no-source) case and is a no-op once the migration // sentinel is set. So this pre-source call is intentionally a no-op @@ -86,15 +84,18 @@ func runStartupMigrationsForIngest(s *store.Store) error { return nil } -// runPostSourceCreateMigrations re-runs startup migrations after the caller -// has just created a source. The legacy identity migration defers when no -// source exists at startup, so on a fresh install the very first -// add-account / add-imap / add-o365 / import-* invocation needs a second -// pass to actually apply the migration on the same invocation that created -// the first source. Subsequent calls are O(1) — once the migration sentinel -// is set, MigrateLegacyIdentityConfig short-circuits. -func runPostSourceCreateMigrations(s *store.Store) error { - return runStartupMigrations(s) +func runPostSourceCreateMigrationsForInvocation(s *store.Store, state *invocation) error { + if state == nil { + return errors.New("invocation state is required") + } + return runStartupMigrationsContext(context.Background(), s, state) +} + +func invocationConfigLogger(state *invocation) (*config.Config, *slog.Logger) { + if state != nil { + return state.cfg, state.logger + } + return nil, nil } // HTTPStoreKind identifies which HTTP endpoint a CLI command is using. @@ -107,56 +108,57 @@ const ( ) // Agent delegation flags — populated by init, consumed in openAgentDelegatedStore. -var ( - agentURL string - agentTokenFile string - agentAllowInsecure bool - agentFlags *pflag.FlagSet -) - -func init() { - agentFlags = rootCmd.PersistentFlags() - agentFlags.StringVar(&agentURL, "agent-url", "", +func registerAgentFlags(root *cobra.Command) { + flags := root.PersistentFlags() + flags.String("agent-url", "", "Daemon URL for agent-delegated mode (requires --agent-token-file)") - agentFlags.StringVar(&agentTokenFile, "agent-token-file", "", + flags.String("agent-token-file", "", "Path to a file containing the agent grant secret (requires --agent-url)") - agentFlags.BoolVar(&agentAllowInsecure, "agent-allow-insecure", false, + flags.Bool("agent-allow-insecure", false, "Allow plain HTTP for agent-delegated connections (trusted networks only)") } // isAgentMode returns true when either --agent-url or --agent-token-file is // provided, including an explicit empty value. Either flag signals a delegation // request; openAgentDelegatedStore requires both values before opening a client. -func isAgentMode() bool { - return agentURL != "" || agentTokenFile != "" || - agentFlags.Changed("agent-url") || agentFlags.Changed("agent-token-file") +func isAgentMode(state *invocation) bool { + if state != nil { + o := state.options + return o.agentURL != "" || o.agentTokenFile != "" || o.agentURLChanged || o.agentTokenChanged + } + return false } // openAgentDelegatedStore creates a daemonclient.Client authenticated with an // agent grant secret read from the file named by --agent-token-file. -func openAgentDelegatedStore(ctx context.Context) (*daemonclient.Client, HTTPStoreInfo, error) { - if agentURL == "" { +func openAgentDelegatedStore(ctx context.Context, state *invocation) (*daemonclient.Client, HTTPStoreInfo, error) { + state = invocationState(ctx, state) + if state == nil { + return nil, HTTPStoreInfo{}, errors.New("invocation state is required") + } + o := state.options + if o.agentURL == "" { return nil, HTTPStoreInfo{}, errors.New("--agent-url is required for agent-delegated mode") } - if agentTokenFile == "" { + if o.agentTokenFile == "" { return nil, HTTPStoreInfo{}, errors.New("--agent-token-file is required for agent-delegated mode") } - if useLocal { + if o.useLocal { return nil, HTTPStoreInfo{}, errors.New( "--local and --agent-url are incompatible: agent-delegated mode targets a specific remote daemon") } - raw, err := os.ReadFile(agentTokenFile) + raw, err := os.ReadFile(o.agentTokenFile) if err != nil { - return nil, HTTPStoreInfo{}, fmt.Errorf("read agent token file %q: %w", agentTokenFile, err) + return nil, HTTPStoreInfo{}, fmt.Errorf("read agent token file %q: %w", o.agentTokenFile, err) } token := strings.TrimSpace(string(raw)) if token == "" { - return nil, HTTPStoreInfo{}, fmt.Errorf("agent token file %q is empty", agentTokenFile) + return nil, HTTPStoreInfo{}, fmt.Errorf("agent token file %q is empty", o.agentTokenFile) } st, err := newDaemonCLIClient(ctx, daemonclient.Config{ - URL: agentURL, + URL: o.agentURL, AgentToken: token, - AllowInsecure: agentAllowInsecure, + AllowInsecure: o.agentAllowInsecure, }) if err != nil { return nil, HTTPStoreInfo{}, err @@ -185,7 +187,7 @@ func openAgentDelegatedStore(ctx context.Context) (*daemonclient.Client, HTTPSto } return st, HTTPStoreInfo{ Kind: HTTPStoreAgentDelegated, - URL: agentURL, + URL: o.agentURL, }, nil } @@ -205,11 +207,20 @@ type HTTPStoreInfo struct { // 1. --local flag → local daemon // 2. [remote].url set in config → configured remote daemon // 3. Default → local daemon -func IsRemoteMode() bool { - if useLocal { +func IsRemoteMode(state *invocation) bool { + return isRemoteModeFor(state) +} + +func isRemoteModeFor(state *invocation) bool { + if state == nil { + return false + } + options := state.options + currentCfg := state.cfg + if options.useLocal { return false } - return cfg != nil && cfg.Remote.URL != "" + return currentCfg != nil && currentCfg.Remote.URL != "" } // OpenHTTPStore returns the HTTP store that ordinary CLI commands should use. @@ -224,32 +235,40 @@ func openHTTPStoreWithStartupCacheIntent( ctx context.Context, intent startupCacheBuildIntent, ) (*daemonclient.Client, HTTPStoreInfo, error) { + inv := invocationFromContext(ctx) // Agent-delegated mode is checked first: it operates without a local config. - if isAgentMode() { - return openAgentDelegatedStore(ctx) + if isAgentMode(inv) { + return openAgentDelegatedStore(ctx, inv) } - if cfg == nil { + if inv == nil { + return nil, HTTPStoreInfo{}, errors.New("invocation state is required") + } + currentCfg := inv.cfg + if currentCfg == nil { return nil, HTTPStoreInfo{}, errors.New("nil config") } - if IsRemoteMode() { - st, err := openRemoteStore(ctx) + if isRemoteModeFor(inv) { + st, err := openRemoteStore(ctx, inv) if err != nil { return nil, HTTPStoreInfo{}, err } return st, HTTPStoreInfo{ Kind: HTTPStoreConfiguredRemote, - URL: cfg.Remote.URL, + URL: currentCfg.Remote.URL, }, nil } - rt, startup, err := ensureLocalDaemonRuntimeWithStartupCacheIntent(ctx, cfg, intent) + rt, startup, err := ensureLocalDaemonRuntimeWithStartupCacheIntent(ctx, currentCfg, intent) if err != nil { return nil, HTTPStoreInfo{}, err } + if err := currentCfg.ResolveServerKey(); err != nil { + return nil, HTTPStoreInfo{}, fmt.Errorf("resolve server API key after local daemon startup: %w", err) + } url := urlFromDaemonRuntime(rt) st, err := newDaemonCLIClient(ctx, daemonclient.Config{ URL: url, - APIKey: cfg.Server.APIKey, + APIKey: currentCfg.Server.AuthenticationKey(), LocalDaemonToken: rt.Record.Metadata[runtimeShutdownToken], AllowInsecure: true, }) @@ -278,11 +297,19 @@ func newDaemonCLIClient(ctx context.Context, clientConfig daemonclient.Config) ( return daemonclient.New(clientConfig) } -func openRemoteStore(ctx context.Context) (*daemonclient.Client, error) { +func openRemoteStore(ctx context.Context, state *invocation) (*daemonclient.Client, error) { + state = invocationState(ctx, state) + if state == nil || state.cfg == nil { + return nil, errors.New("invocation state is required") + } + currentCfg := state.cfg + if err := currentCfg.ResolveRemoteKey(); err != nil { + return nil, err + } st, err := newDaemonCLIClient(ctx, daemonclient.Config{ - URL: cfg.Remote.URL, - APIKey: cfg.Remote.APIKey, - AllowInsecure: cfg.Remote.AllowInsecure, + URL: currentCfg.Remote.URL, + APIKey: currentCfg.Remote.AuthenticationKey(), + AllowInsecure: currentCfg.Remote.AllowInsecure, }) if err != nil { return nil, err @@ -295,6 +322,13 @@ func openRemoteStore(ctx context.Context) (*daemonclient.Client, error) { return st, nil } +func invocationState(ctx context.Context, state *invocation) *invocation { + if state != nil { + return state + } + return invocationFromContext(ctx) +} + // remoteAPISchemaCheckEnabled gates the remote schema probe. Production code // never clears it; the CLI test package disables it in TestMain because its // stub remote daemons serve single routes without /api/v1/health, and the @@ -344,11 +378,22 @@ func ensureLocalDaemonRuntimeWithStartupCacheIntent( if c == nil { return nil, localDaemonStartupInfo{}, errors.New("nil config") } + if err := c.ResolveServerKey(); err != nil { + return nil, localDaemonStartupInfo{}, err + } + // With auto-start disabled a supervisor owns the daemon lifecycle, and + // replacing a daemon means starting its successor, so reuse any compatible + // daemon that is running instead of restarting it. + autoStart := c.Server.DaemonAutoStartEnabled() + restartPolicy := c.Server.DaemonAutoRestart + if !autoStart { + restartPolicy = config.DaemonAutoRestartNever + } if err := os.MkdirAll(c.Data.DataDir, 0o700); err != nil { return nil, localDaemonStartupInfo{}, fmt.Errorf("create data directory: %w", err) } if rt := findDaemonRuntime(c.Data.DataDir); rt != nil && - !shouldUpgradeDaemonRuntimeWithPolicy(rt, Version, c.Server.DaemonAutoRestart) { + !shouldUpgradeDaemonRuntimeWithPolicy(rt, Version, restartPolicy) { if err := probeLocalDaemonAuth(ctx, rt, c); err != nil { return nil, localDaemonStartupInfo{}, err } @@ -385,7 +430,7 @@ func ensureLocalDaemonRuntimeWithStartupCacheIntent( "Another msgvault daemon start is in progress; waiting up to %s for readiness.\n", compactDuration(localDaemonAutoStartReadyTimeout)) rt, acquiredLock, err := waitForUsableBackgroundRuntimeOrLaunchLock( - ctx, c.Data.DataDir, c.Server.DaemonAutoRestart, localDaemonAutoStartReadyTimeout, + ctx, c.Data.DataDir, restartPolicy, localDaemonAutoStartReadyTimeout, ) if err != nil { return nil, localDaemonStartupInfo{}, err @@ -405,7 +450,11 @@ func ensureLocalDaemonRuntimeWithStartupCacheIntent( } defer func() { _ = launchLock.Unlock() }() - prep, err := prepareBackgroundDaemonStart(c, "run `msgvault daemon stop` or retry with --local") + incompatibleGuidance := "run `msgvault daemon stop` or retry with --local" + if !autoStart { + incompatibleGuidance = "restart or upgrade the supervised service" + } + prep, err := prepareBackgroundDaemonStart(c, restartPolicy, incompatibleGuidance, loggerFromContext(ctx)) if err != nil { return nil, localDaemonStartupInfo{}, err } @@ -415,10 +464,18 @@ func ensureLocalDaemonRuntimeWithStartupCacheIntent( } return rt, localDaemonStartupInfo{}, nil } + if !autoStart { + return nil, localDaemonStartupInfo{}, localDaemonAutoStartDisabledError(c.Data.DataDir) + } startedAt := time.Now() + options := invocationOptions{} + if state := invocationFromContext(ctx); state != nil { + options = state.options + } proc, err := startServeBackgroundProcessForRun(c, backgroundServeStartOptions{ CacheBuildIntent: intent, + Invocation: &options, }) if err != nil { return nil, localDaemonStartupInfo{}, fmt.Errorf("start background daemon: %w", err) @@ -480,6 +537,17 @@ func ensureLocalDaemonRuntimeWithStartupCacheIntent( }, nil } +// errLocalDaemonAutoStartDisabled marks a local resolution that found no +// usable daemon while [server].daemon_auto_start is false. +var errLocalDaemonAutoStartDisabled = errors.New("local daemon auto-start is disabled") + +func localDaemonAutoStartDisabledError(dataDir string) error { + return fmt.Errorf( + "%w: no usable msgvault daemon is running for %s and [server] daemon_auto_start is false; "+ + "start the supervised daemon or run `msgvault daemon start`, then retry", + errLocalDaemonAutoStartDisabled, dataDir) +} + // waitForStartupCacheBuildOutcome waits after HTTP readiness for a daemon // started with an explicit cache intent to publish its outcome. A non-empty // outcome means the background initializer has finished; unconsumed is @@ -950,13 +1018,13 @@ func probeLocalDaemonAuth(ctx context.Context, rt *DaemonRuntime, c *config.Conf if err := localDaemonAuthIdentityError(url, rt, c); err != nil { return err } - if c.Server.APIKey == "" && daemonRuntimeAuthFingerprint(rt) == daemonAPIKeyFingerprint("") { + if c.Server.AuthenticationKey() == "" && daemonRuntimeAuthFingerprint(rt) == daemonAPIKeyFingerprint("") { return nil } probeCtx, cancel := context.WithTimeout(ctx, localDaemonAuthProbeTimeout) defer cancel() - client, err := localDaemonAPIClient(url, c.Server.APIKey) + client, err := localDaemonAPIClient(url, c.Server.AuthenticationKey()) if err != nil { return fmt.Errorf("create local daemon auth probe: %w", err) } @@ -993,9 +1061,12 @@ func localDaemonAuthIdentityError(url string, rt *DaemonRuntime, c *config.Confi if rt == nil || c == nil { return nil } - want := daemonAPIKeyFingerprint(c.Server.APIKey) + if err := c.ResolveServerKey(); err != nil { + return err + } + want := daemonAPIKeyFingerprint(c.Server.AuthenticationKey()) got := daemonRuntimeAuthFingerprint(rt) - if got == "" && c.Server.APIKey == "" { + if got == "" && c.Server.AuthenticationKey() == "" { return nil } if got != want { diff --git a/cmd/msgvault/cmd/store_resolver_remote_schema_test.go b/cmd/msgvault/cmd/store_resolver_remote_schema_test.go index c869a4009..643d3df9e 100644 --- a/cmd/msgvault/cmd/store_resolver_remote_schema_test.go +++ b/cmd/msgvault/cmd/store_resolver_remote_schema_test.go @@ -1,7 +1,9 @@ package cmd import ( + "context" "encoding/json" + "errors" "net/http" "net/http/httptest" "os" @@ -19,12 +21,16 @@ import ( // stub a remote daemon with single-route handlers that do not serve // /api/v1/health. The probe itself is exercised by the // TestOpenRemoteStore*APISchema* tests, which re-enable it per test. +// It also blocks real daemon launches, which would rerun this test binary as "serve". func TestMain(m *testing.M) { remoteAPISchemaCheckEnabled = false + startServeBackgroundProcessForRun = func(*config.Config, backgroundServeStartOptions) (*backgroundServeProcess, error) { + return nil, errors.New("test reached the real background daemon launcher; stub it with stubStartServeBackgroundProcess or disable daemon auto-start") + } os.Exit(m.Run()) } -func remoteSchemaStub(t *testing.T, health func(w http.ResponseWriter)) *atomic.Int32 { +func remoteSchemaStub(t *testing.T, health func(w http.ResponseWriter)) (*atomic.Int32, context.Context) { t.Helper() var healthRequests atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -37,24 +43,24 @@ func remoteSchemaStub(t *testing.T, health func(w http.ResponseWriter)) *atomic. _ = json.NewEncoder(w).Encode(map[string]any{"persons": []any{}}) })) t.Cleanup(server.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, }) remoteAPISchemaCheckEnabled = true t.Cleanup(func() { remoteAPISchemaCheckEnabled = false }) - return &healthRequests + return &healthRequests, testCtx } func TestOpenRemoteStoreVerifiesAPISchemaOnMatchingMajor(t *testing.T) { require := require.New(t) - healthRequests := remoteSchemaStub(t, func(w http.ResponseWriter) { + healthRequests, testCtx := remoteSchemaStub(t, func(w http.ResponseWriter) { w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]any{ "status": "ok", "api_schema_version": api.APISchemaVersion, }) }) - client, info, err := OpenHTTPStore(t.Context()) + client, info, err := OpenHTTPStore(testCtx) require.NoError(err) t.Cleanup(func() { _ = client.Close() }) assert.Equal(t, HTTPStoreConfiguredRemote, info.Kind) @@ -63,42 +69,42 @@ func TestOpenRemoteStoreVerifiesAPISchemaOnMatchingMajor(t *testing.T) { func TestOpenRemoteStoreRejectsAPISchemaMajorMismatch(t *testing.T) { require := require.New(t) - _ = remoteSchemaStub(t, func(w http.ResponseWriter) { + _, testCtx := remoteSchemaStub(t, func(w http.ResponseWriter) { w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]any{ "status": "ok", "api_schema_version": "1.44.0", }) }) - _, _, err := OpenHTTPStore(t.Context()) + _, _, err := OpenHTTPStore(testCtx) require.ErrorContains(err, `daemon API schema version "1.44.0" is incompatible`) } -func TestOpenRemoteStoreRejectsOlderMinorSchema(t *testing.T) { +func TestOpenRemoteStoreRejectsPreviousMajorSchema(t *testing.T) { require := require.New(t) - _ = remoteSchemaStub(t, func(w http.ResponseWriter) { + _, testCtx := remoteSchemaStub(t, func(w http.ResponseWriter) { w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]any{ - "status": "ok", "api_schema_version": "2.13.0", + "status": "ok", "api_schema_version": "2.35.0", }) }) - client, _, err := OpenHTTPStore(t.Context()) + client, _, err := OpenHTTPStore(testCtx) if client != nil { t.Cleanup(func() { _ = client.Close() }) } - require.ErrorContains(err, "requires API schema 2.14.0 or newer") + require.ErrorContains(err, `daemon API schema version "2.35.0" is incompatible`) } -func TestOpenRemoteStoreAcceptsCompatiblePreviousMinorSchema(t *testing.T) { - healthRequests := remoteSchemaStub(t, func(w http.ResponseWriter) { +func TestOpenRemoteStoreAcceptsCompatibleNewerMinorSchema(t *testing.T) { + healthRequests, testCtx := remoteSchemaStub(t, func(w http.ResponseWriter) { w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]any{ - "status": "ok", "api_schema_version": "2.15.0", + "status": "ok", "api_schema_version": "3.1.0", }) }) - client, _, err := OpenHTTPStore(t.Context()) + client, _, err := OpenHTTPStore(testCtx) require.NoError(t, err) t.Cleanup(func() { _ = client.Close() }) assert.Equal(t, int32(1), healthRequests.Load()) @@ -106,23 +112,71 @@ func TestOpenRemoteStoreAcceptsCompatiblePreviousMinorSchema(t *testing.T) { func TestOpenRemoteStoreRejectsDaemonWithoutSchemaVersion(t *testing.T) { require := require.New(t) - _ = remoteSchemaStub(t, func(w http.ResponseWriter) { + _, testCtx := remoteSchemaStub(t, func(w http.ResponseWriter) { w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]any{"status": "ok"}) }) - _, _, err := OpenHTTPStore(t.Context()) + _, _, err := OpenHTTPStore(testCtx) require.ErrorContains(err, "does not report an API schema version") require.ErrorContains(err, "upgrade the daemon") } +func TestIdentityCommandsRejectIncompatibleDaemonBeforeReadingOrWriting(t *testing.T) { + for _, command := range []struct { + name string + run func(context.Context, *testing.T) (string, error) + }{ + {"list", func(ctx context.Context, t *testing.T) (string, error) { + t.Helper() + return runIdentityMatchesCLI(ctx, t, "", "list") + }}, + {"accept", func(ctx context.Context, t *testing.T) (string, error) { + t.Helper() + return runIdentityMatchesCLI(ctx, t, "", "accept", "17", "--review-token", "review-token-17") + }}, + {"reject", func(ctx context.Context, t *testing.T) (string, error) { + t.Helper() + return runIdentityMatchesCLI(ctx, t, "", "reject", "17", "--review-token", "review-token-17") + }}, + {"scoring", func(ctx context.Context, t *testing.T) (string, error) { + t.Helper() + return runPersonScoringCLI(ctx, t, "status") + }}, + } { + t.Run(command.name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + var operationRequests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v1/health" { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"status":"ok","api_schema_version":"2.35.0"}`)) + return + } + operationRequests.Add(1) + http.NotFound(w, r) + })) + t.Cleanup(server.Close) + ctx := withStoreResolverConfig(t, &config.Config{ + Remote: config.RemoteConfig{URL: server.URL, AllowInsecure: true}, + }) + remoteAPISchemaCheckEnabled = true + t.Cleanup(func() { remoteAPISchemaCheckEnabled = false }) + _, err := command.run(ctx, t) + require.ErrorContains(err, "incompatible") + assert.Zero(operationRequests.Load()) + }) + } +} + func TestOpenRemoteStoreSurfacesHealthProbeFailure(t *testing.T) { require := require.New(t) - _ = remoteSchemaStub(t, func(w http.ResponseWriter) { + _, testCtx := remoteSchemaStub(t, func(w http.ResponseWriter) { w.WriteHeader(http.StatusInternalServerError) }) - _, _, err := OpenHTTPStore(t.Context()) + _, _, err := OpenHTTPStore(testCtx) require.ErrorContains(err, "verify remote daemon API schema version") } @@ -141,18 +195,18 @@ func TestDaemonRuntimeCompatibilityRejectsLegacyRecordWithoutSchemaVersion(t *te require.ErrorContains(daemonRuntimeCompatibilityError(previousMajor), `daemon API schema version "1.44.0" is incompatible`) - previousSupportedMinor := &DaemonRuntime{API: daemonAPIVersion, APISchemaVersion: "2.15.0"} - require.NoError(daemonRuntimeCompatibilityError(previousSupportedMinor)) + newerMinor := &DaemonRuntime{API: daemonAPIVersion, APISchemaVersion: "3.1.0"} + require.NoError(daemonRuntimeCompatibilityError(newerMinor)) - previousMinor := &DaemonRuntime{API: daemonAPIVersion, APISchemaVersion: "2.13.0"} - require.ErrorContains(daemonRuntimeCompatibilityError(previousMinor), - "requires API schema 2.14.0 or newer") + previousSchema := &DaemonRuntime{API: daemonAPIVersion, APISchemaVersion: "2.35.0"} + require.ErrorContains(daemonRuntimeCompatibilityError(previousSchema), + `daemon API schema version "2.35.0" is incompatible`) } // agentDelegatedSchemaStub sets up a stub HTTP server that serves the health // endpoint for openAgentDelegatedStore, enables the schema check, and restores // state on cleanup. -func agentDelegatedSchemaStub(t *testing.T, sessionResponse string, health func(w http.ResponseWriter)) *atomic.Int32 { +func agentDelegatedSchemaStub(t *testing.T, sessionResponse string, health func(w http.ResponseWriter)) (*atomic.Int32, context.Context) { t.Helper() var healthRequests atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -170,29 +224,22 @@ func agentDelegatedSchemaStub(t *testing.T, sessionResponse string, health func( })) t.Cleanup(server.Close) - // Set global agent-mode flags and restore on cleanup. - oldURL := agentURL - oldFile := agentTokenFile - oldInsecure := agentAllowInsecure - agentURL = server.URL - agentAllowInsecure = true - // Write a token file with a fake secret. tokenFile, err := os.CreateTemp(t.TempDir(), "agent-token-*") require.NoError(t, err) _, _ = tokenFile.WriteString("mva1_fakesecretfortesting") _ = tokenFile.Close() - agentTokenFile = tokenFile.Name() - - t.Cleanup(func() { - agentURL = oldURL - agentTokenFile = oldFile - agentAllowInsecure = oldInsecure + ctx := testInvocationContext(t.Context(), config.NewDefaultConfig(), invocationOptions{ + agentURL: server.URL, + agentTokenFile: tokenFile.Name(), + agentAllowInsecure: true, + agentURLChanged: true, + agentTokenChanged: true, }) remoteAPISchemaCheckEnabled = true t.Cleanup(func() { remoteAPISchemaCheckEnabled = false }) - return &healthRequests + return &healthRequests, ctx } // TestOpenAgentDelegatedStoreVerifiesAPISchema verifies that openAgentDelegatedStore @@ -200,14 +247,14 @@ func agentDelegatedSchemaStub(t *testing.T, sessionResponse string, health func( // matching schema version. func TestOpenAgentDelegatedStoreVerifiesAPISchema(t *testing.T) { require := require.New(t) - healthRequests := agentDelegatedSchemaStub(t, `{"auth_mode":"delegated"}`, func(w http.ResponseWriter) { + healthRequests, ctx := agentDelegatedSchemaStub(t, `{"auth_mode":"delegated"}`, func(w http.ResponseWriter) { w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]any{ "status": "ok", "api_schema_version": api.APISchemaVersion, }) }) - client, info, err := openAgentDelegatedStore(t.Context()) + client, info, err := openAgentDelegatedStore(ctx, invocationFromContext(ctx)) require.NoError(err) t.Cleanup(func() { _ = client.Close() }) assert.Equal(t, HTTPStoreAgentDelegated, info.Kind) @@ -218,27 +265,27 @@ func TestOpenAgentDelegatedStoreVerifiesAPISchema(t *testing.T) { // openAgentDelegatedStore rejects a daemon with an incompatible API schema. func TestOpenAgentDelegatedStoreRejectsMismatchedSchema(t *testing.T) { require := require.New(t) - _ = agentDelegatedSchemaStub(t, "", func(w http.ResponseWriter) { + _, ctx := agentDelegatedSchemaStub(t, "", func(w http.ResponseWriter) { w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]any{ "status": "ok", "api_schema_version": "1.0.0", }) }) - _, _, err := openAgentDelegatedStore(t.Context()) + _, _, err := openAgentDelegatedStore(ctx, invocationFromContext(ctx)) require.ErrorContains(err, "incompatible") } func TestOpenAgentDelegatedStoreReportsAuthenticationFailure(t *testing.T) { assert := assert.New(t) require := require.New(t) - agentDelegatedSchemaStub(t, "", func(w http.ResponseWriter) { + _, ctx := agentDelegatedSchemaStub(t, "", func(w http.ResponseWriter) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusUnauthorized) _, _ = w.Write([]byte(`{"error":"unauthorized","message":"Invalid or missing API key"}`)) }) - _, _, err := openAgentDelegatedStore(t.Context()) + _, _, err := openAgentDelegatedStore(ctx, invocationFromContext(ctx)) require.ErrorContains(err, "agent authentication failed") assert.NotContains(err.Error(), "schema version") var apiErr *daemonclient.APIError @@ -256,14 +303,14 @@ func TestOpenAgentDelegatedStoreRequiresDelegatedAuthentication(t *testing.T) { {"missing session endpoint", ""}, } { t.Run(tc.name, func(t *testing.T) { - agentDelegatedSchemaStub(t, tc.session, func(w http.ResponseWriter) { + _, ctx := agentDelegatedSchemaStub(t, tc.session, func(w http.ResponseWriter) { w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]any{ "status": "ok", "api_schema_version": api.APISchemaVersion, }) }) - client, _, err := OpenHTTPStore(t.Context()) + client, _, err := OpenHTTPStore(ctx) if client != nil { t.Cleanup(func() { _ = client.Close() }) } diff --git a/cmd/msgvault/cmd/store_resolver_test.go b/cmd/msgvault/cmd/store_resolver_test.go index 63cb99a06..77d4dd110 100644 --- a/cmd/msgvault/cmd/store_resolver_test.go +++ b/cmd/msgvault/cmd/store_resolver_test.go @@ -5,6 +5,7 @@ import ( "context" "errors" "io" + "log/slog" "net" "net/http" "net/http/httptest" @@ -13,6 +14,7 @@ import ( "strconv" "sync/atomic" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -23,10 +25,11 @@ import ( "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/daemonauth" "go.kenn.io/msgvault/internal/daemonclient" + "go.kenn.io/msgvault/internal/providercredentials" ) func TestOpenHTTPStoreUsesConfiguredRemoteWithoutDaemonAutostart(t *testing.T) { - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{ URL: "http://daemonclient.example:8080", AllowInsecure: true, @@ -37,7 +40,7 @@ func TestOpenHTTPStoreUsesConfiguredRemoteWithoutDaemonAutostart(t *testing.T) { return nil, errors.New("unreachable") }) - st, info, err := OpenHTTPStore(context.Background()) + st, info, err := OpenHTTPStore(testCtx) require.NoError(t, err, "OpenHTTPStore") t.Cleanup(func() { _ = st.Close() }) @@ -45,6 +48,30 @@ func TestOpenHTTPStoreUsesConfiguredRemoteWithoutDaemonAutostart(t *testing.T) { assert.Equal(t, "http://daemonclient.example:8080", info.URL) } +func TestOpenHTTPStoreDisabledAutoStartKeepsConfiguredRemote(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + testCtx := withStoreResolverConfig(t, &config.Config{ + Remote: config.RemoteConfig{ + URL: "http://daemonclient.example:8080", + AllowInsecure: true, + }, + Server: config.ServerConfig{DaemonAutoStart: new(false)}, + }) + stubStartServeBackgroundProcess(t, func(*config.Config, backgroundServeStartOptions) (*backgroundServeProcess, error) { + require.FailNow("configured remote must not start a local daemon") + return nil, errors.New("unreachable") + }) + + st, info, err := OpenHTTPStore(testCtx) + require.NoError(err, "OpenHTTPStore") + t.Cleanup(func() { _ = st.Close() }) + + assert.Equal(HTTPStoreConfiguredRemote, info.Kind) + assert.Equal("http://daemonclient.example:8080", info.URL) +} + func TestOpenHTTPStoreUsesCLIModeForConfiguredRemote(t *testing.T) { assert := assert.New(t) require := require.New(t) @@ -59,7 +86,7 @@ func TestOpenHTTPStoreUsesCLIModeForConfiguredRemote(t *testing.T) { })) t.Cleanup(srv.Close) - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ Remote: config.RemoteConfig{ URL: srv.URL, APIKey: "remote-daemon-secret", @@ -67,12 +94,12 @@ func TestOpenHTTPStoreUsesCLIModeForConfiguredRemote(t *testing.T) { }, }) - st, _, err := OpenHTTPStore(context.Background()) + st, _, err := OpenHTTPStore(testCtx) require.NoError(err, "OpenHTTPStore") t.Cleanup(func() { _ = st.Close() }) assert.Zero(st.Timeout(), "configured remote operations use caller duration") - _, err = st.GetHealth(context.Background()) + _, err = st.GetHealth(testCtx) require.NoError(err, "GetHealth") assert.Equal(apiprotocol.ClientClassCLI, marker.Load()) } @@ -81,6 +108,7 @@ func TestOpenHTTPStoreRootContextCancelsLocalDaemonRequest(t *testing.T) { assert := assert.New(t) require := require.New(t) var marker atomic.Value + requestStarted := make(chan struct{}, 1) requestCanceled := make(chan struct{}) mux := http.NewServeMux() mux.Handle("/api/ping", daemon.NewPingHandler(daemon.PingHandlerOptions{ @@ -94,6 +122,7 @@ func TestOpenHTTPStoreRootContextCancelsLocalDaemonRequest(t *testing.T) { }) mux.HandleFunc("/api/v1/stats", func(_ http.ResponseWriter, r *http.Request) { marker.Store(r.Header.Get(apiprotocol.ClientClassHeader)) + requestStarted <- struct{}{} <-r.Context().Done() close(requestCanceled) }) @@ -101,12 +130,12 @@ func TestOpenHTTPStoreRootContextCancelsLocalDaemonRequest(t *testing.T) { t.Cleanup(srv.Close) dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) rt := daemonRuntimeForHTTPServer(t, srv, daemonAPIKeyFingerprint("")) _, err := daemonRuntimeStore(dataDir).Write(rt.Record) require.NoError(err, "write daemon runtime") - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(testCtx) t.Cleanup(cancel) st, _, err := OpenHTTPStore(ctx) require.NoError(err, "OpenHTTPStore") @@ -118,19 +147,17 @@ func TestOpenHTTPStoreRootContextCancelsLocalDaemonRequest(t *testing.T) { _, err := st.GetStats() done <- err }() - require.Eventually(func() bool { - return marker.Load() != nil - }, 2*time.Second, 10*time.Millisecond, "stats request starts") + select { + case <-requestStarted: + case <-time.After(2 * time.Second): + require.FailNow("stats request did not start") + } cancel() - - require.Eventually(func() bool { - select { - case <-requestCanceled: - return true - default: - return false - } - }, 2*time.Second, 10*time.Millisecond, "root cancellation reaches stats request") + select { + case <-requestCanceled: + case <-time.After(2 * time.Second): + require.FailNow("root cancellation did not reach stats request") + } assert.Equal(apiprotocol.ClientClassCLI, marker.Load()) require.Error(<-done, "canceled stats request") } @@ -139,7 +166,7 @@ func TestOpenHTTPStoreStartsLocalDaemonWhenNoRemoteConfigured(t *testing.T) { assert := assert.New(t) dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) waitCh := make(chan error) var started bool stubStartServeBackgroundProcess(t, func(c *config.Config, _ backgroundServeStartOptions) (*backgroundServeProcess, error) { @@ -168,7 +195,7 @@ func TestOpenHTTPStoreStartsLocalDaemonWhenNoRemoteConfigured(t *testing.T) { }, true, nil }) - st, info, err := OpenHTTPStore(context.Background()) + st, info, err := OpenHTTPStore(testCtx) require.NoError(t, err, "OpenHTTPStore") t.Cleanup(func() { _ = st.Close() }) assert.True(started, "local daemon should be started") @@ -176,11 +203,255 @@ func TestOpenHTTPStoreStartsLocalDaemonWhenNoRemoteConfigured(t *testing.T) { assert.Equal("http://127.0.0.1:9911", info.URL) } +func TestOpenHTTPStoreUsesMintedKeyAfterLocalDaemonStartup(t *testing.T) { //nolint:paralleltest // process environment and daemon startup hooks + unsetServerKeyEnvironmentForTest(t) + dataDir := t.TempDir() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + key, err := providercredentials.ReadSecretFile(filepath.Join(dataDir, "tokens", providercredentials.ServerKeyFilename)) + if err != nil || r.Header.Get("X-Api-Key") != key { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"status":"ok"}`)) + })) + t.Cleanup(server.Close) + + cfg := lifecycleTestConfig(dataDir) + cfg.Server.BindAddr = "0.0.0.0" + testCtx := withStoreResolverConfig(t, cfg) + waitCh := make(chan error) + stubStartServeBackgroundProcess(t, func(c *config.Config, _ backgroundServeStartOptions) (*backgroundServeProcess, error) { + _, err := providercredentials.EnsureServerKey(filepath.Dir(c.ServerKeyFilePath())) + require.NoError(t, err) + return &backgroundServeProcess{PID: 4242, LogPath: "/tmp/msgvault-serve.log", Wait: waitCh}, nil + }) + stubWaitForBackgroundServeReady(t, func( + ctx context.Context, + _ string, + _ <-chan error, + _ time.Duration, + ) (*DaemonRuntime, bool, error) { + require.NoError(t, ctx.Err()) + key, err := providercredentials.ReadSecretFile(filepath.Join(dataDir, "tokens", providercredentials.ServerKeyFilename)) + require.NoError(t, err) + return daemonRuntimeForHTTPServer(t, server, daemonAPIKeyFingerprint(key)), true, nil + }) + + st, info, err := OpenHTTPStore(testCtx) + require.NoError(t, err) + t.Cleanup(func() { _ = st.Close() }) + require.True(t, info.StartedLocalDaemon) + _, err = st.GetHealth(testCtx) + require.NoError(t, err, "the CLI client must use the key minted by the daemon") +} + +func unsetServerKeyEnvironmentForTest(t *testing.T) { + t.Helper() + for _, name := range []string{"MSGVAULT_API_KEY", "MSGVAULT_API_KEY_FILE", "MSGVAULT_API_KEY_ENV"} { + envName := name + original, _ := os.LookupEnv(envName) + t.Setenv(envName, original) + require.NoError(t, os.Unsetenv(envName)) + } +} + +func TestOpenHTTPStoreDisabledAutoStartDoesNotStartDaemon(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + dataDir := t.TempDir() + c := lifecycleTestConfig(dataDir) + c.Server.DaemonAutoStart = new(false) + testCtx := withStoreResolverConfig(t, c) + stubStartServeBackgroundProcess(t, func(*config.Config, backgroundServeStartOptions) (*backgroundServeProcess, error) { + require.FailNow("disabled auto-start must not start a local daemon") + return nil, errors.New("unreachable") + }) + stubStopDaemonRuntimeForUpgrade(t, func(config.Config, *DaemonRuntime, *slog.Logger) error { + require.FailNow("disabled auto-start must not stop a daemon") + return errors.New("unreachable") + }) + + st, _, err := OpenHTTPStore(testCtx) + assert.Nil(st) + require.ErrorIs(err, errLocalDaemonAutoStartDisabled) + assert.Contains(err.Error(), dataDir) + assert.Contains(err.Error(), "msgvault daemon start") + + held, err := daemonOwnerLockHeld(dataDir) + require.NoError(err, "check daemon ownership") + assert.False(held, "disabled auto-start must leave daemon ownership free") + owner, err := claimServeOwnership(testCtx, c, "127.0.0.1", 8123, "v-test") + require.NoError(err, "supervised serve should claim ownership") + require.NoError(owner.Close(), "release supervised ownership") +} + +func TestOpenHTTPStoreDisabledAutoStartReusesOlderDaemonWithoutRestart(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + withTestVersion(t, "v1.1.0") + dataDir := t.TempDir() + c := lifecycleTestConfig(dataDir) + c.Server.DaemonAutoStart = new(false) + c.Server.DaemonAutoRestart = config.DaemonAutoRestartAlways + testCtx := withStoreResolverConfig(t, c) + ping := httptestPingDaemon(t) + portText := strconv.Itoa(ping.Port) + _, err := daemonRuntimeStore(dataDir).Write(daemon.RuntimeRecord{ + PID: os.Getpid(), + Network: daemon.NetworkTCP, + Address: net.JoinHostPort(ping.Host, portText), + Service: daemonService, + Version: "v1.0.0", + Metadata: map[string]string{ + runtimeHost: ping.Host, + runtimePort: portText, + runtimeAPIVersion: strconv.Itoa(daemonAPIVersion), + runtimeAPISchemaVersion: api.APISchemaVersion, + runtimeAuthFingerprint: daemonAPIKeyFingerprint(""), + runtimeCreateTime: matchingProcessCreateTime(t), + }, + }) + require.NoError(err, "write runtime") + + stubStopDaemonRuntimeForUpgrade(t, func(config.Config, *DaemonRuntime, *slog.Logger) error { + require.FailNow("disabled auto-start must not stop an older daemon") + return errors.New("unreachable") + }) + stubStartServeBackgroundProcess(t, func(*config.Config, backgroundServeStartOptions) (*backgroundServeProcess, error) { + require.FailNow("disabled auto-start must not start a replacement daemon") + return nil, errors.New("unreachable") + }) + + st, info, err := OpenHTTPStore(testCtx) + require.NoError(err, "OpenHTTPStore") + t.Cleanup(func() { _ = st.Close() }) + + assert.Equal(HTTPStoreLocalDaemon, info.Kind) + assert.Equal("http://"+net.JoinHostPort(ping.Host, portText), info.URL) +} + +func TestOpenHTTPStoreDisabledAutoStartReportsIncompatibleDaemonWithoutStopping(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + withTestVersion(t, "v1.1.0") + dataDir := t.TempDir() + c := lifecycleTestConfig(dataDir) + c.Server.DaemonAutoStart = new(false) + c.Server.DaemonAutoRestart = config.DaemonAutoRestartAlways + testCtx := withStoreResolverConfig(t, c) + ping := httptestPingDaemon(t) + portText := strconv.Itoa(ping.Port) + _, err := daemonRuntimeStore(dataDir).Write(daemon.RuntimeRecord{ + PID: os.Getpid(), + Network: daemon.NetworkTCP, + Address: net.JoinHostPort(ping.Host, portText), + Service: daemonService, + Version: "v1.0.0", + Metadata: map[string]string{ + runtimeHost: ping.Host, + runtimePort: portText, + runtimeAPIVersion: strconv.Itoa(daemonAPIVersion - 1), + runtimeAPISchemaVersion: api.APISchemaVersion, + runtimeCreateTime: matchingProcessCreateTime(t), + }, + }) + require.NoError(err, "write runtime") + + stubStopDaemonRuntimeForUpgrade(t, func(config.Config, *DaemonRuntime, *slog.Logger) error { + require.FailNow("disabled auto-start must not stop an incompatible daemon") + return errors.New("unreachable") + }) + stubStartServeBackgroundProcess(t, func(*config.Config, backgroundServeStartOptions) (*backgroundServeProcess, error) { + require.FailNow("disabled auto-start must not start over an incompatible daemon") + return nil, errors.New("unreachable") + }) + + st, info, err := OpenHTTPStore(testCtx) + assert.Nil(st) + require.Error(err, "OpenHTTPStore") + assert.Equal(HTTPStoreInfo{}, info) + assert.Contains(err.Error(), "incompatible daemon is already running") + assert.Contains(err.Error(), "daemon API version") + assert.Contains(err.Error(), "restart or upgrade the supervised service") + assert.NotContains(err.Error(), "msgvault daemon stop") + assert.NotContains(err.Error(), "--local") +} + +func TestOpenHTTPStoreDisabledAutoStartWaitsForStartingDaemon(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + dataDir := t.TempDir() + c := lifecycleTestConfig(dataDir) + c.Server.DaemonAutoStart = new(false) + testCtx := withStoreResolverConfig(t, c) + _, err := daemonRuntimeStore(dataDir).Write(daemon.RuntimeRecord{ + PID: os.Getpid(), + Network: daemon.NetworkTCP, + Address: "127.0.0.1:1", + Service: daemonService, + Version: Version, + Metadata: map[string]string{ + runtimeHost: "127.0.0.1", + runtimePort: "1", + runtimeAPIVersion: strconv.Itoa(daemonAPIVersion), + runtimeAPISchemaVersion: api.APISchemaVersion, + }, + }) + require.NoError(err, "write runtime") + + stubStartServeBackgroundProcess(t, func(*config.Config, backgroundServeStartOptions) (*backgroundServeProcess, error) { + require.FailNow("disabled auto-start must wait instead of starting a daemon") + return nil, errors.New("unreachable") + }) + + ctx, cancel := context.WithTimeout(testCtx, 300*time.Millisecond) + t.Cleanup(cancel) + var st *daemonclient.Client + var info HTTPStoreInfo + var openErr error + stderr := captureStderrDuring(t, func() { + st, info, openErr = OpenHTTPStore(ctx) + }) + assert.Nil(st) + assert.Equal(HTTPStoreInfo{}, info) + require.ErrorIs(openErr, context.DeadlineExceeded) + assert.Contains(stderr, "Another msgvault daemon start is in progress") + launchLock, ok := acquireBackgroundLaunchLock(dataDir) + require.True(ok, "disabled auto-start must release the launch lock after waiting") + require.NoError(launchLock.Unlock()) +} + +func TestOpenHTTPStoreDisabledAutoStartLocalFlagStaysLocal(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + dataDir := t.TempDir() + c := lifecycleTestConfig(dataDir) + c.Remote.URL = "http://daemonclient.example:8080" + c.Remote.AllowInsecure = true + c.Server.DaemonAutoStart = new(false) + ctx := testInvocationContext(t.Context(), c, invocationOptions{useLocal: true}) + stubStartServeBackgroundProcess(t, func(*config.Config, backgroundServeStartOptions) (*backgroundServeProcess, error) { + require.FailNow("--local must not start a daemon when auto-start is disabled") + return nil, errors.New("unreachable") + }) + + st, info, err := OpenHTTPStore(ctx) + assert.Nil(st) + require.ErrorIs(err, errLocalDaemonAutoStartDisabled) + assert.Equal(HTTPStoreInfo{}, info) +} + func TestOpenHTTPStoreReportsFulfilledStartupCacheBuild(t *testing.T) { assert := assert.New(t) dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) waitCh := make(chan error) var gotIntent startupCacheBuildIntent stubStartServeBackgroundProcess(t, func( @@ -226,7 +497,7 @@ func TestOpenHTTPStoreReportsFulfilledStartupCacheBuild(t *testing.T) { var err error captureStderrDuring(t, func() { st, info, err = openHTTPStoreWithStartupCacheIntent( - context.Background(), startupCacheBuildIntentDefault, + testCtx, startupCacheBuildIntentDefault, ) }) require.NoError(t, err) @@ -239,36 +510,38 @@ func TestOpenHTTPStoreReportsFulfilledStartupCacheBuild(t *testing.T) { } func TestWaitForStartupCacheBuildOutcomeWaitsAfterHTTPReadiness(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - dataDir := t.TempDir() - record := daemon.RuntimeRecord{ - PID: os.Getpid(), - Network: daemon.NetworkTCP, - Address: "127.0.0.1:1", - Service: daemonService, - Version: Version, - Metadata: map[string]string{}, - } - _, err := daemonRuntimeStore(dataDir).Write(record) - require.NoError(err) - rt := &DaemonRuntime{Record: record} - - go func() { - time.Sleep(50 * time.Millisecond) - updated := record - updated.Metadata = map[string]string{ - runtimeStartupCacheBuildOutcome: string(startupCacheBuildOutcomeFulfilled), + synctest.Test(t, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + dataDir := t.TempDir() + record := daemon.RuntimeRecord{ + PID: os.Getpid(), + Network: daemon.NetworkTCP, + Address: "127.0.0.1:1", + Service: daemonService, + Version: Version, + Metadata: map[string]string{}, } - _, _ = daemonRuntimeStore(dataDir).Write(updated) - }() + _, err := daemonRuntimeStore(dataDir).Write(record) + require.NoError(err) + rt := &DaemonRuntime{Record: record} + + go func() { + synctest.Sleep(50 * time.Millisecond) + updated := record + updated.Metadata = map[string]string{ + runtimeStartupCacheBuildOutcome: string(startupCacheBuildOutcomeFulfilled), + } + _, _ = daemonRuntimeStore(dataDir).Write(updated) + }() - gotRT, outcome, err := waitForStartupCacheBuildOutcome( - context.Background(), dataDir, &backgroundServeProcess{}, rt, time.Second, - ) - require.NoError(err) - require.NotNil(gotRT) - assert.Equal(startupCacheBuildOutcomeFulfilled, outcome) + gotRT, outcome, err := waitForStartupCacheBuildOutcome( + context.Background(), dataDir, &backgroundServeProcess{}, rt, time.Second, + ) + require.NoError(err) + require.NotNil(gotRT) + assert.Equal(startupCacheBuildOutcomeFulfilled, outcome) + }) } func TestWaitForStartupCacheBuildOutcomeReportsProcessExit(t *testing.T) { @@ -315,7 +588,7 @@ func TestOpenHTTPStoreReportsLocalDaemonStartupToStderr(t *testing.T) { assert := assert.New(t) require := require.New(t) dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) waitCh := make(chan error) stubStartServeBackgroundProcess(t, func(*config.Config, backgroundServeStartOptions) (*backgroundServeProcess, error) { return &backgroundServeProcess{ @@ -341,7 +614,7 @@ func TestOpenHTTPStoreReportsLocalDaemonStartupToStderr(t *testing.T) { var st *daemonclient.Client var err error stderr := captureStderrDuring(t, func() { - st, _, err = OpenHTTPStore(context.Background()) + st, _, err = OpenHTTPStore(testCtx) }) require.NoError(err, "OpenHTTPStore") t.Cleanup(func() { _ = st.Close() }) @@ -358,7 +631,7 @@ func TestOpenHTTPStoreIncludesLastDaemonLogWhenStartupExits(t *testing.T) { require := require.New(t) dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) logPath := filepath.Join(dataDir, "serve.log") require.NoError(os.WriteFile(logPath, []byte("Error: API server address unavailable at 127.0.0.1:8080\n"), 0o600), "write serve log") waitCh := make(chan error) @@ -378,7 +651,7 @@ func TestOpenHTTPStoreIncludesLastDaemonLogWhenStartupExits(t *testing.T) { return nil, false, errors.New("exit status 1") }) - st, _, err := OpenHTTPStore(context.Background()) + st, _, err := OpenHTTPStore(testCtx) if st != nil { t.Cleanup(func() { _ = st.Close() }) } @@ -391,7 +664,7 @@ func TestOpenHTTPStoreIncludesLastDaemonLogWhenStartupExits(t *testing.T) { func TestCommandAwareDaemonAutostartCancellationStopsStartedDaemon(t *testing.T) { dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) waitCh := make(chan error) proc := &backgroundServeProcess{ PID: 4242, @@ -401,7 +674,7 @@ func TestCommandAwareDaemonAutostartCancellationStopsStartedDaemon(t *testing.T) stubStartServeBackgroundProcess(t, func(*config.Config, backgroundServeStartOptions) (*backgroundServeProcess, error) { return proc, nil }) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(testCtx) stubWaitForBackgroundServeReady(t, func( context.Context, string, @@ -431,7 +704,7 @@ func TestCommandAwareDaemonAutostartCancellationStopsStartedDaemon(t *testing.T) func TestOrdinaryDaemonAutostartCancellationLeavesDetachedDaemonRunning(t *testing.T) { dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) waitCh := make(chan error) stubStartServeBackgroundProcess(t, func(*config.Config, backgroundServeStartOptions) (*backgroundServeProcess, error) { return &backgroundServeProcess{ @@ -440,7 +713,7 @@ func TestOrdinaryDaemonAutostartCancellationLeavesDetachedDaemonRunning(t *testi Wait: waitCh, }, nil }) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(testCtx) stubWaitForBackgroundServeReady(t, func( context.Context, string, @@ -470,7 +743,7 @@ func TestOpenHTTPStoreTakesOverWhenConcurrentDaemonStartExits(t *testing.T) { require := require.New(t) dataDir := t.TempDir() - withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) + testCtx := withStoreResolverConfig(t, lifecycleTestConfig(dataDir)) heldLock, ok := acquireBackgroundLaunchLock(dataDir) require.True(ok, "test should hold background launch lock") t.Cleanup(func() { _ = heldLock.Unlock() }) @@ -502,7 +775,7 @@ func TestOpenHTTPStoreTakesOverWhenConcurrentDaemonStartExits(t *testing.T) { }, true, nil }) - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + ctx, cancel := context.WithTimeout(testCtx, 2*time.Second) defer cancel() st, info, err := OpenHTTPStore(ctx) require.NoError(err, "OpenHTTPStore") @@ -524,7 +797,7 @@ func TestOpenHTTPStoreUsesServerAPIKeyForLocalDaemon(t *testing.T) { dataDir := t.TempDir() localCfg := lifecycleTestConfig(dataDir) localCfg.Server.APIKey = "local-daemon-secret" - withStoreResolverConfig(t, localCfg) + testCtx := withStoreResolverConfig(t, localCfg) var gotAPIKey string mux := http.NewServeMux() @@ -573,7 +846,7 @@ func TestOpenHTTPStoreUsesServerAPIKeyForLocalDaemon(t *testing.T) { require.NoError( err, "write runtime") - st, info, err := OpenHTTPStore(context.Background()) + st, info, err := OpenHTTPStore(testCtx) require.NoError( err, "OpenHTTPStore") @@ -600,7 +873,7 @@ func TestOpenHTTPStoreReadsFromProvedDaemonWithMismatchedCreateTime(t *testing.T dataDir := t.TempDir() localCfg := lifecycleTestConfig(dataDir) localCfg.Server.APIKey = apiKey - withStoreResolverConfig(t, localCfg) + testCtx := withStoreResolverConfig(t, localCfg) stubProcessCreateTimeMillis(t, func(int) (int64, bool) { return 1_000, true }) stubStartServeBackgroundProcess(t, func(*config.Config, backgroundServeStartOptions) (*backgroundServeProcess, error) { require.FailNow("proved clock-stepped daemon must not be restarted") @@ -664,7 +937,7 @@ func TestOpenHTTPStoreReadsFromProvedDaemonWithMismatchedCreateTime(t *testing.T }) require.NoError(err, "write runtime") - st, info, err := OpenHTTPStore(context.Background()) + st, info, err := OpenHTTPStore(testCtx) require.NoError(err, "OpenHTTPStore") t.Cleanup(func() { _ = st.Close() }) stats, err := st.GetStats() @@ -684,7 +957,7 @@ func TestOpenHTTPStoreRejectsLocalDaemonWithStaleServerAPIKey(t *testing.T) { dataDir := t.TempDir() localCfg := lifecycleTestConfig(dataDir) localCfg.Server.APIKey = "new-local-daemon-secret" - withStoreResolverConfig(t, localCfg) + testCtx := withStoreResolverConfig(t, localCfg) var statsCalled bool mux := http.NewServeMux() @@ -723,7 +996,7 @@ func TestOpenHTTPStoreRejectsLocalDaemonWithStaleServerAPIKey(t *testing.T) { }) require.NoError(err, "write runtime") - st, _, err := OpenHTTPStore(context.Background()) + st, _, err := OpenHTTPStore(testCtx) if st != nil { t.Cleanup(func() { _ = st.Close() }) } @@ -741,7 +1014,7 @@ func TestOpenHTTPStoreRejectsLocalDaemonWithChangedServerAPIKeyFingerprint(t *te dataDir := t.TempDir() localCfg := lifecycleTestConfig(dataDir) localCfg.Server.APIKey = "new-local-daemon-secret" - withStoreResolverConfig(t, localCfg) + testCtx := withStoreResolverConfig(t, localCfg) var statsCalled bool mux := http.NewServeMux() @@ -778,7 +1051,7 @@ func TestOpenHTTPStoreRejectsLocalDaemonWithChangedServerAPIKeyFingerprint(t *te }) require.NoError(err, "write runtime") - st, _, err := OpenHTTPStore(context.Background()) + st, _, err := OpenHTTPStore(testCtx) if st != nil { t.Cleanup(func() { _ = st.Close() }) } @@ -795,7 +1068,7 @@ func TestOpenHTTPStoreRejectsLegacyLocalDaemonAfterServerAPIKeyRemoved(t *testin dataDir := t.TempDir() localCfg := lifecycleTestConfig(dataDir) - withStoreResolverConfig(t, localCfg) + testCtx := withStoreResolverConfig(t, localCfg) var statsCalled bool mux := http.NewServeMux() @@ -833,7 +1106,7 @@ func TestOpenHTTPStoreRejectsLegacyLocalDaemonAfterServerAPIKeyRemoved(t *testin }) require.NoError(err, "write runtime") - st, _, err := OpenHTTPStore(context.Background()) + st, _, err := OpenHTTPStore(testCtx) if st != nil { t.Cleanup(func() { _ = st.Close() }) } @@ -852,10 +1125,9 @@ func TestProbeLocalDaemonAuthDoesNotWaitForStats(t *testing.T) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"status":"ok"}`)) }) - mux.HandleFunc("/api/v1/stats", func(w http.ResponseWriter, _ *http.Request) { + mux.HandleFunc("/api/v1/stats", func(_ http.ResponseWriter, r *http.Request) { statsCalled.Store(true) - time.Sleep(3 * localDaemonAuthProbeTimeout) - w.WriteHeader(http.StatusOK) + <-r.Context().Done() }) server := httptest.NewServer(mux) t.Cleanup(server.Close) @@ -973,7 +1245,7 @@ func TestOpenHTTPStoreHonorsNeverAutoRestartPolicy(t *testing.T) { dataDir := t.TempDir() localCfg := lifecycleTestConfig(dataDir) localCfg.Server.DaemonAutoRestart = config.DaemonAutoRestartNever - withStoreResolverConfig(t, localCfg) + testCtx := withStoreResolverConfig(t, localCfg) mux := http.NewServeMux() mux.Handle("/api/ping", daemon.NewPingHandler(daemon.PingHandlerOptions{ @@ -1020,7 +1292,7 @@ func TestOpenHTTPStoreHonorsNeverAutoRestartPolicy(t *testing.T) { return nil, errors.New("unreachable") }) - st, info, err := OpenHTTPStore(context.Background()) + st, info, err := OpenHTTPStore(testCtx) require.NoError( err, "OpenHTTPStore") @@ -1041,8 +1313,7 @@ func TestOpenHTTPStoreLocalFlagUsesLocalDaemonInsteadOfConfiguredRemote(t *testi c := lifecycleTestConfig(dataDir) c.Remote.URL = "http://daemonclient.example:8080" c.Remote.AllowInsecure = true - withStoreResolverConfig(t, c) - useLocal = true + ctx := testInvocationContext(t.Context(), c, invocationOptions{useLocal: true}) waitCh := make(chan error) var started bool @@ -1072,7 +1343,7 @@ func TestOpenHTTPStoreLocalFlagUsesLocalDaemonInsteadOfConfiguredRemote(t *testi }, true, nil }) - st, info, err := OpenHTTPStore(context.Background()) + st, info, err := OpenHTTPStore(ctx) require.NoError(t, err, "OpenHTTPStore") t.Cleanup(func() { _ = st.Close() }) assert.True(started, "--local should start/use the local daemon") @@ -1195,16 +1466,9 @@ func TestWaitForUsableBackgroundRuntimeTakesOverUpgradeEligibleDaemon(t *testing _ = lock.Unlock() } -func withStoreResolverConfig(t *testing.T, c *config.Config) { +func withStoreResolverConfig(t *testing.T, c *config.Config) context.Context { t.Helper() - oldCfg := cfg - oldUseLocal := useLocal - cfg = c - useLocal = false - t.Cleanup(func() { - cfg = oldCfg - useLocal = oldUseLocal - }) + return testInvocationContext(t.Context(), c, invocationOptions{}) } func captureStderrDuring(t *testing.T, fn func()) string { diff --git a/cmd/msgvault/cmd/sync.go b/cmd/msgvault/cmd/sync.go index e233717c3..d6d9b63ef 100644 --- a/cmd/msgvault/cmd/sync.go +++ b/cmd/msgvault/cmd/sync.go @@ -31,6 +31,10 @@ Requires a prior full sync to establish the history ID baseline. IMAP accounts use folder-based sync. Unchanged folders are skipped when UIDVALIDITY/UIDNEXT high water marks are available. +Microsoft Graph mail accounts (add-o365 --graph) use one delta cursor per +folder. The first sync downloads every folder; later syncs fetch only the +changes, including moves and deletes. + If no email is specified, syncs all accounts that have credentials configured. Accounts without tokens or history IDs are skipped. @@ -50,11 +54,17 @@ Examples: } func runSyncIncrementalLocal(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil || state.logger == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger selector, selectorSet, err := syncSourceSelector(cmd, args) if err != nil { return usageErr(cmd, err) } - s, cleanup, err := openWritableStoreAndInit() + s, cleanup, err := openWritableStoreAndInitForInvocation(state) if err != nil { return err } @@ -79,7 +89,7 @@ func runSyncIncrementalLocal(cmd *cobra.Command, args []string) error { // embed worker (msgvault embeddings build / the serve daemon) // picks them up. - getOAuthMgr := oauthManagerCache() + getOAuthMgr := oauthManagerCache(invocationFromCommand(cmd)) // Determine which accounts to sync. type syncTarget struct { @@ -88,6 +98,7 @@ func runSyncIncrementalLocal(cmd *cobra.Command, args []string) error { } var gmailTargets []syncTarget var imapTargets []*store.Source + var msmailTargets []*store.Source var syncErrors []string if selectorSet { @@ -103,11 +114,13 @@ func runSyncIncrementalLocal(cmd *cobra.Command, args []string) error { gmailTargets = append(gmailTargets, syncTarget{source: src, email: src.Identifier}) case sourceTypeIMAP: imapTargets = append(imapTargets, src) + case sourceTypeMSMail: + msmailTargets = append(msmailTargets, src) } } - if len(gmailTargets) == 0 && len(imapTargets) == 0 { + if len(gmailTargets) == 0 && len(imapTargets) == 0 && len(msmailTargets) == 0 { if len(allMatches) > 0 { - return fmt.Errorf("%s exists but its source type cannot be synced (only gmail and imap are supported)", syncSelectorLabel(selector)) + return fmt.Errorf("%s exists but its source type cannot be synced (only gmail, imap and msmail are supported)", syncSelectorLabel(selector)) } if legacy { // Token not in DB — assume Gmail (legacy behaviour). @@ -149,7 +162,7 @@ func runSyncIncrementalLocal(cmd *cobra.Command, args []string) error { } gmailTargets = append(gmailTargets, syncTarget{source: src, email: src.Identifier}) case sourceTypeIMAP: - skipMsg, parseErr := imapSkipReason(src) + skipMsg, parseErr := imapSkipReason(src, cfg, logger) if parseErr != nil { syncErrors = append(syncErrors, fmt.Sprintf("%s: malformed sync_config: %v", src.Identifier, parseErr)) continue @@ -159,11 +172,17 @@ func runSyncIncrementalLocal(cmd *cobra.Command, args []string) error { continue } imapTargets = append(imapTargets, src) + case sourceTypeMSMail: + if !newGraphMailManager(state).HasToken(src.Identifier) { + fmt.Printf("Skipping %s (no Microsoft Graph token - run 'add-o365 %s --graph' first)\n", src.Identifier, src.Identifier) + continue + } + msmailTargets = append(msmailTargets, src) default: continue } } - if len(gmailTargets) == 0 && len(imapTargets) == 0 { + if len(gmailTargets) == 0 && len(imapTargets) == 0 && len(msmailTargets) == 0 { if len(syncErrors) > 0 { // Surface the collected errors (e.g. broken OAuth config). return fmt.Errorf("%s", syncErrors[0]) @@ -178,11 +197,25 @@ func runSyncIncrementalLocal(cmd *cobra.Command, args []string) error { break } fmt.Printf("Note: IMAP account %s uses folder-based sync. Unchanged folders are skipped when high water marks are available.\n\n", src.Identifier) - if err := runFullSync(ctx, s, getOAuthMgr, src); err != nil { + if err := runFullSync(ctx, s, getOAuthMgr, src, state); err != nil { syncErrors = append(syncErrors, fmt.Sprintf("%s: %v", src.Identifier, err)) } } + // Sync Microsoft Graph mail sources. The first run walks every folder. + for _, src := range msmailTargets { + if ctx.Err() != nil { + break + } + fmt.Printf("Syncing Microsoft Graph mail for %s\n", src.Identifier) + sum, err := runMSMailSync(ctx, s, src.Identifier, func(line string) { fmt.Println(line) }, state) + if err != nil { + syncErrors = append(syncErrors, fmt.Sprintf("%s: %v", src.Identifier, err)) + continue + } + writeMSMailSyncSummary(os.Stdout, src.Identifier, sum) + } + // Sync Gmail sources via incremental sync. for _, target := range gmailTargets { if ctx.Err() != nil { @@ -192,14 +225,14 @@ func runSyncIncrementalLocal(cmd *cobra.Command, args []string) error { syncErrors = append(syncErrors, target.email+": no source found - run 'sync-full' first") continue } - if err := runIncrementalSync(ctx, s, getOAuthMgr, target.source); err != nil { + if err := runIncrementalSync(ctx, s, getOAuthMgr, target.source, state); err != nil { syncErrors = append(syncErrors, fmt.Sprintf("%s: %v", target.email, err)) continue } } // Rebuild analytics cache. - cacheErr := rebuildCacheAfterWrite(dbPath) + cacheErr := rebuildCacheAfterManualSync(dbPath, state) if len(syncErrors) > 0 { fmt.Println() @@ -216,7 +249,15 @@ func runSyncIncrementalLocal(cmd *cobra.Command, args []string) error { return cacheErr } -func runIncrementalSync(ctx context.Context, s *store.Store, getOAuthMgr func(string) (*oauth.Manager, error), source *store.Source) error { +func runIncrementalSync(ctx context.Context, s *store.Store, getOAuthMgr func(string) (*oauth.Manager, error), source *store.Source, state *invocation) error { + if state == nil { + state = invocationFromContext(ctx) + } + if state == nil || state.cfg == nil || state.logger == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger if !source.SyncCursor.Valid || source.SyncCursor.String == "" { return errors.New("no history ID - run 'sync-full' first") } @@ -261,7 +302,7 @@ func runIncrementalSync(ctx context.Context, s *store.Store, getOAuthMgr func(st opts.AttachmentsDir = cfg.AttachmentsDir() // Create syncer with progress reporter - syncer := newMessageSyncer(client, s, opts). + syncer := newMessageSyncer(client, s, opts, state). WithLogger(logger). WithProgress(&CLIProgress{}) @@ -312,5 +353,5 @@ func init() { syncIncrementalCmd.Flags().Int64("source-id", 0, "Exact source ID to sync") syncIncrementalCmd.Flags().StringArrayVar(&syncFolders, "folder", []string{}, "IMAP folder to scan (repeatable)") syncIncrementalCmd.Flags().StringArrayVar(&syncSkipFolders, "skip-folder", []string{}, "IMAP folder to skip (repeatable)") - rootCmd.AddCommand(syncIncrementalCmd) + rootCmd.AddCommand(addManualSyncCacheFlags(syncIncrementalCmd)) } diff --git a/cmd/msgvault/cmd/sync_beeper.go b/cmd/msgvault/cmd/sync_beeper.go index 3bdbb6baf..3112fa405 100644 --- a/cmd/msgvault/cmd/sync_beeper.go +++ b/cmd/msgvault/cmd/sync_beeper.go @@ -5,14 +5,19 @@ import ( "errors" "fmt" "io" + "log/slog" "os" "os/signal" + "slices" "strings" + "sync" "syscall" "time" "github.com/spf13/cobra" "go.kenn.io/msgvault/internal/beeper" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/jobctx" "go.kenn.io/msgvault/internal/store" ) @@ -45,11 +50,16 @@ Examples: msgvault sync-beeper --full`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg if !isDaemonCLISubprocess() { return runDaemonCLICommandHTTPFromCobra(cmd, args) } - imp, accountIDs, dbPath, cleanup, err := openBeeperImporter(syncBeeperAccounts) + imp, accountIDs, dbPath, cleanup, err := openBeeperImporter(syncBeeperAccounts, state) if err != nil { return err } @@ -63,7 +73,7 @@ Examples: break } _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Syncing Beeper account %s\n", accountID) - opts := beeperImportOptions(accountID) + opts := beeperImportOptions(accountID, cfg) opts.Limit = syncBeeperLimit opts.Full = syncBeeperFull opts.NoMedia = opts.NoMedia || syncBeeperNoMedia @@ -84,7 +94,7 @@ Examples: // Successful accounts' messages must reach the analytics cache // regardless of interruptions or per-account failures. - cacheErr := rebuildCacheAfterWrite(dbPath) + cacheErr := rebuildCacheAfterManualSync(dbPath, state) if ctx.Err() != nil { _, _ = fmt.Fprintln(cmd.OutOrStdout(), "\nInterrupted — re-run sync-beeper to resume.") return cacheErr @@ -159,12 +169,15 @@ func writeBeeperMediaSkipSummary(out io.Writer, sum *beeper.ImportSummary) { if otherSkipped > 0 { _, _ = fmt.Fprintf(out, ", %d media skipped by policy", otherSkipped) } + if sum.AttachmentsUnavailable > 0 { + _, _ = fmt.Fprintf(out, ", %d media no longer available at source", sum.AttachmentsUnavailable) + } } // resolveBeeperSyncAccounts returns the Beeper accountIDs to sync: the // explicit flag values, or every registered beeper source that passes the // config include/exclude filters. -func resolveBeeperSyncAccounts(s *store.Store, flagAccounts []string) ([]string, error) { +func resolveBeeperSyncAccounts(s *store.Store, flagAccounts []string, cfg *config.Config) ([]string, error) { sources, err := s.ListSources(sourceTypeBeeper) if err != nil { return nil, fmt.Errorf("list beeper sources: %w", err) @@ -189,6 +202,9 @@ func resolveBeeperSyncAccounts(s *store.Store, flagAccounts []string) ([]string, return out, nil } var out []string + if cfg == nil { + return nil, errors.New("configuration is unavailable") + } for _, src := range sources { if cfg.Beeper.AccountIncluded(src.Identifier) { out = append(out, src.Identifier) @@ -200,9 +216,37 @@ func resolveBeeperSyncAccounts(s *store.Store, flagAccounts []string) ([]string, return out, nil } +// filterBeeperReanchorMarkedAccounts leaves accounts that scheduled sync may +// safely import. A source marked by an anchor mismatch stays out of the +// rotation until a manual import verifies it against the current installation. +func filterBeeperReanchorMarkedAccounts( + ctx context.Context, + s *store.Store, + accountIDs []string, +) ([]string, error) { + eligible := make([]string, 0, len(accountIDs)) + for _, accountID := range accountIDs { + source, err := s.GetSourceByTypeAndIdentifier(sourceTypeBeeper, accountID) + if err != nil { + return nil, fmt.Errorf("resolve beeper source %q for anchor marker: %w", accountID, err) + } + reason, marked, err := s.GetArchiveMarker(ctx, store.BeeperReanchorMarkerKey(source.ID)) + if err != nil { + return nil, err + } + if marked { + slog.Warn("skipping scheduled Beeper sync until manual anchor verification", + "account", accountID, "reason", reason) + continue + } + eligible = append(eligible, accountID) + } + return eligible, nil +} + // beeperImportOptions builds the config-derived import options shared by the // CLI and scheduler paths (flag overlays are applied by the CLI caller). -func beeperImportOptions(accountID string) beeper.ImportOptions { +func beeperImportOptions(accountID string, cfg *config.Config) beeper.ImportOptions { policy := cfg.Beeper.MediaPolicy(accountID) return beeper.ImportOptions{ AccountID: accountID, @@ -215,8 +259,12 @@ func beeperImportOptions(accountID string) beeper.ImportOptions { // openBeeperImporter performs the shared beeper-command prologue: open the // store, load the token, resolve the target accounts, and build the importer. // The returned cleanup closes the store. -func openBeeperImporter(flagAccounts []string) (imp *beeper.Importer, accountIDs []string, dbPath string, cleanup func(), err error) { - s, cleanup, err := openWritableStoreAndInitForIngest() +func openBeeperImporter(flagAccounts []string, state *invocation) (imp *beeper.Importer, accountIDs []string, dbPath string, cleanup func(), err error) { + if state == nil { + return nil, nil, "", nil, errors.New("configuration is unavailable") + } + cfg := state.cfg + s, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) if err != nil { return nil, nil, "", nil, err } @@ -225,12 +273,12 @@ func openBeeperImporter(flagAccounts []string) (imp *beeper.Importer, accountIDs cleanup() return nil, nil, "", nil, err } - accountIDs, err = resolveBeeperSyncAccounts(s, flagAccounts) + accountIDs, err = resolveBeeperSyncAccounts(s, flagAccounts, cfg) if err != nil { cleanup() return nil, nil, "", nil, err } - return beeper.NewImporter(s, beeperClient(token)), accountIDs, cfg.DatabaseDSN(), cleanup, nil + return beeper.NewImporter(s, beeperClient(cfg, token)), accountIDs, cfg.DatabaseDSN(), cleanup, nil } // withInterruptCancel derives a context canceled on SIGINT/SIGTERM, printing @@ -255,39 +303,119 @@ func withInterruptCancel(cmd *cobra.Command, note string) (context.Context, func // so one broken account does not starve the others, and the analytics cache is // rebuilt after any attempt so partial writes become visible too. func runConfiguredBeeperSync(ctx context.Context, s *store.Store) error { - token, err := beeper.LoadToken(cfg.TokensDir()) + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + accountIDs, err := resolveBeeperSyncAccounts(s, nil, cfg) if err != nil { return err } - accountIDs, err := resolveBeeperSyncAccounts(s, nil) + accountIDs, err = filterBeeperReanchorMarkedAccounts(ctx, s, accountIDs) if err != nil { return err } - imp := beeper.NewImporter(s, beeperClient(token)) - return runScheduledBeeperAttempts(ctx, accountIDs, - func(accountID string) error { - _, err := imp.Import(ctx, beeperImportOptions(accountID)) - return err + if len(accountIDs) == 0 { + return nil + } + token, err := beeper.LoadToken(cfg.TokensDir()) + if err != nil { + return err + } + imp := beeper.NewImporter(s, beeperClient(cfg, token)) + deadline := time.Now().Add(scheduledBeeperBudget - 15*time.Second) + shouldStop := func() bool { + return time.Now().After(deadline) || jobctx.PreemptionRequested(ctx) + } + return runScheduledBeeperAttempts(ctx, accountIDs, scheduledBeeperRotation, shouldStop, + func(accountID string) (bool, error) { + opts := beeperImportOptions(accountID, cfg) + opts.Scheduled = true + opts.ShouldStop = shouldStop + opts.StopAt = deadline + sum, err := imp.Import(ctx, opts) + return sum != nil && sum.Stopped, err }, func() error { return rebuildCacheAfterScheduledSync(context.WithoutCancel(ctx), "beeper") }, ) } +// scheduledBeeperBudget bounds one scheduled Beeper job across all accounts. +// Imports stop at the next chat or backfill-page boundary once it is spent +// and resume from their cursors on the next run. Fifteen seconds are reserved +// for bounded checkpoint and terminal sync writes after imports stop. +const scheduledBeeperBudget = 3 * time.Minute + +// scheduledBeeperRotation remembers where the last budget-limited scheduled +// run stopped, so accounts late in store order are not starved. +var scheduledBeeperRotation = &beeperAccountRotation{} + +// beeperAccountRotation orders scheduled Beeper accounts starting from the +// account after the previous run's last attempt. +type beeperAccountRotation struct { + mu sync.Mutex + next string +} + +func (r *beeperAccountRotation) order(accountIDs []string) []string { + r.mu.Lock() + next := r.next + r.mu.Unlock() + start := slices.Index(accountIDs, next) + if start <= 0 { + return slices.Clone(accountIDs) + } + return append(slices.Clone(accountIDs[start:]), accountIDs[:start]...) +} + +func (r *beeperAccountRotation) resumeAt(accountID string) { + r.mu.Lock() + r.next = accountID + r.mu.Unlock() +} + // runScheduledBeeperAttempts keeps per-account failures isolated while // rebuilding analytics after any import attempt, since even a failed or -// canceled attempt may have committed messages from healthy chats. -func runScheduledBeeperAttempts(ctx context.Context, accountIDs []string, attempt func(string) error, rebuild func() error) error { +// canceled attempt may have committed messages from healthy chats. An +// attempt that reports it stopped early, or a spent budget (shouldStop) +// before the next account, ends the job; the rotation starts the next run +// with the next account. +func runScheduledBeeperAttempts( + ctx context.Context, + accountIDs []string, + rotation *beeperAccountRotation, + shouldStop func() bool, + attempt func(string) (bool, error), + rebuild func() error, +) error { var errs []error attempted := 0 - for _, accountID := range accountIDs { + resumeAt := "" + ordered := rotation.order(accountIDs) + for idx, accountID := range ordered { if ctx.Err() != nil { + // Cancelled between accounts (a yield or shutdown): start here next. + resumeAt = accountID + break + } + if attempted > 0 && shouldStop != nil && shouldStop() { + resumeAt = accountID break } attempted++ - if err := attempt(accountID); err != nil { + stopped, err := attempt(accountID) + if err != nil { errs = append(errs, fmt.Errorf("beeper %s: %w", accountID, err)) } + if stopped || ctx.Err() != nil { + // A large account must not consume every tick. Its own cursors + // preserve progress while the next account gets a turn. + resumeAt = ordered[(idx+1)%len(ordered)] + break + } } + rotation.resumeAt(resumeAt) if attempted > 0 { if err := rebuild(); err != nil { errs = append(errs, err) @@ -300,5 +428,5 @@ func runScheduledBeeperAttempts(ctx context.Context, accountIDs []string, attemp } func init() { - rootCmd.AddCommand(newSyncBeeperCmd()) + rootCmd.AddCommand(addManualSyncCacheFlags(newSyncBeeperCmd())) } diff --git a/cmd/msgvault/cmd/sync_beeper_routing_test.go b/cmd/msgvault/cmd/sync_beeper_routing_test.go index e74cd8cc7..f6f0180ab 100644 --- a/cmd/msgvault/cmd/sync_beeper_routing_test.go +++ b/cmd/msgvault/cmd/sync_beeper_routing_test.go @@ -4,12 +4,14 @@ import ( "bytes" "context" "errors" + "log/slog" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.kenn.io/msgvault/internal/beeper" "go.kenn.io/msgvault/internal/clirun" + "go.kenn.io/msgvault/internal/store" "go.kenn.io/msgvault/internal/testutil" ) @@ -23,14 +25,38 @@ func TestResolveBeeperSyncAccountsValidatesAndDeduplicatesExplicitIDs(t *testing _, err = st.GetOrCreateSource(sourceTypeBeeper, "telegram") require.NoError(err) - accounts, err := resolveBeeperSyncAccounts(st, []string{"signal", "signal", "telegram"}) + accounts, err := resolveBeeperSyncAccounts(st, []string{"signal", "signal", "telegram"}, testConfigValue()) require.NoError(err) assert.Equal([]string{"signal", "telegram"}, accounts) - _, err = resolveBeeperSyncAccounts(st, []string{"signal", "typo"}) + _, err = resolveBeeperSyncAccounts(st, []string{"signal", "typo"}, testConfigValue()) require.ErrorContains(err, `beeper account "typo" is not registered`) } +func TestFilterBeeperReanchorMarkedAccounts(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := testutil.NewTestStore(t) + + blocked, err := st.GetOrCreateSource(sourceTypeBeeper, "blocked") + require.NoError(err) + _, err = st.GetOrCreateSource(sourceTypeBeeper, "ready") + require.NoError(err) + require.NoError(st.SetArchiveMarker(t.Context(), store.BeeperReanchorMarkerKey(blocked.ID), + "manual verification required")) + + previousLogger := slog.Default() + var logs bytes.Buffer + slog.SetDefault(slog.New(slog.NewTextHandler(&logs, nil))) + t.Cleanup(func() { slog.SetDefault(previousLogger) }) + + eligible, err := filterBeeperReanchorMarkedAccounts(t.Context(), st, []string{"blocked", "ready"}) + require.NoError(err) + assert.Equal([]string{"ready"}, eligible) + assert.Contains(logs.String(), "skipping scheduled Beeper sync until manual anchor verification") + assert.Contains(logs.String(), "account=blocked") +} + func TestScheduledBeeperAttemptsRebuildAfterPartialFailure(t *testing.T) { assert := assert.New(t) require := require.New(t) @@ -40,12 +66,14 @@ func TestScheduledBeeperAttemptsRebuildAfterPartialFailure(t *testing.T) { err := runScheduledBeeperAttempts( context.Background(), []string{"signal", "telegram"}, - func(accountID string) error { + &beeperAccountRotation{}, + nil, + func(accountID string) (bool, error) { attempted = append(attempted, accountID) if accountID == "signal" { - return errors.New("partial sync") + return false, errors.New("partial sync") } - return nil + return false, nil }, func() error { rebuilds++ @@ -65,7 +93,9 @@ func TestScheduledBeeperAttemptsReturnsRefreshError(t *testing.T) { err := runScheduledBeeperAttempts( context.Background(), []string{"signal"}, - func(string) error { return importErr }, + &beeperAccountRotation{}, + nil, + func(string) (bool, error) { return false, importErr }, func() error { return refreshErr }, ) @@ -73,6 +103,75 @@ func TestScheduledBeeperAttemptsReturnsRefreshError(t *testing.T) { require.ErrorIs(t, err, refreshErr) } +func TestScheduledBeeperAttemptsRotatesAfterStop(t *testing.T) { + assert := assert.New(t) + rotation := &beeperAccountRotation{} + accounts := []string{"a", "b", "c"} + noRebuild := func() error { return nil } + + var first []string + require.NoError(t, runScheduledBeeperAttempts(context.Background(), accounts, rotation, nil, + func(id string) (bool, error) { + first = append(first, id) + return id == "b", nil + }, noRebuild)) + assert.Equal([]string{"a", "b"}, first, "a stopped account ends the job") + + var second []string + require.NoError(t, runScheduledBeeperAttempts(context.Background(), accounts, rotation, nil, + func(id string) (bool, error) { + second = append(second, id) + return false, nil + }, noRebuild)) + assert.Equal([]string{"c", "a", "b"}, second, "the next run reaches accounts after the stopped account") + + var third []string + require.NoError(t, runScheduledBeeperAttempts(context.Background(), accounts, rotation, nil, + func(id string) (bool, error) { + third = append(third, id) + return false, nil + }, noRebuild)) + assert.Equal([]string{"a", "b", "c"}, third, "a complete run restarts from the first account") +} + +func TestScheduledBeeperAttemptsStopBetweenAccounts(t *testing.T) { + assert := assert.New(t) + rotation := &beeperAccountRotation{} + var attempted []string + stopped := false + require.NoError(t, runScheduledBeeperAttempts(context.Background(), []string{"a", "b", "c"}, rotation, + func() bool { return stopped }, + func(id string) (bool, error) { + attempted = append(attempted, id) + stopped = true // budget spent by the first account + return false, nil + }, func() error { return nil })) + assert.Equal([]string{"a"}, attempted) + assert.Equal([]string{"b", "c", "a"}, rotation.order([]string{"a", "b", "c"}), + "the next run starts with the first account not reached") +} + +func TestScheduledBeeperAttemptsResumeAtAccountInterruptedByYield(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + rotation := &beeperAccountRotation{} + ctx, cancel := context.WithCancel(context.Background()) + var attempted []string + err := runScheduledBeeperAttempts(ctx, []string{"a", "b", "c"}, rotation, nil, + func(id string) (bool, error) { + attempted = append(attempted, id) + if id == "b" { + cancel() // an API request made the scheduled job yield mid-account + return false, context.Canceled + } + return false, nil + }, func() error { return nil }) + require.ErrorIs(err, context.Canceled) + assert.Equal([]string{"a", "b"}, attempted) + assert.Equal([]string{"c", "a", "b"}, rotation.order([]string{"a", "b", "c"}), + "the next run reaches accounts after the interrupted account") +} + func TestPrintBeeperSummaryReportsIdentityReplayPending(t *testing.T) { assert := assert.New(t) cmd := newSyncBeeperCmd() @@ -117,10 +216,12 @@ func TestSyncBeeperCommandUsesDaemonRunner(t *testing.T) { "--limit=25", }, req.Args, "args") }, `{"type":"stdout","data":"Syncing Beeper account signal\n"}`, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx var stdout bytes.Buffer cmd := newSyncBeeperCmd() + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetErr(&stdout) cmd.SetArgs([]string{ @@ -142,11 +243,13 @@ func TestAddBeeperCommandForwardsTokenEnv(t *testing.T) { assert.Equal([]string{"add-beeper"}, req.Args, "args") assert.Equal("test-token-123", req.Env[clirun.EnvBeeperToken], "token env forwarded") }, `{"type":"stdout","data":"Added signal\n"}`, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx t.Setenv(clirun.EnvBeeperToken, "test-token-123") var stdout bytes.Buffer cmd := newAddBeeperCmd() + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetErr(&stdout) cmd.SetArgs([]string{}) @@ -157,3 +260,16 @@ func TestAddBeeperCommandForwardsTokenEnv(t *testing.T) { assert.Equal(1, int(requests.Load()), "runner endpoint calls") assert.Contains(stdout.String(), "Added signal") } + +func TestScheduledBeeperAttemptsRotateWhenEveryAccountStops(t *testing.T) { + rotation := &beeperAccountRotation{} + var attempted []string + for range 3 { + require.NoError(t, runScheduledBeeperAttempts(t.Context(), []string{"a", "b", "c"}, rotation, nil, + func(id string) (bool, error) { + attempted = append(attempted, id) + return true, nil + }, func() error { return nil })) + } + assert.Equal(t, []string{"a", "b", "c"}, attempted, "each stopped account yields the next tick") +} diff --git a/cmd/msgvault/cmd/sync_cache_refresh.go b/cmd/msgvault/cmd/sync_cache_refresh.go new file mode 100644 index 000000000..371d44012 --- /dev/null +++ b/cmd/msgvault/cmd/sync_cache_refresh.go @@ -0,0 +1,110 @@ +package cmd + +import ( + "errors" + "fmt" + + "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/store" +) + +// addManualSyncCacheFlags is called by each archive-writing sync command. +// Cobra forwards changed local flags through the generic daemon CLI proxy. +func addManualSyncCacheFlags(cmd *cobra.Command) *cobra.Command { + cmd.Flags().Bool("build-cache", false, "refresh analytics cache after this sync") + cmd.Flags().Bool("no-build-cache", false, "skip analytics cache refresh after this sync") + cmd.MarkFlagsMutuallyExclusive("build-cache", "no-build-cache") + return cmd +} + +func manualSyncCacheFlags(cmd *cobra.Command) (force, skip bool, err error) { + if cmd == nil || cmd.Flags().Lookup("build-cache") == nil { + return false, false, nil + } + force, err = cmd.Flags().GetBool("build-cache") + if err != nil { + return false, false, fmt.Errorf("read --build-cache: %w", err) + } + skip, err = cmd.Flags().GetBool("no-build-cache") + if err != nil { + return false, false, fmt.Errorf("read --no-build-cache: %w", err) + } + if force && skip { + return false, false, errors.New("--build-cache and --no-build-cache are mutually exclusive") + } + return force, skip, nil +} + +func manualSyncCLICommand(args []string) bool { + if len(args) == 0 { + return false + } + switch args[0] { + case "sync-slack", "sync-teams", "sync-beeper", "sync-discord", "sync-circleback", "sync-plaud", + "sync-notion-meetings", "sync-granola", "sync-calendar", "sync-synctech-sms": + if args[0] == "sync-circleback" || args[0] == "sync-plaud" || args[0] == "sync-notion-meetings" { + for _, arg := range args[1:] { + if arg == "--probe" || arg == "--probe=true" { + return false + } + } + } + return true + default: + return false + } +} + +func manualSyncCacheFlagValues(args []string) (force, skip bool) { + for _, arg := range args { + switch arg { + case "--build-cache", "--build-cache=true": + force = true + case "--no-build-cache", "--no-build-cache=true": + skip = true + case "--build-cache=false": + force = false + case "--no-build-cache=false": + skip = false + } + } + return force, skip +} + +// queueCacheRefreshAfterManualSync runs in the daemon, after its CLI child +// returns. The background builder validates files before applying the rebuild +// interval, so a recent marker cannot postpone repair of a damaged publication. +func (a *storeAPIAdapter) queueCacheRefreshAfterManualSync(force, skip bool) error { + if a == nil || a.cacheJobs == nil || skip { + return nil + } + if a.config == nil { + return errors.New("configuration is unavailable") + } + cfg := a.config + logger := repairLogger(a.logger) + if !force && !cfg.Analytics.AutoBuildCache { + return nil + } + if a.cacheJobs.ctx.Err() != nil { + return nil //nolint:nilerr // Shutdown must not turn a completed sync into a failure. + } + if store.IsPostgresURL(cfg.DatabaseDSN()) { + return nil + } + mode := buildCacheModeScheduledAuto + if force { + mode = buildCacheModeAuto + } + job, err := a.cacheJobs.acceptAfterWrite(mode) + if err != nil { + if a.cacheJobs.ctx.Err() != nil { + return nil //nolint:nilerr // Shutdown must not turn a completed sync into a failure. + } + return fmt.Errorf("queue analytics cache build after sync: %w", err) + } + if job.JobID != "" { + logger.Info("queued analytics cache build after manual sync", "job_id", job.JobID) + } + return nil +} diff --git a/cmd/msgvault/cmd/sync_cache_refresh_test.go b/cmd/msgvault/cmd/sync_cache_refresh_test.go new file mode 100644 index 000000000..df6e9d812 --- /dev/null +++ b/cmd/msgvault/cmd/sync_cache_refresh_test.go @@ -0,0 +1,127 @@ +package cmd + +import ( + "context" + "database/sql" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/store" +) + +func TestManualSyncRefreshVerifiesConversationOnlyChangesInBackground(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + c, s := openTestDaemonAnalyticsStore(t) + c.Analytics.AutoBuildCache = true + c.Analytics.MinRebuildInterval = 0 + source, err := s.GetOrCreateSource("gmail", "user@example.test") + require.NoError(err) + conversationID, err := s.EnsureConversationWithType(source.ID, "thread-1", "email_thread", "Original title") + require.NoError(err) + _, err = s.UpsertMessage(&store.Message{ + SourceID: source.ID, SourceMessageID: "message-1", ConversationID: conversationID, + MessageType: "email", SentAt: sql.NullTime{Time: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), Valid: true}, + }) + require.NoError(err) + _, err = buildCache(c.DatabaseDSN(), c.AnalyticsDir(), true) + require.NoError(err) + _, err = s.EnsureConversationWithType(source.ID, "thread-1", "email_thread", "Updated title") + require.NoError(err) + light, err := cacheNeedsBuildForServing(t.Context(), c.DatabaseDSN(), c.AnalyticsDir()) + require.NoError(err) + require.False(light.NeedsBuild, "conversation titles have no indexed staleness signal") + + ctx, cancel := context.WithCancel(t.Context()) + verified := make(chan cacheStaleness, 1) + jobs := newCacheBuildJobs(ctx, nil, func(context.Context, buildCacheMode) error { + verified <- cacheNeedsBuild(c.DatabaseDSN(), c.AnalyticsDir()) + return nil + }) + t.Cleanup(func() { + cancel() + waitCtx, stop := context.WithTimeout(context.Background(), serveLifecycleTestTimeout) + defer stop() + require.True(jobs.waitContext(waitCtx), "background verification must finish before store cleanup") + }) + adapter := &storeAPIAdapter{store: s, config: c, cacheJobs: jobs} + require.NoError(adapter.queueCacheRefreshAfterManualSync(false, false)) + waitCtx, stop := context.WithTimeout(t.Context(), serveLifecycleTestTimeout) + defer stop() + require.True(jobs.waitContext(waitCtx)) + select { + case full := <-verified: + assert.True(full.NeedsBuild) + assert.True(full.HasConversationTypeDrift) + assert.Contains(full.Reason, "conversation metadata changed") + default: + require.FailNow("manual sync did not queue full cache verification") + } +} + +func TestManualSyncRefreshChecksFilesBeforeThrottling(t *testing.T) { + for _, damaged := range []bool{false, true} { + name := "usable" + if damaged { + name = "missing shard" + } + t.Run(name, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + c, s := openPublishedQueryTestStore(t) + c.Analytics.AutoBuildCache = true + c.Analytics.MinRebuildInterval = 6 * time.Hour + _, err := s.DB().Exec(`INSERT INTO messages + (id, source_id, source_message_id, conversation_id, message_type, sent_at) + VALUES (2, 1, 'message-2', 1, 'email', '2024-01-02 00:00:00')`) + require.NoError(err) + if damaged { + shards, err := filepath.Glob(filepath.Join(c.AnalyticsDir(), "messages", "*", "*.parquet")) + require.NoError(err) + require.Len(shards, 1) + require.NoError(os.Remove(shards[0])) + } + ctx, cancel := context.WithCancel(t.Context()) + completed := make(chan *buildResult, 1) + jobs := newCacheBuildJobs(ctx, nil, func(context.Context, buildCacheMode) error { + result, err := buildCacheScheduled(c.DatabaseDSN(), c.AnalyticsDir(), + c.Analytics.MinRebuildInterval, time.Now) + completed <- result + return err + }) + t.Cleanup(func() { + cancel() + waitCtx, stop := context.WithTimeout(context.Background(), serveLifecycleTestTimeout) + defer stop() + require.True(jobs.waitContext(waitCtx)) + }) + adapter := &storeAPIAdapter{store: s, config: c, cacheJobs: jobs} + require.NoError(adapter.queueCacheRefreshAfterManualSync(false, false)) + waitCtx, stop := context.WithTimeout(t.Context(), serveLifecycleTestTimeout) + defer stop() + require.True(jobs.waitContext(waitCtx)) + select { + case result := <-completed: + require.NotNil(result) + assert.Equal(!damaged, result.Skipped) + default: + require.FailNow("manual sync did not queue cache verification") + } + engine, err := openDaemonDuckDBEngine(c, s) + require.NoError(err) + t.Cleanup(func() { _ = engine.Close() }) + result, err := engine.QuerySQL(t.Context(), "SELECT COUNT(*) FROM messages") + require.NoError(err) + require.Len(result.Rows, 1) + wantCount := 1 + if damaged { + wantCount = 2 + } + assert.EqualValues(wantCount, result.Rows[0][0]) + }) + } +} diff --git a/cmd/msgvault/cmd/sync_discord.go b/cmd/msgvault/cmd/sync_discord.go index a6d375a15..dc05fd660 100644 --- a/cmd/msgvault/cmd/sync_discord.go +++ b/cmd/msgvault/cmd/sync_discord.go @@ -40,6 +40,9 @@ argument is supplied. Guilds run sequentially in stable source order, and one guild failure does not prevent later guilds from running.`, Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + if invocationFromContext(cmd.Context()) != nil && deps.bind != nil { + deps = deps.bind(cmd.Context()) + } selector := "" if len(args) == 1 { selector = args[0] @@ -49,6 +52,7 @@ guild failure does not prevent later guilds from running.`, } cmd.Flags().BoolVar(&opts.Full, "full", false, "ignore stored cursors and re-fetch all available history") cmd.Flags().StringVar(&opts.After, "after", "", "exclusive lower bound (YYYY-MM-DD or RFC3339)") + addManualSyncCacheFlags(cmd) return cmd } diff --git a/cmd/msgvault/cmd/sync_http.go b/cmd/msgvault/cmd/sync_http.go index aee2ed8c7..65b4241b9 100644 --- a/cmd/msgvault/cmd/sync_http.go +++ b/cmd/msgvault/cmd/sync_http.go @@ -9,20 +9,27 @@ import ( "github.com/mattn/go-isatty" "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/daemonclient" "golang.org/x/oauth2" ) func runSyncIncrementalHTTP(cmd *cobra.Command, args []string) error { + force, skip, flagErr := manualSyncCacheFlags(cmd) + if flagErr != nil { + return usageErr(cmd, flagErr) + } selector, _, err := syncSourceSelector(cmd, args) if err != nil { return usageErr(cmd, err) } req := daemonclient.CLISyncRequest{ - Folders: parseFolderFilter(syncFolders), - SkipFolders: parseFolderFilter(syncSkipFolders), - SourceID: selector.SourceID, - SourceIDSet: selector.SourceIDSet, + BuildCache: force, + NoBuildCache: skip, + Folders: parseFolderFilter(syncFolders), + SkipFolders: parseFolderFilter(syncSkipFolders), + SourceID: selector.SourceID, + SourceIDSet: selector.SourceIDSet, } if len(args) == 1 { req.Email = selector.Account @@ -31,21 +38,27 @@ func runSyncIncrementalHTTP(cmd *cobra.Command, args []string) error { } func runSyncFullHTTP(cmd *cobra.Command, args []string) error { + force, skip, flagErr := manualSyncCacheFlags(cmd) + if flagErr != nil { + return usageErr(cmd, flagErr) + } selector, _, err := syncSourceSelector(cmd, args) if err != nil { return usageErr(cmd, err) } req := daemonclient.CLISyncRequest{ - Full: true, - Query: syncQuery, - NoResume: syncNoResume, - Before: syncBefore, - After: syncAfter, - Limit: syncLimit, - Folders: parseFolderFilter(syncFolders), - SkipFolders: parseFolderFilter(syncSkipFolders), - SourceID: selector.SourceID, - SourceIDSet: selector.SourceIDSet, + BuildCache: force, + NoBuildCache: skip, + Full: true, + Query: syncQuery, + NoResume: syncNoResume, + Before: syncBefore, + After: syncAfter, + Limit: syncLimit, + Folders: parseFolderFilter(syncFolders), + SkipFolders: parseFolderFilter(syncSkipFolders), + SourceID: selector.SourceID, + SourceIDSet: selector.SourceIDSet, } if len(args) == 1 { req.Email = selector.Account @@ -60,7 +73,7 @@ func runSyncHTTP(cmd *cobra.Command, req daemonclient.CLISyncRequest) error { } defer func() { _ = st.Close() }() - if err := preflightReauth(cmd.Context(), buildSyncPreflight(st, info), req.Email, req.SourceID); err != nil { + if err := preflightReauth(cmd.Context(), buildSyncPreflight(st, info, invocationFromCommand(cmd)), req.Email, req.SourceID); err != nil { return err } @@ -118,13 +131,25 @@ type preflightConfig struct { // buildSyncPreflight wires the production preflight config from global config, // the daemon client, and the resolved store endpoint. -func buildSyncPreflight(st *daemonclient.Client, info HTTPStoreInfo) preflightConfig { - getMgr := oauthManagerCache() +func buildSyncPreflight(st *daemonclient.Client, info HTTPStoreInfo, state *invocation) preflightConfig { + getMgr := oauthManagerCache(state) + var cfg *config.Config + if state != nil { + cfg = state.cfg + } + var oauthConfigured bool + var serviceAccountKey func(string) string + if cfg != nil { + oauthConfigured = cfg.OAuth.HasAnyConfig() + serviceAccountKey = cfg.OAuth.ServiceAccountKeyFor + } else { + serviceAccountKey = func(string) string { return "" } + } return preflightConfig{ Local: info.Kind == HTTPStoreLocalDaemon, Interactive: isatty.IsTerminal(os.Stdin.Fd()) || isatty.IsCygwinTerminal(os.Stdin.Fd()), - OAuthConfigured: cfg.OAuth.HasAnyConfig(), + OAuthConfigured: oauthConfigured, Out: os.Stdout, ListGmailAccounts: func(ctx context.Context) ([]preflightAccount, error) { accounts, err := st.GetCLIAccounts(ctx) @@ -145,7 +170,7 @@ func buildSyncPreflight(st *daemonclient.Client, info HTTPStoreInfo) preflightCo } return gmail, nil }, - ServiceAccountKey: cfg.OAuth.ServiceAccountKeyFor, + ServiceAccountKey: serviceAccountKey, ManagerFor: func(appName string) (preflightReauthManager, error) { mgr, err := getMgr(appName) if err != nil { diff --git a/cmd/msgvault/cmd/sync_http_test.go b/cmd/msgvault/cmd/sync_http_test.go index 7c0853260..79118a091 100644 --- a/cmd/msgvault/cmd/sync_http_test.go +++ b/cmd/msgvault/cmd/sync_http_test.go @@ -37,12 +37,13 @@ func TestSyncUsesConfiguredRemoteHTTPAndPreservesOutput(t *testing.T) { })) t.Cleanup(server.Close) - configureRemoteSyncTest(t, server.URL) + testCtx := configureRemoteSyncTest(t, server.URL) resetSyncFullFlagsForTest(t) var stdout bytes.Buffer var stderr bytes.Buffer cmd := &cobra.Command{Use: syncIncrementalCmd.Use, Args: syncIncrementalCmd.Args, RunE: syncIncrementalCmd.RunE} + cmd.SetContext(testCtx) cmd.Flags().StringArrayVar(&syncFolders, "folder", []string{}, "IMAP folders to include") cmd.Flags().StringArrayVar(&syncSkipFolders, "skip-folder", []string{}, "IMAP folders to exclude") cmd.SetArgs([]string{ @@ -74,10 +75,11 @@ func TestSyncSourceIDUsesConfiguredRemoteHTTP(t *testing.T) { _, _ = w.Write([]byte(`{"type":"complete"}` + "\n")) })) t.Cleanup(server.Close) - configureRemoteSyncTest(t, server.URL) + testCtx := configureRemoteSyncTest(t, server.URL) resetSyncFullFlagsForTest(t) cmd := &cobra.Command{Use: syncIncrementalCmd.Use, Args: syncIncrementalCmd.Args, RunE: syncIncrementalCmd.RunE} + cmd.SetContext(testCtx) cmd.Flags().Int64("source-id", 0, "Exact source ID") cmd.SetArgs([]string{"--source-id", "42"}) require.NoError(t, cmd.Execute()) @@ -126,12 +128,13 @@ func TestSyncFullUsesConfiguredRemoteHTTPAndPreservesOutput(t *testing.T) { })) t.Cleanup(server.Close) - configureRemoteSyncTest(t, server.URL) + testCtx := configureRemoteSyncTest(t, server.URL) resetSyncFullFlagsForTest(t) var stdout bytes.Buffer var stderr bytes.Buffer cmd := &cobra.Command{Use: syncFullCmd.Use, Args: syncFullCmd.Args, RunE: syncFullCmd.RunE} + cmd.SetContext(testCtx) cmd.Flags().StringVar(&syncQuery, "query", "", "Gmail search query") cmd.Flags().BoolVar(&syncNoResume, "noresume", false, "Force fresh sync") cmd.Flags().StringVar(&syncBefore, "before", "", "Only messages before this date") @@ -155,11 +158,12 @@ func TestSyncFullUsesConfiguredRemoteHTTPAndPreservesOutput(t *testing.T) { assert.Equal("sync-full warning\n", stderr.String()) } -func configureRemoteSyncTest(t *testing.T, remoteURL string) { +func configureRemoteSyncTest(t *testing.T, remoteURL string) context.Context { t.Helper() + logger := testLoggerValue() dataDir := t.TempDir() - withStoreResolverConfig(t, &config.Config{ + testCtx := withStoreResolverConfig(t, &config.Config{ HomeDir: dataDir, Data: config.DataConfig{DataDir: dataDir}, Remote: config.RemoteConfig{ @@ -170,6 +174,7 @@ func configureRemoteSyncTest(t *testing.T, remoteURL string) { oldLogger := logger logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) t.Cleanup(func() { logger = oldLogger }) + return testCtx } // newValidPreflightManager returns a mock whose token source always succeeds. diff --git a/cmd/msgvault/cmd/sync_msmail.go b/cmd/msgvault/cmd/sync_msmail.go new file mode 100644 index 000000000..402b3f31c --- /dev/null +++ b/cmd/msgvault/cmd/sync_msmail.go @@ -0,0 +1,80 @@ +package cmd + +import ( + "context" + "fmt" + "io" + "time" + + "go.kenn.io/msgvault/internal/microsoft" + "go.kenn.io/msgvault/internal/msmail" + "go.kenn.io/msgvault/internal/store" +) + +// msmailQPS is the Graph mail request rate. Microsoft documents 10,000 +// requests per 10 minutes per mailbox; 15 per second stays under it. +const msmailQPS = 15 + +func newGraphMailManager(state *invocation) *microsoft.GraphManager { + cfg := state.cfg + return microsoft.NewGraphMailManager( + cfg.Microsoft.ClientID, + cfg.Microsoft.EffectiveTenantID(), + cfg.Microsoft.EffectiveRedirectURI(), + cfg.TokensDir(), + state.logger, + ) +} + +// newGraphMailWriteManager requests Mail.ReadWrite on top of the sync scopes. +// delete-staged uses it. +func newGraphMailWriteManager(state *invocation) *microsoft.GraphManager { + cfg := state.cfg + return microsoft.NewGraphMailWriteManager( + cfg.Microsoft.ClientID, + cfg.Microsoft.EffectiveTenantID(), + cfg.Microsoft.EffectiveRedirectURI(), + cfg.TokensDir(), + state.logger, + ) +} + +// runMSMailSync syncs one Graph mail account. The first run downloads every +// folder; later runs fetch only the changes. +func runMSMailSync(ctx context.Context, s *store.Store, email string, progress func(string), state *invocation) (*msmail.Summary, error) { + cfg := state.cfg + tokenFn, err := newGraphMailManager(state).TokenSource(ctx, email) + if err != nil { + return nil, err + } + client := msmail.NewClient(msmail.GraphBaseURL, tokenFn, msmailQPS) + return msmail.Import(ctx, s, client, msmail.Options{ + Email: email, + AttachmentsDir: cfg.AttachmentsDir(), + Progress: progress, + }, state.logger) +} + +// runScheduledMSMailSync is the daemon path. Like runScheduledTeamsSync, it +// seeds the "me" identity and runs pending migrations before the sync. +func runScheduledMSMailSync(ctx context.Context, src *store.Source, s *store.Store, state *invocation) error { + confirmDefaultIdentity(io.Discard, s, src.ID, src.Identifier, src.Identifier, "account-identifier", state.logger) + if err := runPostSourceCreateMigrationsForInvocation(s, state); err != nil { + return fmt.Errorf("post-source-create migrations: %w", err) + } + _, err := runMSMailSync(ctx, s, src.Identifier, nil, state) + return err +} + +func writeMSMailSyncSummary(out io.Writer, email string, sum *msmail.Summary) { + _, _ = fmt.Fprintf(out, "\nMicrosoft Graph mail sync complete for %s\n", email) + _, _ = fmt.Fprintf(out, " Duration: %s\n", sum.Duration.Round(time.Second)) + _, _ = fmt.Fprintf(out, " Folders: %d\n", sum.Folders) + _, _ = fmt.Fprintf(out, " Messages added: %d\n", sum.Added) + _, _ = fmt.Fprintf(out, " Updated: %d\n", sum.Updated) + _, _ = fmt.Fprintf(out, " Moved: %d\n", sum.Moved) + _, _ = fmt.Fprintf(out, " Deleted: %d\n", sum.Deleted) + if sum.Errors > 0 { + _, _ = fmt.Fprintf(out, " Errors: %d\n", sum.Errors) + } +} diff --git a/cmd/msgvault/cmd/sync_slack.go b/cmd/msgvault/cmd/sync_slack.go index 5a1190905..0284f18e1 100644 --- a/cmd/msgvault/cmd/sync_slack.go +++ b/cmd/msgvault/cmd/sync_slack.go @@ -5,10 +5,14 @@ import ( "errors" "fmt" "io" + "slices" "strings" + "sync" "time" "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/jobctx" "go.kenn.io/msgvault/internal/slack" "go.kenn.io/msgvault/internal/store" "go.kenn.io/msgvault/internal/textutil" @@ -23,14 +27,16 @@ var ( ) func newSyncSlackCmd() *cobra.Command { + var syncPrivateChannels, syncDMs, syncGroupDMs bool cmd := &cobra.Command{ Use: "sync-slack [team-id]", Short: "Sync Slack conversations (channels, group DMs, DMs)", Long: `Sync Slack conversations for registered workspaces. The first run backfills each conversation's full history; later runs are -incremental, fetching new messages and discovering late thread replies with -search plus periodic canonical audits. Backfills and audits are resumable: +incremental. Tokens with search:read use search plus periodic history audits +to discover late thread replies. Without it, each sync revisits conversation +history for replies on old threads. Backfills and audits are resumable: re-run after an interruption and the sync continues where it stopped. Requires a workspace added with 'add-slack'. Use --full to start a repair @@ -45,6 +51,11 @@ Examples: msgvault sync-slack --full`, Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg if !isDaemonCLISubprocess() { return runDaemonCLICommandHTTPFromCobra(cmd, args) } @@ -53,7 +64,7 @@ Examples: if len(args) > 0 { flagTeam = args[0] } - s, cleanup, err := openWritableStoreAndInitForIngest() + s, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) if err != nil { return err } @@ -82,7 +93,8 @@ Examples: continue } imp := slack.NewImporter(s, slack.NewClient("", token), teamID) - opts := slackImportOptions(teamID, userID) + opts := slackImportOptions(teamID, userID, cfg) + applySlackConversationOverrides(cmd, &opts, syncPrivateChannels, syncDMs, syncGroupDMs) opts.Limit = syncSlackLimit opts.Full = syncSlackFull opts.NoThreads = syncSlackNoThreads @@ -104,7 +116,7 @@ Examples: // Successful workspaces' messages must reach the analytics cache // regardless of interruptions or per-workspace failures. - cacheErr := rebuildCacheAfterWrite(cfg.DatabaseDSN()) + cacheErr := rebuildCacheAfterManualSync(cfg.DatabaseDSN(), state) if ctx.Err() != nil { _, _ = fmt.Fprintln(cmd.OutOrStdout(), "\nInterrupted — re-run sync-slack to resume.") } else if len(syncErrors) > 0 { @@ -121,9 +133,24 @@ Examples: cmd.Flags().BoolVar(&syncSlackNoThreads, "no-threads", false, "skip thread-reply fetching (backfill inline fetches and the reply sweep) for this run") cmd.Flags().BoolVar(&syncSlackMaintenance, "maintenance", false, "run the maintenance rescan: repair edits and reaction changes on recent messages (archives ignore post-capture mutations by default)") cmd.Flags().BoolVar(&syncSlackNoMedia, "no-media", false, "skip file downloads for this run (files are recorded as pending; backfill-slack-media fetches them later)") + cmd.Flags().BoolVar(&syncDMs, "dms", true, "include one-to-one DMs for this run, overriding config (true or false)") + cmd.Flags().BoolVar(&syncPrivateChannels, "private-channels", true, "include private channels for this run, overriding config (true or false)") + cmd.Flags().BoolVar(&syncGroupDMs, "group-dms", true, "include group DMs for this run, overriding config (true or false)") return cmd } +func applySlackConversationOverrides(cmd *cobra.Command, opts *slack.ImportOptions, privateChannels, dms, groupDMs bool) { + if cmd.Flags().Changed("private-channels") { + opts.ExcludePrivateChannels = !privateChannels + } + if cmd.Flags().Changed("dms") { + opts.ExcludeDMs = !dms + } + if cmd.Flags().Changed("group-dms") { + opts.ExcludeGroupDMs = !groupDMs + } +} + func writeSlackProgress(out io.Writer, line string) { _, _ = fmt.Fprintln(out, " "+textutil.SanitizeTerminal(line)) } @@ -199,16 +226,19 @@ func slackSyncExit(ctxErr error, syncErrors []string, cacheErr error) error { // slackImportOptions builds the config-derived import options shared by the // CLI and scheduler paths (flag overlays are applied by the CLI caller). -func slackImportOptions(teamID, userID string) slack.ImportOptions { +func slackImportOptions(teamID, userID string, cfg *config.Config) slack.ImportOptions { policy := cfg.Slack.MediaPolicy(teamID) return slack.ImportOptions{ - TeamID: teamID, - UserID: userID, - AttachmentsDir: cfg.AttachmentsDir(), - MaxMediaBytes: policy.MaxBytes, - MediaPolicy: policy, - IncludeChannels: cfg.Slack.Channels, - ExcludeChannels: cfg.Slack.ExcludeChannels, + TeamID: teamID, + UserID: userID, + AttachmentsDir: cfg.AttachmentsDir(), + MaxMediaBytes: policy.MaxBytes, + MediaPolicy: policy, + IncludeChannels: cfg.Slack.Channels, + ExcludeChannels: cfg.Slack.ExcludeChannels, + ExcludePrivateChannels: !cfg.Slack.PrivateChannelsEnabled(), + ExcludeDMs: !cfg.Slack.DMsEnabled(), + ExcludeGroupDMs: !cfg.Slack.GroupDMsEnabled(), } } @@ -216,37 +246,103 @@ func slackImportOptions(teamID, userID string) slack.ImportOptions { // sync of every registered Slack workspace. Per-workspace failures are // collected so one broken workspace does not starve the others. func runConfiguredSlackSync(ctx context.Context, s *store.Store) error { + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg sources, err := resolveSlackSyncSources(s, "") if err != nil { return err } - var errs []error - attempted := 0 - for _, src := range sources { - if ctx.Err() != nil { + return runScheduledSlackAttempts(ctx, sources, scheduledSlackRotation, + func(src *store.Source) (bool, error) { + teamID, userID, ok := splitSlackIdentifier(src.Identifier) + if !ok { + return false, fmt.Errorf("slack %s: malformed identifier", src.Identifier) + } + token, terr := slack.LoadToken(cfg.TokensDir(), teamID, userID) + if terr != nil { + return false, fmt.Errorf("slack %s: %w", teamID, terr) + } + imp := slack.NewImporter(s, slack.NewClient("", token), teamID) + if _, serr := imp.Import(ctx, slackImportOptions(teamID, userID, cfg)); serr != nil { + return true, fmt.Errorf("slack %s: %w", teamID, serr) + } + return true, nil + }, func() error { + // Rebuild analytics after any import attempt: even a failed or + // canceled attempt may have committed messages from healthy channels. + return rebuildCacheAfterScheduledSync(context.WithoutCancel(ctx), "slack") + }) +} + +// scheduledSlackRotation remembers where a preempted scheduled sync stopped, +// so one long-running workspace cannot starve workspaces later in store order. +var scheduledSlackRotation = &slackWorkspaceRotation{} + +type slackWorkspaceRotation struct { + mu sync.Mutex + next string +} + +func (r *slackWorkspaceRotation) order(sources []*store.Source) []*store.Source { + r.mu.Lock() + next := r.next + r.mu.Unlock() + start := -1 + for idx, src := range sources { + if src.Identifier == next { + start = idx break } - teamID, userID, ok := splitSlackIdentifier(src.Identifier) - if !ok { - errs = append(errs, fmt.Errorf("slack %s: malformed identifier", src.Identifier)) - continue + } + if start <= 0 { + return slices.Clone(sources) + } + return append(slices.Clone(sources[start:]), sources[:start]...) +} + +func (r *slackWorkspaceRotation) resumeAt(identifier string) { + r.mu.Lock() + r.next = identifier + r.mu.Unlock() +} + +// runScheduledSlackAttempts isolates workspace failures, rebuilds analytics +// after import attempts, and resumes after the workspace interrupted by a +// scheduler yield. A cooperative preemption request also ends the current run +// after the current workspace has had a chance to checkpoint. +func runScheduledSlackAttempts( + ctx context.Context, + sources []*store.Source, + rotation *slackWorkspaceRotation, + attempt func(*store.Source) (bool, error), + rebuild func() error, +) error { + var errs []error + attempted := false + resumeAt := "" + ordered := rotation.order(sources) + for idx, src := range ordered { + if ctx.Err() != nil || jobctx.PreemptionRequested(ctx) { + resumeAt = src.Identifier + break } - token, terr := slack.LoadToken(cfg.TokensDir(), teamID, userID) - if terr != nil { - errs = append(errs, fmt.Errorf("slack %s: %w", teamID, terr)) - continue + started, err := attempt(src) + attempted = attempted || started + if err != nil { + errs = append(errs, err) } - attempted++ - imp := slack.NewImporter(s, slack.NewClient("", token), teamID) - if _, serr := imp.Import(ctx, slackImportOptions(teamID, userID)); serr != nil { - errs = append(errs, fmt.Errorf("slack %s: %w", teamID, serr)) + if ctx.Err() != nil || jobctx.PreemptionRequested(ctx) { + resumeAt = ordered[(idx+1)%len(ordered)].Identifier + break } } - // Rebuild analytics after any attempt: even a failed or canceled attempt - // may have committed messages from healthy conversations. - if attempted > 0 { - if rerr := rebuildCacheAfterScheduledSync(context.WithoutCancel(ctx), "slack"); rerr != nil { - errs = append(errs, rerr) + rotation.resumeAt(resumeAt) + if attempted { + if err := rebuild(); err != nil { + errs = append(errs, err) } } if ctx.Err() != nil { @@ -256,5 +352,5 @@ func runConfiguredSlackSync(ctx context.Context, s *store.Store) error { } func init() { - rootCmd.AddCommand(newSyncSlackCmd()) + rootCmd.AddCommand(addManualSyncCacheFlags(newSyncSlackCmd())) } diff --git a/cmd/msgvault/cmd/sync_slack_routing_test.go b/cmd/msgvault/cmd/sync_slack_routing_test.go index 1370b90e9..b398550fb 100644 --- a/cmd/msgvault/cmd/sync_slack_routing_test.go +++ b/cmd/msgvault/cmd/sync_slack_routing_test.go @@ -5,11 +5,14 @@ import ( "context" "testing" + "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.kenn.io/msgvault/internal/attachmentpolicy" "go.kenn.io/msgvault/internal/clirun" "go.kenn.io/msgvault/internal/config" + "go.kenn.io/msgvault/internal/slack" + "go.kenn.io/msgvault/internal/store" "go.kenn.io/msgvault/internal/testutil" ) @@ -64,6 +67,8 @@ func TestResolveSlackSyncSourcesFiltersByTeam(t *testing.T) { } func TestRunConfiguredSlackSyncIsolatesBrokenWorkspaces(t *testing.T) { + cfg := testConfigValue() + require := require.New(t) st := testutil.NewTestStore(t) @@ -82,28 +87,71 @@ func TestRunConfiguredSlackSyncIsolatesBrokenWorkspaces(t *testing.T) { HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx - err = runConfiguredSlackSync(context.Background(), st) + err = runConfiguredSlackSync(testCtx, st) require.ErrorContains(err, "malformed identifier") require.ErrorContains(err, "no Slack token for UME in workspace T09") } +func TestScheduledSlackAttemptsResumeAfterInterruptedWorkspace(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + st := testutil.NewTestStore(t) + for _, identifier := range []string{"T01:U01", "T02:U02", "T03:U03"} { + _, err := st.GetOrCreateSource(sourceTypeSlack, identifier) + require.NoError(err) + } + sources, err := resolveSlackSyncSources(st, "") + require.NoError(err) + rotation := &slackWorkspaceRotation{} + ctx, cancel := context.WithCancel(context.Background()) + var first []string + err = runScheduledSlackAttempts(ctx, sources, rotation, func(src *store.Source) (bool, error) { + first = append(first, src.Identifier) + cancel() // the scheduler's hard yield interrupted this workspace + return true, context.Canceled + }, func() error { return nil }) + require.ErrorIs(err, context.Canceled) + assert.Equal([]string{sources[0].Identifier}, first) + + var resumed []string + err = runScheduledSlackAttempts(context.Background(), sources, rotation, + func(src *store.Source) (bool, error) { + resumed = append(resumed, src.Identifier) + return true, nil + }, func() error { return nil }) + require.NoError(err) + assert.Equal(append(append([]string{}, sources[1].Identifier, sources[2].Identifier), sources[0].Identifier), resumed, + "the next scheduler run resumes after the interrupted workspace") +} + func TestSlackImportOptionsDeriveFromConfig(t *testing.T) { + cfg := testConfigValue() + assert := assert.New(t) savedCfg := cfg t.Cleanup(func() { cfg = savedCfg }) media := false + dms := false + groupDMs := true cfg = &config.Config{ HomeDir: t.TempDir(), Slack: config.SlackConfig{ + PrivateChannels: new(false), Channels: []string{"eng"}, ExcludeChannels: []string{"noise"}, + DMs: &dms, + GroupDMs: &groupDMs, Media: &media, MaxMediaMB: 7, }, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx - opts := slackImportOptions("T01", "UME") + opts := slackImportOptions("T01", "UME", cfg) assert.Equal("T01", opts.TeamID) assert.Equal("UME", opts.UserID) assert.False(opts.NoMedia, "persistent config is represented by typed policy, not the one-run flag") @@ -111,6 +159,40 @@ func TestSlackImportOptionsDeriveFromConfig(t *testing.T) { assert.Equal(int64(7)<<20, opts.MaxMediaBytes) assert.Equal([]string{"eng"}, opts.IncludeChannels) assert.Equal([]string{"noise"}, opts.ExcludeChannels) + assert.True(opts.ExcludeDMs) + assert.True(opts.ExcludePrivateChannels) + assert.False(opts.ExcludeGroupDMs) +} + +func TestApplySlackConversationOverrides(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + configured := &cobra.Command{} + configured.Flags().Bool("dms", true, "") + configured.Flags().Bool("group-dms", true, "") + configured.Flags().Bool("private-channels", true, "") + configuredOpts := slack.ImportOptions{ExcludeDMs: true, ExcludePrivateChannels: true} + applySlackConversationOverrides(configured, &configuredOpts, false, false, false) + assert.True(configuredOpts.ExcludePrivateChannels) + assert.True(configuredOpts.ExcludeDMs) + assert.False(configuredOpts.ExcludeGroupDMs) + + cmd := &cobra.Command{} + dms := true + groupDMs := false + cmd.Flags().Bool("dms", true, "") + cmd.Flags().Bool("group-dms", true, "") + cmd.Flags().Bool("private-channels", true, "") + require.NoError(cmd.Flags().Set("dms", "true")) + require.NoError(cmd.Flags().Set("group-dms", "false")) + require.NoError(cmd.Flags().Set("private-channels", "true")) + + opts := slack.ImportOptions{ExcludeDMs: true, ExcludePrivateChannels: true} + applySlackConversationOverrides(cmd, &opts, true, dms, groupDMs) + assert.False(opts.ExcludePrivateChannels) + + assert.False(opts.ExcludeDMs) + assert.True(opts.ExcludeGroupDMs) } func TestWriteSlackProgressSanitizesProviderNames(t *testing.T) { @@ -157,23 +239,31 @@ func TestSyncSlackCommandUsesDaemonRunner(t *testing.T) { server, requests := newDaemonCLIRunnerTestServer(t, func(req daemonCLIRunTestRequest) { assert.Equal([]string{ "sync-slack", + "--dms=false", "--full", + "--group-dms=false", "--limit=25", "--no-threads", + "--private-channels=false", "T0123456789", }, req.Args, "args") }, `{"type":"stdout","data":"Syncing Slack workspace T0123456789\n"}`, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx var stdout bytes.Buffer cmd := newSyncSlackCmd() + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetErr(&stdout) cmd.SetArgs([]string{ "T0123456789", + "--dms=false", "--full", + "--group-dms=false", "--limit", "25", "--no-threads", + "--private-channels=false", }) require.NoError(t, cmd.Execute(), "sync-slack") @@ -188,11 +278,13 @@ func TestAddSlackCommandForwardsTokenEnv(t *testing.T) { assert.Equal([]string{"add-slack"}, req.Args, "args") assert.Equal("xoxp-test-123", req.Env[clirun.EnvSlackToken], "token env forwarded") }, `{"type":"stdout","data":"Added Slack workspace Testers\n"}`, `{"type":"complete"}`) - configureRemoteDaemonForTest(t, server.URL) + testCtx := configureRemoteDaemonForTest(t, server.URL) + _ = testCtx t.Setenv(clirun.EnvSlackToken, "xoxp-test-123") var stdout bytes.Buffer cmd := newAddSlackCmd() + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetErr(&stdout) cmd.SetArgs([]string{}) diff --git a/cmd/msgvault/cmd/sync_teams.go b/cmd/msgvault/cmd/sync_teams.go index 2a9de24b4..fc0e69dcb 100644 --- a/cmd/msgvault/cmd/sync_teams.go +++ b/cmd/msgvault/cmd/sync_teams.go @@ -1,17 +1,12 @@ package cmd import ( - "context" "errors" "fmt" "io" - "os" - "os/signal" - "syscall" "time" "github.com/spf13/cobra" - "go.kenn.io/msgvault/internal/microsoft" "go.kenn.io/msgvault/internal/teams" ) @@ -40,59 +35,36 @@ Examples: msgvault sync-teams user@company.com --full`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil || state.logger == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger if !isDaemonCLISubprocess() { return runDaemonCLICommandHTTPFromCobra(cmd, args) } email := args[0] - s, cleanup, err := openWritableStoreAndInitForIngest() + s, cleanup, err := openWritableStoreAndInitForIngestInvocation(state) if err != nil { return err } defer cleanup() dbPath := cfg.DatabaseDSN() - if cfg.Microsoft.ClientID == "" { - return errors.New("microsoft OAuth not configured\n\n" + - "Add to your config.toml:\n\n" + - " [microsoft]\n" + - " client_id = \"your-azure-app-client-id\"\n\n" + - "See docs for Azure AD app registration setup") + if err := requireMicrosoftOAuthConfig(cfg); err != nil { + return err } - - mgr := microsoft.NewGraphManager( - cfg.Microsoft.ClientID, - cfg.Microsoft.EffectiveTenantID(), - cfg.Microsoft.EffectiveRedirectURI(), - cfg.TokensDir(), - logger, - ) - tokenFn, err := mgr.TokenSource(cmd.Context(), email) + client, err := newTeamsClient(cmd.Context(), cfg, logger, email) if err != nil { return fmt.Errorf("load Teams token: %w (run 'add-teams' first)", err) } - ctx, cancel := context.WithCancel(cmd.Context()) - defer cancel() - - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) - defer signal.Stop(sigChan) - go func() { - select { - case <-sigChan: - _, _ = fmt.Fprintln(cmd.ErrOrStderr(), "\nInterrupted. Saving checkpoint...") - cancel() - case <-ctx.Done(): - } - }() - - qps := float64(cfg.Sync.RateLimitQPS) - if qps <= 0 { - qps = 5 - } - client := teams.NewClient("https://graph.microsoft.com/v1.0", teams.TokenFunc(tokenFn), qps) + ctx, stop := withInterruptCancel(cmd, "\nInterrupted. Saving checkpoint...") + defer stop() + imp := teams.NewImporter(s, client) _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Syncing Microsoft Teams for %s\n\n", email) @@ -109,7 +81,7 @@ Examples: sum, err := imp.Import(ctx, opts) if ctx.Err() != nil { _, _ = fmt.Fprintln(cmd.OutOrStdout(), "\nInterrupted — re-run sync-teams to resume.") - return rebuildCacheAfterWrite(dbPath) + return rebuildCacheAfterManualSync(dbPath, state) } if err != nil { return fmt.Errorf("teams sync failed: %w", err) @@ -117,7 +89,7 @@ Examples: writeTeamsSyncSummary(cmd.OutOrStdout(), sum) - return rebuildCacheAfterWrite(dbPath) + return rebuildCacheAfterManualSync(dbPath, state) }, } @@ -141,5 +113,5 @@ func init() { syncTeamsCmd.Flags().BoolVar(&syncTeamsNoChannels, "no-channels", false, "sync chats only (skip team channels)") syncTeamsCmd.Flags().IntVar(&syncTeamsLimit, "limit", 0, "max messages per conversation (0 = no limit)") syncTeamsCmd.Flags().BoolVar(&syncTeamsFull, "full", false, "ignore stored cursor and re-fetch every message (repairs/backfills existing rows in place)") - rootCmd.AddCommand(syncTeamsCmd) + rootCmd.AddCommand(addManualSyncCacheFlags(syncTeamsCmd)) } diff --git a/cmd/msgvault/cmd/sync_test.go b/cmd/msgvault/cmd/sync_test.go index f2796f20f..f9201ecda 100644 --- a/cmd/msgvault/cmd/sync_test.go +++ b/cmd/msgvault/cmd/sync_test.go @@ -45,6 +45,9 @@ func runSyncFullLocalForTest(cmd *cobra.Command, args []string) error { // scaffolding so the test exercises runIncrementalSync, not just // the OAuth manager setup. func TestSyncCmd_DuplicateIdentifierRoutesCorrectly(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + require := require.New(t) assert := assert.New(t) tmpDir := t.TempDir() @@ -89,6 +92,8 @@ func TestSyncCmd_DuplicateIdentifierRoutesCorrectly(t *testing.T) { Data: config.DataConfig{DataDir: tmpDir}, OAuth: config.OAuthConfig{ClientSecrets: secretsPath}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) testCmd := &cobra.Command{ @@ -98,6 +103,7 @@ func TestSyncCmd_DuplicateIdentifierRoutesCorrectly(t *testing.T) { } root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(testCmd) root.SetArgs([]string{"sync", "shared@example.com"}) @@ -147,6 +153,9 @@ func TestResolveSyncSourcesSourceIDIsExact(t *testing.T) { } func TestSyncFullSourceIDTreatsLegacyEmptyTypeAsGmail(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + require := require.New(t) assert := assert.New(t) tmpDir := t.TempDir() @@ -167,10 +176,13 @@ func TestSyncFullSourceIDTreatsLegacyEmptyTypeAsGmail(t *testing.T) { HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) command := &cobra.Command{} - command.SetContext(t.Context()) + command.SetContext(testCtx) + command.SetContext(testCtx) command.Flags().Int64("source-id", 0, "") require.NoError(command.Flags().Set("source-id", strconv.FormatInt(legacy.ID, 10))) err = runSyncFullLocal(command, nil) @@ -204,6 +216,9 @@ func TestResolveSyncSourcesNumericTokenDoesNotBecomeSourceID(t *testing.T) { // TestSyncCmd_SingleSourceNoAmbiguity verifies that a single // source for an identifier works without the legacy fallback. func TestSyncCmd_SingleSourceNoAmbiguity(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + require := require.New(t) assert := assert.New(t) tmpDir := t.TempDir() @@ -228,6 +243,8 @@ func TestSyncCmd_SingleSourceNoAmbiguity(t *testing.T) { HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) testCmd := &cobra.Command{ @@ -237,6 +254,7 @@ func TestSyncCmd_SingleSourceNoAmbiguity(t *testing.T) { } root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(testCmd) root.SetArgs([]string{"sync", "solo@example.com"}) @@ -267,6 +285,9 @@ func TestSyncCmd_SingleSourceNoAmbiguity(t *testing.T) { // returns a clear error instead of falling back to the legacy // Gmail path. func TestSyncCmd_MboxIdentifierDoesNotFallback(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + tmpDir := t.TempDir() dbPath := tmpDir + "/msgvault.db" @@ -289,6 +310,8 @@ func TestSyncCmd_MboxIdentifierDoesNotFallback(t *testing.T) { HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) // Test both sync and sync-full commands. @@ -307,6 +330,7 @@ func TestSyncCmd_MboxIdentifierDoesNotFallback(t *testing.T) { } root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(testCmd) root.SetArgs([]string{ tc.name, "imported@example.com", @@ -323,6 +347,9 @@ func TestSyncCmd_MboxIdentifierDoesNotFallback(t *testing.T) { // mixed Gmail+IMAP setup without OAuth configured, sync-full skips // the Gmail source and still syncs the IMAP source. func TestSyncFullCmd_OAuthSkipDoesNotBlockIMAP(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + require := require.New(t) assert := assert.New(t) tmpDir := t.TempDir() @@ -350,6 +377,8 @@ func TestSyncFullCmd_OAuthSkipDoesNotBlockIMAP(t *testing.T) { HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) testCmd := &cobra.Command{ @@ -359,6 +388,7 @@ func TestSyncFullCmd_OAuthSkipDoesNotBlockIMAP(t *testing.T) { } root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(testCmd) root.SetArgs([]string{"sync-full"}) @@ -386,6 +416,9 @@ func TestSyncFullCmd_OAuthSkipDoesNotBlockIMAP(t *testing.T) { // the no-args discovery path. The OAuth error should be reported // after IMAP work completes. func TestSyncCmd_BrokenOAuthDoesNotBlockIMAP(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + for _, tc := range []struct { name string runE func(*cobra.Command, []string) error @@ -442,6 +475,8 @@ func TestSyncCmd_BrokenOAuthDoesNotBlockIMAP(t *testing.T) { logger = slog.New( slog.NewTextHandler(os.Stderr, nil), ) + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + invocationFromContext(testCtx).logger = logger testCmd := &cobra.Command{ Use: tc.name + " [email]", @@ -450,6 +485,7 @@ func TestSyncCmd_BrokenOAuthDoesNotBlockIMAP(t *testing.T) { } root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(testCmd) root.SetArgs([]string{tc.name}) @@ -484,6 +520,9 @@ func TestSyncCmd_BrokenOAuthDoesNotBlockIMAP(t *testing.T) { // even in a mixed Gmail+IMAP setup where Gmail would otherwise // succeed first. func TestSyncFullCmd_MalformedDateRejectsBeforeSync(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + require := require.New(t) assert := assert.New(t) tmpDir := t.TempDir() @@ -526,6 +565,8 @@ func TestSyncFullCmd_MalformedDateRejectsBeforeSync(t *testing.T) { Data: config.DataConfig{DataDir: tmpDir}, OAuth: config.OAuthConfig{ClientSecrets: secretsPath}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) syncAfter = "not-a-date" @@ -537,6 +578,7 @@ func TestSyncFullCmd_MalformedDateRejectsBeforeSync(t *testing.T) { } root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(testCmd) root.SetArgs([]string{"sync-full"}) @@ -555,6 +597,9 @@ func TestSyncFullCmd_MalformedDateRejectsBeforeSync(t *testing.T) { // --after/--before flags produce a clear error for IMAP sources // instead of silently syncing the entire mailbox. func TestSyncFullCmd_MalformedIMAPDateFlagErrors(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + require := require.New(t) tmpDir := t.TempDir() dbPath := tmpDir + "/msgvault.db" @@ -585,6 +630,8 @@ func TestSyncFullCmd_MalformedIMAPDateFlagErrors(t *testing.T) { HomeDir: tmpDir, Data: config.DataConfig{DataDir: tmpDir}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) for _, tc := range []struct { @@ -608,6 +655,7 @@ func TestSyncFullCmd_MalformedIMAPDateFlagErrors(t *testing.T) { } root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(testCmd) root.SetArgs([]string{ "sync-full", "i@example.com", @@ -625,6 +673,9 @@ func TestSyncFullCmd_MalformedIMAPDateFlagErrors(t *testing.T) { // only Gmail sources exist and OAuth is broken, the actual error is // returned, not "no accounts are ready to sync". func TestSyncCmd_GmailOnlyBrokenOAuthSurfacesError(t *testing.T) { + cfg := testConfigValue() + logger := testLoggerValue() + for _, tc := range []struct { name string runE func(*cobra.Command, []string) error @@ -672,6 +723,8 @@ func TestSyncCmd_GmailOnlyBrokenOAuthSurfacesError(t *testing.T) { logger = slog.New( slog.NewTextHandler(os.Stderr, nil), ) + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + invocationFromContext(testCtx).logger = logger testCmd := &cobra.Command{ Use: tc.name + " [email]", @@ -680,6 +733,7 @@ func TestSyncCmd_GmailOnlyBrokenOAuthSurfacesError(t *testing.T) { } root := newTestRootCmd() + root.SetContext(testCtx) root.AddCommand(testCmd) root.SetArgs([]string{tc.name}) @@ -787,3 +841,23 @@ func TestTrimFolderFilter_DoesNotBlockOnErrorInSyncFull(t *testing.T) { require.NotPanics(func() { parseFolderFilter([]string{" ", " "}) }) require.NotPanics(func() { parseFolderFilter([]string{"", " ", ""}) }) } + +func TestSyncFullGraphAccountPointsToSync(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + dir := t.TempDir() + st, err := store.Open(filepath.Join(dir, "msgvault.db")) + require.NoError(err) + require.NoError(st.InitSchema()) + src, err := st.GetOrCreateSource(sourceTypeMSMail, "mail@example.com") + require.NoError(err) + require.NoError(st.Close()) + cfg := &config.Config{HomeDir: dir, Data: config.DataConfig{DataDir: dir}} + cmd := &cobra.Command{} + cmd.SetContext(testInvocationContext(t.Context(), cfg, invocationOptions{})) + cmd.Flags().Int64("source-id", 0, "") + require.NoError(cmd.Flags().Set("source-id", strconv.FormatInt(src.ID, 10))) + err = runSyncFullLocal(cmd, nil) + require.Error(err) + assert.Contains(err.Error(), "msgvault sync --source-id "+strconv.FormatInt(src.ID, 10)) +} diff --git a/cmd/msgvault/cmd/syncfull.go b/cmd/msgvault/cmd/syncfull.go index 6bd25ebd6..ce34acad6 100644 --- a/cmd/msgvault/cmd/syncfull.go +++ b/cmd/msgvault/cmd/syncfull.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "log/slog" "os" "os/signal" "strings" @@ -13,6 +14,7 @@ import ( "github.com/mattn/go-isatty" "github.com/spf13/cobra" + "go.kenn.io/msgvault/internal/config" "go.kenn.io/msgvault/internal/gmail" imaplib "go.kenn.io/msgvault/internal/imap" "go.kenn.io/msgvault/internal/microsoft" @@ -83,18 +85,24 @@ func validateSyncFullFlags(cmd *cobra.Command) error { } func runSyncFullLocal(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil || state.logger == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger selector, selectorSet, err := syncSourceSelector(cmd, args) if err != nil { return usageErr(cmd, err) } - s, cleanup, err := openWritableStoreAndInit() + s, cleanup, err := openWritableStoreAndInitForInvocation(state) if err != nil { return err } defer cleanup() dbPath := cfg.DatabaseDSN() - getOAuthMgr := oauthManagerCache() + getOAuthMgr := oauthManagerCache(invocationFromCommand(cmd)) // Determine which sources to sync var sources []*store.Source @@ -114,6 +122,11 @@ func runSyncFullLocal(cmd *cobra.Command, args []string) error { } } if len(sources) == 0 { + for _, src := range allMatches { + if src.SourceType == sourceTypeMSMail { + return fmt.Errorf("use 'msgvault sync --source-id %d' for initial and later Microsoft Graph mail downloads", src.ID) + } + } if len(allMatches) > 0 { // Identifier exists but has no syncable source types. return fmt.Errorf("%s exists but its source type cannot be synced (only gmail and imap are supported)", syncSelectorLabel(selector)) @@ -153,7 +166,7 @@ func runSyncFullLocal(cmd *cobra.Command, args []string) error { } } case sourceTypeIMAP: - skipMsg, parseErr := imapSkipReason(src) + skipMsg, parseErr := imapSkipReason(src, cfg, logger) if parseErr != nil { syncErrors = append(syncErrors, fmt.Sprintf("%s: malformed sync_config: %v", src.Identifier, parseErr)) continue @@ -210,14 +223,14 @@ func runSyncFullLocal(cmd *cobra.Command, args []string) error { } } - if err := runFullSync(ctx, s, getOAuthMgr, src); err != nil { + if err := runFullSync(ctx, s, getOAuthMgr, src, state); err != nil { syncErrors = append(syncErrors, fmt.Sprintf("%s: %v", src.Identifier, err)) continue } } // Rebuild analytics cache. - cacheErr := rebuildCacheAfterWrite(dbPath) + cacheErr := rebuildCacheAfterManualSync(dbPath, state) if len(syncErrors) > 0 { fmt.Println() @@ -241,6 +254,12 @@ func runSyncFullLocal(cmd *cobra.Command, args []string) error { // oauth.ScopesDeletion (or another set) for workflows that need elevated // access. func buildAPIClient(ctx context.Context, src *store.Source, getOAuthMgr func(string) (*oauth.Manager, error), saScopes []string, imapOpts ...imaplib.Option) (gmail.API, error) { + state := invocationFromContext(ctx) + if state == nil || state.cfg == nil || state.logger == nil { + return nil, errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger switch src.SourceType { case sourceTypeGmail, "": appName := sourceOAuthApp(src) @@ -347,7 +366,7 @@ func buildAPIClient(ctx context.Context, src *store.Source, getOAuthMgr func(str // accounts: another source with a same-named mailbox gets no entry. Nil-safe // because several command tests run without a loaded global config; a // missing entry means no explicit trust. -func configuredTrustedSentMailboxes(identifier string) []string { +func configuredTrustedSentMailboxes(identifier string, cfg *config.Config) []string { if cfg == nil { return nil } @@ -383,6 +402,8 @@ func imapFolderStateOptions( s *store.Store, src *store.Source, forceRescan bool, + cfg *config.Config, + logger *slog.Logger, ) []imaplib.Option { if src.SourceType != sourceTypeIMAP { return nil @@ -391,7 +412,7 @@ func imapFolderStateOptions( var opts []imaplib.Option states, err := loadIMAPFolderStates(s, src.ID) if err != nil { - logger.Warn("failed to load IMAP folder states", "source", src.Identifier, "error", err) + repairLogger(logger).Warn("failed to load IMAP folder states", "source", src.Identifier, "error", err) } else if len(states) > 0 { opts = append(opts, imaplib.WithFolderStates(states)) } @@ -417,7 +438,7 @@ func imapFolderStateOptions( } return result, nil }), - imaplib.WithTrustedSentMailboxes(configuredTrustedSentMailboxes(src.Identifier))) + imaplib.WithTrustedSentMailboxes(configuredTrustedSentMailboxes(src.Identifier, cfg))) if forceRescan { opts = append(opts, imaplib.WithForceFullEnumeration()) } @@ -545,13 +566,21 @@ func saveIMAPFolderStates( return nil } -func runFullSync(ctx context.Context, s *store.Store, getOAuthMgr func(string) (*oauth.Manager, error), src *store.Source) error { +func runFullSync(ctx context.Context, s *store.Store, getOAuthMgr func(string) (*oauth.Manager, error), src *store.Source, state *invocation) error { + if state == nil { + state = invocationFromContext(ctx) + } + if state == nil || state.cfg == nil || state.logger == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger progress := &CLIProgress{} // --noresume promises a fresh sync, so it must also bypass the // saved folder high water marks and re-enumerate every mailbox. A clean // completed run still saves fresh high water marks afterwards. - imapOpts := imapFolderStateOptions(s, src, syncNoResume) + imapOpts := imapFolderStateOptions(s, src, syncNoResume, cfg, logger) // Pass CLI folder filter strings to the IMAP client. The IMAP // client selects the effective include list (CLI --folder when @@ -608,7 +637,7 @@ func runFullSync(ctx context.Context, s *store.Store, getOAuthMgr func(string) ( } // Create syncer with progress reporter - syncer := newMessageSyncer(apiClient, s, opts). + syncer := newMessageSyncer(apiClient, s, opts, state). WithLogger(logger). WithProgress(progress) @@ -936,7 +965,10 @@ func (p *CLIProgress) OnError(err error) { // - ("", nil) — credentials present, source is ready // - ("msg", nil) — credentials absent; print the message and skip // - ("", err) — sync_config is malformed; add to the error list -func imapSkipReason(src *store.Source) (string, error) { +func imapSkipReason(src *store.Source, cfg *config.Config, logger *slog.Logger) (string, error) { + if cfg == nil { + return "", errors.New("configuration is unavailable") + } if !src.SyncConfig.Valid || src.SyncConfig.String == "" { if !imaplib.HasCredentials(cfg.TokensDir(), src.Identifier) { return fmt.Sprintf("Skipping %s (no credentials — run 'add-imap' or 'add-o365' first)", src.Identifier), nil @@ -957,7 +989,7 @@ func imapSkipReason(src *store.Source) (string, error) { cfg.Microsoft.EffectiveTenantID(), cfg.Microsoft.EffectiveRedirectURI(), cfg.TokensDir(), - logger, + repairLogger(logger), ) if !msMgr.HasToken(imapCfg.Username) { return fmt.Sprintf("Skipping %s (no Microsoft token — run 'add-o365' first)", src.Identifier), nil @@ -981,5 +1013,5 @@ func init() { syncFullCmd.Flags().IntVar(&syncLimit, "limit", 0, "Limit number of messages (for testing)") syncFullCmd.Flags().StringArrayVar(&syncFolders, "folder", []string{}, "IMAP folder to scan (repeatable)") syncFullCmd.Flags().StringArrayVar(&syncSkipFolders, "skip-folder", []string{}, "IMAP folder to skip (repeatable)") - rootCmd.AddCommand(syncFullCmd) + rootCmd.AddCommand(addManualSyncCacheFlags(syncFullCmd)) } diff --git a/cmd/msgvault/cmd/tui.go b/cmd/msgvault/cmd/tui.go index a6bacf606..25d7fda09 100644 --- a/cmd/msgvault/cmd/tui.go +++ b/cmd/msgvault/cmd/tui.go @@ -12,6 +12,7 @@ import ( "github.com/spf13/cobra" "go.kenn.io/msgvault/internal/api" "go.kenn.io/msgvault/internal/daemonclient" + "go.kenn.io/msgvault/internal/logging" "go.kenn.io/msgvault/internal/peoplebrowser" "go.kenn.io/msgvault/internal/query" "go.kenn.io/msgvault/internal/tui" @@ -75,13 +76,23 @@ HTTP Mode: Otherwise it starts or reuses the local daemon. Use --local to force the local daemon when a remote is configured.`, RunE: func(cmd *cobra.Command, args []string) error { - backend, err := openTUIBackend(cmd.Context()) + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + currentCfg := state.cfg + currentLogResult := state.logResult + local, err := cmd.Flags().GetBool(localValue) + if err != nil { + return fmt.Errorf("read --local flag: %w", err) + } + backend, err := openTUIBackendWithLocal(withInvocation(cmd.Context(), state), local) if err != nil { return err } defer backend.cleanup() if backend.info.Kind == HTTPStoreConfiguredRemote { - fmt.Printf("Connected to remote: %s\n", cfg.Remote.URL) + fmt.Printf("Connected to remote: %s\n", currentCfg.Remote.URL) } // The shipped daemon engine provides Texts directly. People uses the @@ -98,8 +109,8 @@ HTTP Mode: semanticSearch := tuiSemanticSearcher(cmd.Context(), backend.client, backend.engine) collectionScopes := tuiCollectionScopes(cmd.Context(), backend.client, backend.engine) model := tui.New(backend.engine, tui.Options{ - DataDir: cfg.Data.DataDir, - ExportDir: cfg.ExportDir(), + DataDir: currentCfg.Data.DataDir, + ExportDir: currentCfg.ExportDir(), Version: Version, TextEngine: textEngine, PeopleBackend: peopleBackend, @@ -128,20 +139,29 @@ HTTP Mode: // the render. The daily log file still receives // everything, so 'msgvault logs -f' in another pane // continues to work for diagnostics. - prevLogger := slog.Default() - if logResult != nil { - slog.SetDefault(logResult.FileOnlyLogger()) - } - defer slog.SetDefault(prevLogger) - - if _, err := p.Run(); err != nil { - return fmt.Errorf("run tui: %w", err) + if err := withTUIFileLogger(currentLogResult, func() error { + _, err := p.Run() + if err != nil { + return fmt.Errorf("run tui: %w", err) + } + return nil + }); err != nil { + return err } return nil }, } +func withTUIFileLogger(result *logging.Result, run func() error) error { + previous := slog.Default() + if result != nil { + slog.SetDefault(result.FileOnlyLogger()) + } + defer slog.SetDefault(previous) + return run() +} + func tuiSemanticSearcher( ctx context.Context, client *daemonclient.Client, @@ -294,10 +314,22 @@ func refreshAnalyticsCacheNotice( } func openTUIBackend(ctx context.Context) (*tuiBackend, error) { - if forceLocalTUI { - previousUseLocal := useLocal - useLocal = true - defer func() { useLocal = previousUseLocal }() + state := invocationFromContext(ctx) + if state == nil { + return nil, errors.New("configuration is unavailable") + } + return openTUIBackendWithLocal(ctx, forceLocalTUI) +} + +func openTUIBackendWithLocal(ctx context.Context, forceLocal bool) (*tuiBackend, error) { + if forceLocal { + inv := invocationFromContext(ctx) + if inv == nil { + return nil, errors.New("configuration is unavailable") + } + local := *inv + local.options.useLocal = true + ctx = withInvocation(ctx, &local) } st, info, err := OpenHTTPStore(ctx) diff --git a/cmd/msgvault/cmd/tui_people_codex_login.go b/cmd/msgvault/cmd/tui_people_codex_login.go new file mode 100644 index 000000000..becd4f9f0 --- /dev/null +++ b/cmd/msgvault/cmd/tui_people_codex_login.go @@ -0,0 +1,156 @@ +package cmd + +import ( + "context" + "errors" + "net/http" + + "go.kenn.io/msgvault/internal/tui" + "go.kenn.io/msgvault/pkg/client/generated" +) + +func (b *tuiDaemonSettingsBackend) StartCodexLogin(ctx context.Context, name string) (tui.CodexDeviceLogin, error) { + if name == "" { + return tui.CodexDeviceLogin{}, errors.New("codex profile name is required") + } + client, err := b.peopleInferenceClient() + if err != nil { + return tui.CodexDeviceLogin{}, err + } + response, err := client.StartSettingsPeopleCodexLoginWithResponse(ctx, + &generated.StartSettingsPeopleCodexLoginRequestOptions{ + Body: &generated.PeopleCodexLoginRequest{Name: name}, + }) + if err != nil || response == nil || response.StatusCode != http.StatusOK { + return tui.CodexDeviceLogin{}, peopleInferenceHTTPError("start Codex device login", response, err) + } + if response.JSON200 == nil || response.JSON200.SessionID == "" || + response.JSON200.VerificationURL == "" || response.JSON200.UserCode == "" || + response.JSON200.LocalDeadline.IsZero() { + return tui.CodexDeviceLogin{}, errors.New("start Codex device login: incomplete response") + } + return tui.CodexDeviceLogin{ + SessionID: response.JSON200.SessionID, + URL: response.JSON200.VerificationURL, Code: response.JSON200.UserCode, + Deadline: response.JSON200.LocalDeadline, + }, nil +} + +func (b *tuiDaemonSettingsBackend) PollCodexLogin(ctx context.Context, session string) (tui.CodexLoginPoll, error) { + client, err := b.peopleInferenceClient() + if err != nil { + return tui.CodexLoginPoll{}, err + } + response, err := client.GetSettingsPeopleCodexLoginWithResponse(ctx, + &generated.GetSettingsPeopleCodexLoginRequestOptions{ + PathParams: &generated.GetSettingsPeopleCodexLoginPath{ID: session}, + }) + if err != nil || response == nil || response.StatusCode != http.StatusOK { + return tui.CodexLoginPoll{Failed: response != nil && response.StatusCode == http.StatusNotFound}, + peopleInferenceHTTPError("poll Codex device login", response, err) + } + if response.JSON200 == nil { + return tui.CodexLoginPoll{}, errors.New("poll Codex device login: empty response") + } + switch response.JSON200.State { + case "complete": + return tui.CodexLoginPoll{Complete: true}, nil + case "pending": + return tui.CodexLoginPoll{}, nil + case "failed": + return tui.CodexLoginPoll{Failed: true}, errors.New("codex device login failed") + case "cancelled": + return tui.CodexLoginPoll{Failed: true}, errors.New("codex device login was cancelled") + default: + return tui.CodexLoginPoll{Failed: true}, errors.New("codex device login returned an unknown state") + } +} + +func (b *tuiDaemonSettingsBackend) CancelCodexLogin(ctx context.Context, session string) error { + client, err := b.peopleInferenceClient() + if err != nil { + return err + } + response, err := client.CancelSettingsPeopleCodexLoginWithResponse(ctx, + &generated.CancelSettingsPeopleCodexLoginRequestOptions{ + PathParams: &generated.CancelSettingsPeopleCodexLoginPath{ID: session}, + }) + if err != nil || response == nil || response.StatusCode != http.StatusOK { + return peopleInferenceHTTPError("cancel Codex device login", response, err) + } + if response.JSON200 == nil || response.JSON200.State != "cancelled" { + return errors.New("cancel Codex device login: cancellation was not confirmed") + } + return nil +} + +func (b *tuiDaemonSettingsBackend) ListCodexModels(ctx context.Context, session string) ([]tui.CodexModelChoice, error) { + client, err := b.peopleInferenceClient() + if err != nil { + return nil, err + } + response, err := client.GetSettingsPeopleCodexModelsWithResponse(ctx, + &generated.GetSettingsPeopleCodexModelsRequestOptions{ + PathParams: &generated.GetSettingsPeopleCodexModelsPath{ID: session}, + }) + if err != nil || response == nil || response.StatusCode != http.StatusOK { + return nil, peopleInferenceHTTPError("list Codex models", response, err) + } + if response.JSON200 == nil { + return nil, errors.New("list Codex models: empty response") + } + models := make([]tui.CodexModelChoice, 0, len(response.JSON200.Models)) + for _, model := range response.JSON200.Models { + if model.ID != "" { + models = append(models, tui.CodexModelChoice{ + ID: model.ID, DefaultReasoningEffort: model.DefaultReasoningEffort, + ReasoningEfforts: append([]string(nil), model.SupportedEfforts...), + }) + } + } + return models, nil +} + +func (b *tuiDaemonSettingsBackend) SaveCodexProfile( + ctx context.Context, session string, request tui.CodexProfileRequest, +) (string, error) { + client, err := b.peopleInferenceClient() + if err != nil { + return "", err + } + etag, err := b.peopleInferenceETag(ctx, client) + if err != nil { + return "", err + } + body := generated.PeopleCodexProfileRequest{ + Model: request.Model, ReasoningEffort: request.ReasoningEffort, + RetentionPosture: request.RetentionPosture, TrainingPosture: request.TrainingPosture, + AllowedSources: request.AllowedSources, SourceSince: request.SourceSince, + AllowSensitive: request.AllowSensitive, + } + if request.SourceUntil != "" { + body.SourceUntil = &request.SourceUntil + } + response, err := client.PutSettingsPeopleCodexProfileWithResponse(ctx, + &generated.PutSettingsPeopleCodexProfileRequestOptions{ + PathParams: &generated.PutSettingsPeopleCodexProfilePath{ID: session}, + Header: &generated.PutSettingsPeopleCodexProfileHeaders{IfMatch: etag}, + Body: &body, + }) + if response != nil && response.StatusCode == http.StatusPreconditionFailed { + return "", &tui.SettingsConflictError{Scope: tui.SettingsConflictConfig, + Err: peopleInferenceHTTPError("save Codex profile", response, err)} + } + if err != nil || response == nil || response.StatusCode != http.StatusOK { + return "", peopleInferenceHTTPError("save Codex profile", response, err) + } + if response.JSON200 == nil { + return "", errors.New("save Codex profile: empty response") + } + for _, profile := range response.JSON200.Profiles { + if profile.Name == request.Name && profile.Protocol == "codex_app_server" && profile.Model == request.Model { + return profile.Name, nil + } + } + return "", errors.New("save Codex profile: profile was not returned") +} diff --git a/cmd/msgvault/cmd/tui_people_inference.go b/cmd/msgvault/cmd/tui_people_inference.go new file mode 100644 index 000000000..80887169d --- /dev/null +++ b/cmd/msgvault/cmd/tui_people_inference.go @@ -0,0 +1,373 @@ +package cmd + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + + "go.kenn.io/msgvault/internal/daemonclient" + "go.kenn.io/msgvault/internal/tui" + apiclient "go.kenn.io/msgvault/pkg/client" + "go.kenn.io/msgvault/pkg/client/generated" +) + +var _ tui.PeopleInferenceControlBackend = (*tuiDaemonSettingsBackend)(nil) +var _ tui.PeopleInferenceBackend = (*tuiDaemonSettingsBackend)(nil) + +// People inference enrollment runs through daemon routes; the TUI does not +// store Codex credentials or reproduce the daemon's enrollment rules. +func (b *tuiDaemonSettingsBackend) LoadPeopleInferenceStatus(ctx context.Context) (tui.PeopleInferenceStatus, error) { + client, err := b.peopleInferenceClient() + if err != nil { + return tui.PeopleInferenceStatus{}, err + } + response, err := client.GetSettingsPeopleInferenceWithResponse(ctx) + if err != nil || response == nil || response.StatusCode != http.StatusOK { + return tui.PeopleInferenceStatus{}, peopleInferenceHTTPError("load people inference status", response, err) + } + if response.JSON200 == nil { + return tui.PeopleInferenceStatus{}, errors.New("load people inference status: empty response") + } + return tuiPeopleInferenceStatus(response.JSON200), nil +} + +func (b *tuiDaemonSettingsBackend) CheckCodexProfile( + ctx context.Context, name string, +) (tui.PeopleInferenceDisclosure, error) { + client, err := b.peopleInferenceClient() + if err != nil { + return tui.PeopleInferenceDisclosure{}, err + } + etag, err := b.peopleInferenceETag(ctx, client) + if err != nil { + return tui.PeopleInferenceDisclosure{}, err + } + response, err := client.CheckSettingsPeopleInferenceProviderWithResponse(ctx, + &generated.CheckSettingsPeopleInferenceProviderRequestOptions{ + PathParams: &generated.CheckSettingsPeopleInferenceProviderPath{Name: name}, + Header: &generated.CheckSettingsPeopleInferenceProviderHeaders{IfMatch: etag}, + }) + if response != nil && response.StatusCode == http.StatusPreconditionFailed { + return tui.PeopleInferenceDisclosure{}, &tui.SettingsConflictError{Scope: tui.SettingsConflictConfig, + Err: peopleInferenceHTTPError("check people inference profile", response, err)} + } + if err != nil || response == nil || response.StatusCode != http.StatusOK { + return tui.PeopleInferenceDisclosure{}, peopleInferenceHTTPError("check people inference profile", response, err) + } + if response.JSON200 == nil || !response.JSON200.Ok || response.JSON200.Fingerprint == "" { + return tui.PeopleInferenceDisclosure{}, errors.New("check people inference profile: no successful check was returned") + } + profile, _, err := b.peopleInferenceProfileSnapshot(ctx, client, name) + if err != nil { + return tui.PeopleInferenceDisclosure{}, err + } + if profile.Fingerprint == nil || *profile.Fingerprint != response.JSON200.Fingerprint || profile.Model != response.JSON200.Model || !profile.Checked { + return tui.PeopleInferenceDisclosure{}, errors.New("people inference profile changed after check; reload settings") + } + return tui.PeopleInferenceDisclosure{ + Profile: name, Fingerprint: response.JSON200.Fingerprint, Text: peopleInferenceDisclosureText(profile), + }, nil +} + +func (b *tuiDaemonSettingsBackend) ConsentCodexProfile( + ctx context.Context, name, fingerprint string, +) error { + if fingerprint == "" { + return errors.New("check people inference profile before consenting") + } + client, err := b.peopleInferenceClient() + if err != nil { + return err + } + profile, etag, err := b.peopleInferenceProfileSnapshot(ctx, client, name) + if err != nil { + return err + } + if profile.Fingerprint == nil || *profile.Fingerprint != fingerprint { + return errors.New("people inference profile changed after check; reload settings") + } + if !profile.Checked { + return errors.New("run an exact synthetic check before consenting") + } + response, err := client.ConsentSettingsPeopleInferenceProviderWithResponse(ctx, + &generated.ConsentSettingsPeopleInferenceProviderRequestOptions{ + PathParams: &generated.ConsentSettingsPeopleInferenceProviderPath{Name: name}, + Header: &generated.ConsentSettingsPeopleInferenceProviderHeaders{IfMatch: etag}, + Body: &generated.PeopleInferenceConsentRequest{Fingerprint: fingerprint, Confirmed: true}, + }) + if response != nil && response.StatusCode == http.StatusPreconditionFailed { + return &tui.SettingsConflictError{Scope: tui.SettingsConflictConfig, + Err: peopleInferenceHTTPError("consent to people inference profile", response, err)} + } + if response != nil && response.StatusCode == http.StatusConflict && response.JSON409 != nil { + switch response.JSON409.ErrorData { + case "consent_disclosure_changed": + return errors.New("people inference disclosure changed; run the check again") + case "check_required": + return errors.New("run an exact synthetic check before consenting") + } + } + if err != nil || response == nil || response.StatusCode != http.StatusOK { + return peopleInferenceHTTPError("consent to people inference profile", response, err) + } + if response.JSON200 == nil { + return errors.New("consent to people inference profile: empty response") + } + for _, latest := range response.JSON200.Profiles { + if latest.Name == name && latest.Fingerprint != nil && *latest.Fingerprint == fingerprint && latest.ConsentActive { + return nil + } + } + return errors.New("consent to people inference profile: confirmation was not recorded") +} + +func (b *tuiDaemonSettingsBackend) RevokePeopleInferenceConsent( + ctx context.Context, name, fingerprint string, +) (tui.PeopleInferenceStatus, error) { + if fingerprint == "" { + return tui.PeopleInferenceStatus{}, errors.New("people inference profile fingerprint is required") + } + client, err := b.peopleInferenceClient() + if err != nil { + return tui.PeopleInferenceStatus{}, err + } + profile, etag, err := b.peopleInferenceProfileSnapshot(ctx, client, name) + if err != nil { + return tui.PeopleInferenceStatus{}, err + } + if profile.Fingerprint == nil || *profile.Fingerprint != fingerprint { + return tui.PeopleInferenceStatus{}, errors.New("people inference profile changed; reload settings") + } + response, err := client.RevokeSettingsPeopleInferenceProviderWithResponse(ctx, + &generated.RevokeSettingsPeopleInferenceProviderRequestOptions{ + PathParams: &generated.RevokeSettingsPeopleInferenceProviderPath{Name: name}, + Header: &generated.RevokeSettingsPeopleInferenceProviderHeaders{IfMatch: etag}, + }) + if response != nil && response.StatusCode == http.StatusPreconditionFailed { + return tui.PeopleInferenceStatus{}, &tui.SettingsConflictError{Scope: tui.SettingsConflictConfig, + Err: peopleInferenceHTTPError("revoke people inference consent", response, err)} + } + if err != nil || response == nil || response.StatusCode != http.StatusOK { + return tui.PeopleInferenceStatus{}, peopleInferenceHTTPError("revoke people inference consent", response, err) + } + if response.JSON200 == nil { + return tui.PeopleInferenceStatus{}, errors.New("revoke people inference consent: empty response") + } + for _, latest := range response.JSON200.Profiles { + if latest.Name == name && latest.Fingerprint != nil && *latest.Fingerprint == fingerprint && !latest.ConsentActive { + return tuiPeopleInferenceStatus(response.JSON200), nil + } + } + return tui.PeopleInferenceStatus{}, errors.New("revoke people inference consent: revocation was not recorded") +} + +func (b *tuiDaemonSettingsBackend) DisablePeopleInference( + ctx context.Context, fingerprint string, +) (tui.PeopleInferenceStatus, error) { + if fingerprint == "" { + return tui.PeopleInferenceStatus{}, errors.New("configured people inference fingerprint is required") + } + client, err := b.peopleInferenceClient() + if err != nil { + return tui.PeopleInferenceStatus{}, err + } + settings, etag, err := b.peopleInferenceSettingsSnapshot(ctx, client) + if err != nil { + return tui.PeopleInferenceStatus{}, err + } + if settings.ConfiguredFingerprint == nil || *settings.ConfiguredFingerprint != fingerprint { + return tui.PeopleInferenceStatus{}, errors.New("configured people inference profile changed; reload settings") + } + response, err := client.DisableSettingsPeopleInferenceWithResponse(ctx, + &generated.DisableSettingsPeopleInferenceRequestOptions{ + Header: &generated.DisableSettingsPeopleInferenceHeaders{IfMatch: etag}, + }) + if response != nil && response.StatusCode == http.StatusPreconditionFailed { + return tui.PeopleInferenceStatus{}, &tui.SettingsConflictError{Scope: tui.SettingsConflictConfig, + Err: peopleInferenceHTTPError("disable people inference", response, err)} + } + if err != nil || response == nil || response.StatusCode != http.StatusOK { + return tui.PeopleInferenceStatus{}, peopleInferenceHTTPError("disable people inference", response, err) + } + if response.JSON200 == nil || response.JSON200.ConfiguredEnabled { + return tui.PeopleInferenceStatus{}, errors.New("disable people inference: disabled status was not returned") + } + return tuiPeopleInferenceStatus(response.JSON200), nil +} + +func (b *tuiDaemonSettingsBackend) RemovePeopleInferenceProfile( + ctx context.Context, name, fingerprint string, +) (tui.PeopleInferenceStatus, error) { + if fingerprint == "" { + return tui.PeopleInferenceStatus{}, errors.New("people inference profile fingerprint is required") + } + client, err := b.peopleInferenceClient() + if err != nil { + return tui.PeopleInferenceStatus{}, err + } + profile, etag, err := b.peopleInferenceProfileSnapshot(ctx, client, name) + if err != nil { + return tui.PeopleInferenceStatus{}, err + } + if profile.Fingerprint == nil || *profile.Fingerprint != fingerprint { + return tui.PeopleInferenceStatus{}, errors.New("people inference profile changed; reload settings") + } + response, err := client.DeleteSettingsPeopleInferenceProviderWithResponse(ctx, + &generated.DeleteSettingsPeopleInferenceProviderRequestOptions{ + PathParams: &generated.DeleteSettingsPeopleInferenceProviderPath{Name: name}, + Header: &generated.DeleteSettingsPeopleInferenceProviderHeaders{IfMatch: etag}, + }) + if response != nil && response.StatusCode == http.StatusPreconditionFailed { + return tui.PeopleInferenceStatus{}, &tui.SettingsConflictError{Scope: tui.SettingsConflictConfig, + Err: peopleInferenceHTTPError("remove people inference profile", response, err)} + } + if response != nil && response.StatusCode == http.StatusConflict && response.JSON409 != nil && + response.JSON409.ErrorData == "provider_in_use" { + return tui.PeopleInferenceStatus{}, errors.New("disable people inference or configure another profile before removing") + } + if err != nil || response == nil || response.StatusCode != http.StatusOK { + return tui.PeopleInferenceStatus{}, peopleInferenceHTTPError("remove people inference profile", response, err) + } + if response.JSON200 == nil { + return tui.PeopleInferenceStatus{}, errors.New("remove people inference profile: empty response") + } + for _, latest := range response.JSON200.Profiles { + if latest.Name == name { + return tui.PeopleInferenceStatus{}, errors.New("remove people inference profile: profile is still present") + } + } + return tuiPeopleInferenceStatus(response.JSON200), nil +} + +func (b *tuiDaemonSettingsBackend) SelectCodexProfile(ctx context.Context, name string) error { + client, err := b.peopleInferenceClient() + if err != nil { + return err + } + etag, err := b.peopleInferenceETag(ctx, client) + if err != nil { + return err + } + response, err := client.SelectSettingsPeopleInferenceWithResponse(ctx, + &generated.SelectSettingsPeopleInferenceRequestOptions{ + Header: &generated.SelectSettingsPeopleInferenceHeaders{IfMatch: etag}, + Body: &generated.PeopleInferenceSelectionRequest{Name: name}, + }) + if response != nil && response.StatusCode == http.StatusPreconditionFailed { + return &tui.SettingsConflictError{Scope: tui.SettingsConflictConfig, + Err: peopleInferenceHTTPError("select people inference profile", response, err)} + } + if response != nil && response.StatusCode == http.StatusConflict && response.JSON409 != nil { + switch response.JSON409.ErrorData { + case "check_required": + return errors.New("run an exact synthetic check before selecting") + case "consent_required": + return errors.New("grant exact people inference consent before selecting") + } + } + if err != nil || response == nil || response.StatusCode != http.StatusOK { + return peopleInferenceHTTPError("select people inference profile", response, err) + } + return nil +} + +func (b *tuiDaemonSettingsBackend) peopleInferenceClient() (*apiclient.Client, error) { + if b == nil || b.client == nil { + return nil, errors.New("daemon settings client unavailable") + } + return b.client.GeneratedClient() +} + +func (b *tuiDaemonSettingsBackend) peopleInferenceETag(ctx context.Context, client *apiclient.Client) (string, error) { + response, err := client.GetSettingsPeopleInferenceWithResponse(ctx) + if err != nil || response == nil || response.StatusCode != http.StatusOK { + return "", peopleInferenceHTTPError("load people inference config revision", response, err) + } + if response.Headers200 == nil || response.Headers200.ETag == "" { + return "", errors.New("people inference status returned no config ETag") + } + return response.Headers200.ETag, nil +} + +func (b *tuiDaemonSettingsBackend) peopleInferenceProfileSnapshot( + ctx context.Context, client *apiclient.Client, name string, +) (generated.PeopleInferenceProfileSetting, string, error) { + settings, etag, err := b.peopleInferenceSettingsSnapshot(ctx, client) + if err != nil { + return generated.PeopleInferenceProfileSetting{}, "", err + } + for _, profile := range settings.Profiles { + if profile.Name == name { + return profile, etag, nil + } + } + return generated.PeopleInferenceProfileSetting{}, "", errors.New("people inference profile was not found") +} + +func (b *tuiDaemonSettingsBackend) peopleInferenceSettingsSnapshot( + ctx context.Context, client *apiclient.Client, +) (*generated.PeopleInferenceSettingsResponse, string, error) { + response, err := client.GetSettingsPeopleInferenceWithResponse(ctx) + if err != nil || response == nil || response.StatusCode != http.StatusOK { + return nil, "", peopleInferenceHTTPError("load people inference status", response, err) + } + if response.JSON200 == nil || response.Headers200 == nil || response.Headers200.ETag == "" { + return nil, "", errors.New("people inference status returned no body or config ETag") + } + return response.JSON200, response.Headers200.ETag, nil +} + +func peopleInferenceDisclosureText(profile generated.PeopleInferenceProfileSetting) string { + sourceDates := profile.SourceSince + if profile.SourceUntil != nil && *profile.SourceUntil != "" { + sourceDates += " to " + *profile.SourceUntil + } + sensitive := "no" + if profile.AllowSensitive { + sensitive = "yes" + } + lines := []string{ + "Provider: " + profile.Name, + "Fingerprint: " + *profile.Fingerprint, + "Protocol: " + profile.Protocol, + "Model: " + profile.Model, + } + if profile.Endpoint != nil && *profile.Endpoint != "" { + lines = append(lines, "Endpoint: "+*profile.Endpoint) + } + lines = append(lines, + "Sources: "+strings.Join(profile.AllowedSources, ", "), + "Source dates: "+sourceDates, + "Sensitive content: "+sensitive, + "Retention: "+profile.RetentionPosture, + "Training: "+profile.TrainingPosture, + ) + return strings.Join(lines, "\n") +} + +func peopleInferenceHTTPError(operation string, response any, err error) error { + return fmt.Errorf("%s: %w", operation, daemonclient.APIResponseError(response, err)) +} + +func tuiPeopleInferenceStatus(response *generated.PeopleInferenceSettingsResponse) tui.PeopleInferenceStatus { + status := tui.PeopleInferenceStatus{ + ProfileCount: len(response.Profiles), + ConfiguredEnabled: response.ConfiguredEnabled, RunningEnabled: response.RunningEnabled, + PendingRestart: response.PendingRestart, + } + if response.ConfiguredName != nil { + status.Configured = *response.ConfiguredName + } + if response.ConfiguredFingerprint != nil { + status.ConfiguredFingerprint = *response.ConfiguredFingerprint + } + if response.RunningName != nil { + status.Running = *response.RunningName + } + if response.RunningFingerprint != nil { + status.RunningFingerprint = *response.RunningFingerprint + } + return status +} diff --git a/cmd/msgvault/cmd/tui_people_inference_test.go b/cmd/msgvault/cmd/tui_people_inference_test.go new file mode 100644 index 000000000..ac834fefd --- /dev/null +++ b/cmd/msgvault/cmd/tui_people_inference_test.go @@ -0,0 +1,659 @@ +package cmd + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/tui" +) + +func TestTUICodexAdapterUsesGeneratedLoginAndProfileRoutes(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + var calls []string + polls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls = append(calls, r.Method+" "+r.URL.Path) + w.Header().Set("Content-Type", "application/json") + switch r.Method + " " + r.URL.Path { + case "POST /api/v1/settings/people-inference/codex/login": + var body map[string]any + if !assert.NoError(json.NewDecoder(r.Body).Decode(&body)) { + http.Error(w, "invalid request", http.StatusBadRequest) + return + } + assert.Equal(map[string]any{"name": "codex-custom"}, body) + _, _ = io.WriteString(w, `{"session_id":"session-1","verification_url":"https://example.test/device","user_code":"ABCD-EFGH","local_deadline":"2026-09-23T12:05:00Z"}`) + case "GET /api/v1/settings/people-inference/codex/login/session-1": + polls++ + state := "pending" + if polls == 2 { + state = "complete" + } + _, _ = io.WriteString(w, `{"state":"`+state+`"}`) + case "GET /api/v1/settings/people-inference/codex/login/session-1/models": + _, _ = io.WriteString(w, `{"models":[{"id":"codex-model-a","display_name":"Model A","default_reasoning_effort":"medium","supported_efforts":["low","medium"]}]}`) + case "GET /api/v1/settings/people-inference": + w.Header().Set("ETag", `"config-1"`) + _, _ = io.WriteString(w, `{"profiles":[]}`) + case "PUT /api/v1/settings/people-inference/codex/login/session-1/profile": + assert.Equal(`"config-1"`, r.Header.Get("If-Match")) + var body map[string]any + if !assert.NoError(json.NewDecoder(r.Body).Decode(&body)) { + http.Error(w, "invalid request", http.StatusBadRequest) + return + } + assert.Equal(map[string]any{ + "model": "codex-model-a", "reasoning_effort": "low", "retention_posture": "operator assertion: no retention", + "training_posture": "operator assertion: no training", "allowed_sources": []any{"conversation_text"}, + "source_since": "2025-01-01", "source_until": "2025-12-31", "allow_sensitive": false, + }, body) + _, _ = io.WriteString(w, `{"profiles":[{"name":"codex-custom","protocol":"codex_app_server","model":"codex-model-a","output_mode":"native_json_schema","credential_source":"none","allowed_sources":["conversation_text"],"source_since":"2025-01-01","retention_posture":"operator assertion: no retention","training_posture":"operator assertion: no training"}]}`) + case "DELETE /api/v1/settings/people-inference/codex/login/session-1": + _, _ = io.WriteString(w, `{"state":"cancelled"}`) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(server.Close) + backend := newTUISettingsBackend(newTUISettingsDaemonClient(t, server)) + + login, err := backend.StartCodexLogin(context.Background(), "codex-custom") + require.NoError(err) + assert.Equal("session-1", login.SessionID) + assert.Equal("https://example.test/device", login.URL) + assert.Equal("ABCD-EFGH", login.Code) + assert.Equal(time.Date(2026, 9, 23, 12, 5, 0, 0, time.UTC), login.Deadline) + poll, err := backend.PollCodexLogin(context.Background(), login.SessionID) + require.NoError(err) + assert.False(poll.Complete) + poll, err = backend.PollCodexLogin(context.Background(), login.SessionID) + require.NoError(err) + assert.True(poll.Complete) + models, err := backend.ListCodexModels(context.Background(), login.SessionID) + require.NoError(err) + assert.Equal([]tui.CodexModelChoice{{ID: "codex-model-a", DefaultReasoningEffort: "medium", ReasoningEfforts: []string{"low", "medium"}}}, models) + profile, err := backend.SaveCodexProfile(context.Background(), login.SessionID, tui.CodexProfileRequest{ + Name: "codex-custom", Model: models[0].ID, ReasoningEffort: "low", + RetentionPosture: "operator assertion: no retention", TrainingPosture: "operator assertion: no training", + AllowedSources: []string{"conversation_text"}, SourceSince: "2025-01-01", SourceUntil: "2025-12-31", + AllowSensitive: false, + }) + require.NoError(err) + assert.Equal("codex-custom", profile) + require.NoError(backend.CancelCodexLogin(context.Background(), login.SessionID)) + assert.Equal([]string{ + "POST /api/v1/settings/people-inference/codex/login", + "GET /api/v1/settings/people-inference/codex/login/session-1", + "GET /api/v1/settings/people-inference/codex/login/session-1", + "GET /api/v1/settings/people-inference/codex/login/session-1/models", + "GET /api/v1/settings/people-inference", + "PUT /api/v1/settings/people-inference/codex/login/session-1/profile", + "DELETE /api/v1/settings/people-inference/codex/login/session-1", + }, calls) +} + +func TestTUICodexProfileSaveReportsConfigConflict(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.Method + " " + r.URL.Path { + case "GET /api/v1/settings/people-inference": + w.Header().Set("ETag", `"config-1"`) + _, _ = io.WriteString(w, `{"profiles":[]}`) + case "PUT /api/v1/settings/people-inference/codex/login/session-1/profile": + assert.Equal(`"config-1"`, r.Header.Get("If-Match")) + w.WriteHeader(http.StatusPreconditionFailed) + _, _ = io.WriteString(w, `{"error":"settings_conflict"}`) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(server.Close) + backend := newTUISettingsBackend(newTUISettingsDaemonClient(t, server)) + _, err := backend.SaveCodexProfile(context.Background(), "session-1", tui.CodexProfileRequest{ + Name: "codex-custom", Model: "codex-model-a", ReasoningEffort: "low", + RetentionPosture: "operator assertion", TrainingPosture: "operator assertion", + AllowedSources: []string{"conversation_text"}, SourceSince: "2025-01-01", + }) + require.Error(err) + var conflict *tui.SettingsConflictError + require.ErrorAs(err, &conflict) + assert.Equal(tui.SettingsConflictConfig, conflict.Scope) +} + +func TestTUIPeopleInferenceBackendUsesGeneratedStatusRoute(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + var calls []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls = append(calls, r.Method+" "+r.URL.Path) + w.Header().Set("Content-Type", "application/json") + switch r.Method + " " + r.URL.Path { + case "GET /api/v1/settings/people-inference": + w.Header().Set("ETag", `"config-1"`) + _, _ = io.WriteString(w, `{"profiles":[{"name":"old"},{"name":"backup"}],"configured_name":"old","configured_fingerprint":"fp-configured","running_name":"old","running_fingerprint":"fp-running","configured_enabled":true,"running_enabled":true,"pending_restart":false}`) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(server.Close) + backend := newTUISettingsBackend(newTUISettingsDaemonClient(t, server)) + + status, err := backend.LoadPeopleInferenceStatus(context.Background()) + require.NoError(err) + assert.Equal(2, status.ProfileCount) + assert.Equal("old", status.Configured) + assert.Equal("old", status.Running) + assert.Equal("fp-configured", status.ConfiguredFingerprint) + assert.Equal("fp-running", status.RunningFingerprint) + assert.True(status.ConfiguredEnabled) + assert.True(status.RunningEnabled) + assert.False(status.PendingRestart) + + assert.Equal([]string{"GET /api/v1/settings/people-inference"}, calls) +} + +func TestTUIPeopleInferenceBackendCheckUsesExactProfile(t *testing.T) { + require := require.New(t) + + assert := assert.New(t) + var calls []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls = append(calls, r.Method+" "+r.URL.Path) + w.Header().Set("Content-Type", "application/json") + switch r.Method + " " + r.URL.Path { + case "GET /api/v1/settings/people-inference": + w.Header().Set("ETag", `"config-1"`) + _, _ = io.WriteString(w, `{"profiles":[{"name":"codex-profile","fingerprint":"fp-1","protocol":"codex_app_server","model":"codex-model","endpoint":"https://api.example.test/v1","allowed_sources":["conversation_text"],"source_since":"2025-01-01","source_until":"2025-12-31","allow_sensitive":false,"retention_posture":"operator-confirmed","training_posture":"operator-confirmed","checked":true}]}`) + case "POST /api/v1/settings/people-inference/providers/codex-profile/check": + assert.Equal(`"config-1"`, r.Header.Get("If-Match")) + _, _ = io.WriteString(w, `{"ok":true,"fingerprint":"fp-1","model":"codex-model","usage":{}}`) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(server.Close) + backend := newTUISettingsBackend(newTUISettingsDaemonClient(t, server)) + + disclosure, err := backend.CheckCodexProfile(context.Background(), "codex-profile") + require.NoError(err) + assert.Equal("codex-profile", disclosure.Profile) + assert.Equal("fp-1", disclosure.Fingerprint) + for _, field := range []string{"fp-1", "codex_app_server", "codex-model", "https://api.example.test/v1", "conversation_text", "2025-01-01", "2025-12-31", "operator-confirmed", "Sensitive content: no"} { + assert.Contains(disclosure.Text, field) + } + assert.Equal([]string{ + "GET /api/v1/settings/people-inference", + "POST /api/v1/settings/people-inference/providers/codex-profile/check", + "GET /api/v1/settings/people-inference", + }, calls) +} + +func TestTUIPeopleInferenceBackendCheckRejectsChangedFingerprint(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.Method == http.MethodGet { + w.Header().Set("ETag", `"config-1"`) + _, _ = io.WriteString(w, `{"profiles":[{"name":"codex-profile","fingerprint":"fp-new","model":"codex-model"}]}`) + return + } + _, _ = io.WriteString(w, `{"ok":true,"fingerprint":"fp-old","model":"codex-model","usage":{}}`) + })) + t.Cleanup(server.Close) + backend := newTUISettingsBackend(newTUISettingsDaemonClient(t, server)) + + _, err := backend.CheckCodexProfile(context.Background(), "codex-profile") + require.Error(err) + assert.Contains(err.Error(), "changed") +} + +func TestTUIPeopleInferenceBackendConsentSendsExactConfirmation(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + var calls []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls = append(calls, r.Method+" "+r.URL.Path) + w.Header().Set("Content-Type", "application/json") + switch r.Method + " " + r.URL.Path { + case "GET /api/v1/settings/people-inference": + w.Header().Set("ETag", `"config-2"`) + _, _ = io.WriteString(w, `{"profiles":[{"name":"codex-profile","fingerprint":"fp-1","checked":true}]}`) + case "POST /api/v1/settings/people-inference/providers/codex-profile/consent": + assert.Equal(`"config-2"`, r.Header.Get("If-Match")) + var body map[string]any + if !assert.NoError(json.NewDecoder(r.Body).Decode(&body)) { + http.Error(w, "invalid request", http.StatusBadRequest) + return + } + assert.Equal(map[string]any{"confirmed": true, "fingerprint": "fp-1"}, body) + _, _ = io.WriteString(w, `{"profiles":[{"name":"codex-profile","fingerprint":"fp-1","consent_active":true}]}`) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(server.Close) + backend := newTUISettingsBackend(newTUISettingsDaemonClient(t, server)) + + require.NoError(backend.ConsentCodexProfile(context.Background(), "codex-profile", "fp-1")) + assert.Equal([]string{ + "GET /api/v1/settings/people-inference", + "POST /api/v1/settings/people-inference/providers/codex-profile/consent", + }, calls) +} + +func TestTUIPeopleInferenceBackendConsentRejectsStaleFingerprint(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + var calls []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls = append(calls, r.Method+" "+r.URL.Path) + w.Header().Set("Content-Type", "application/json") + w.Header().Set("ETag", `"config-2"`) + _, _ = io.WriteString(w, `{"profiles":[{"name":"codex-profile","fingerprint":"fp-new","checked":true}]}`) + })) + t.Cleanup(server.Close) + backend := newTUISettingsBackend(newTUISettingsDaemonClient(t, server)) + + err := backend.ConsentCodexProfile(context.Background(), "codex-profile", "fp-old") + require.Error(err) + assert.Contains(err.Error(), "changed") + assert.Equal([]string{"GET /api/v1/settings/people-inference"}, calls) +} + +func TestTUIPeopleInferenceBackendRevokeUsesExactFingerprint(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + var calls []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls = append(calls, r.Method+" "+r.URL.Path) + w.Header().Set("Content-Type", "application/json") + switch r.Method + " " + r.URL.Path { + case "GET /api/v1/settings/people-inference": + w.Header().Set("ETag", `"config-3"`) + _, _ = io.WriteString(w, `{"profiles":[{"name":"codex-profile","fingerprint":"fp-1","consent_active":true}],"configured_name":"codex-profile","configured_fingerprint":"fp-1"}`) + case "POST /api/v1/settings/people-inference/providers/codex-profile/revoke": + assert.Equal(`"config-3"`, r.Header.Get("If-Match")) + _, _ = io.WriteString(w, `{"profiles":[{"name":"codex-profile","fingerprint":"fp-1","consent_active":false}],"configured_name":"codex-profile","configured_fingerprint":"fp-1"}`) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(server.Close) + backend := newTUISettingsBackend(newTUISettingsDaemonClient(t, server)) + + status, err := backend.RevokePeopleInferenceConsent(context.Background(), "codex-profile", "fp-1") + require.NoError(err) + assert.Equal("fp-1", status.ConfiguredFingerprint) + assert.Equal([]string{ + "GET /api/v1/settings/people-inference", + "POST /api/v1/settings/people-inference/providers/codex-profile/revoke", + }, calls) +} + +func TestTUIPeopleInferenceBackendDisableUsesExactFingerprint(t *testing.T) { + require := require.New(t) + + assert := assert.New(t) + var calls []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls = append(calls, r.Method+" "+r.URL.Path) + w.Header().Set("Content-Type", "application/json") + switch r.Method + " " + r.URL.Path { + case "GET /api/v1/settings/people-inference": + w.Header().Set("ETag", `"config-3"`) + _, _ = io.WriteString(w, `{"profiles":[],"configured_name":"codex-profile","configured_fingerprint":"fp-1","configured_enabled":true,"running_enabled":true}`) + case "POST /api/v1/settings/people-inference/disable": + assert.Equal(`"config-3"`, r.Header.Get("If-Match")) + _, _ = io.WriteString(w, `{"profiles":[],"configured_name":"codex-profile","configured_fingerprint":"fp-1","configured_enabled":false,"running_enabled":true,"pending_restart":true}`) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(server.Close) + backend := newTUISettingsBackend(newTUISettingsDaemonClient(t, server)) + + status, err := backend.DisablePeopleInference(context.Background(), "fp-1") + require.NoError(err) + assert.False(status.ConfiguredEnabled) + assert.True(status.PendingRestart) + assert.Equal([]string{ + "GET /api/v1/settings/people-inference", + "POST /api/v1/settings/people-inference/disable", + }, calls) +} + +func TestTUIPeopleInferenceBackendRemoveUsesExactFingerprint(t *testing.T) { + require := require.New(t) + + assert := assert.New(t) + var calls []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls = append(calls, r.Method+" "+r.URL.Path) + w.Header().Set("Content-Type", "application/json") + switch r.Method + " " + r.URL.Path { + case "GET /api/v1/settings/people-inference": + w.Header().Set("ETag", `"config-4"`) + _, _ = io.WriteString(w, `{"profiles":[{"name":"router","fingerprint":"fp-router"},{"name":"backup","fingerprint":"fp-backup"}],"configured_name":"router","configured_fingerprint":"fp-router","configured_enabled":false}`) + case "DELETE /api/v1/settings/people-inference/providers/router": + assert.Equal(`"config-4"`, r.Header.Get("If-Match")) + _, _ = io.WriteString(w, `{"profiles":[{"name":"backup","fingerprint":"fp-backup"}],"configured_name":"backup","configured_fingerprint":"fp-backup","pending_restart":true}`) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(server.Close) + backend := newTUISettingsBackend(newTUISettingsDaemonClient(t, server)) + + status, err := backend.RemovePeopleInferenceProfile(context.Background(), "router", "fp-router") + require.NoError(err) + assert.Equal("backup", status.Configured) + assert.True(status.PendingRestart) + assert.Equal([]string{ + "GET /api/v1/settings/people-inference", + "DELETE /api/v1/settings/people-inference/providers/router", + }, calls) +} + +func TestTUIPeopleInferenceBackendRemoveRejectsChangedFingerprint(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + var calls []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls = append(calls, r.Method+" "+r.URL.Path) + w.Header().Set("Content-Type", "application/json") + w.Header().Set("ETag", `"config-new"`) + _, _ = io.WriteString(w, `{"profiles":[{"name":"router","fingerprint":"fp-new"}],"configured_enabled":false}`) + })) + t.Cleanup(server.Close) + backend := newTUISettingsBackend(newTUISettingsDaemonClient(t, server)) + + _, err := backend.RemovePeopleInferenceProfile(context.Background(), "router", "fp-old") + require.Error(err) + assert.Contains(err.Error(), "changed") + assert.Equal([]string{"GET /api/v1/settings/people-inference"}, calls) +} + +func TestTUIPeopleInferenceBackendRemoveReportsConflict(t *testing.T) { + for _, status := range []int{http.StatusConflict, http.StatusPreconditionFailed} { + t.Run(http.StatusText(status), func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.Method == http.MethodGet { + w.Header().Set("ETag", `"config-4"`) + _, _ = io.WriteString(w, `{"profiles":[{"name":"router","fingerprint":"fp-router"}],"configured_enabled":false}`) + return + } + assert.Equal(`"config-4"`, r.Header.Get("If-Match")) + w.WriteHeader(status) + _, _ = io.WriteString(w, `{"error":"provider_in_use"}`) + })) + t.Cleanup(server.Close) + backend := newTUISettingsBackend(newTUISettingsDaemonClient(t, server)) + + _, err := backend.RemovePeopleInferenceProfile(context.Background(), "router", "fp-router") + require.Error(err) + if status == http.StatusPreconditionFailed { + var conflict *tui.SettingsConflictError + require.ErrorAs(err, &conflict) + assert.Equal(tui.SettingsConflictConfig, conflict.Scope) + } else { + assert.Contains(err.Error(), "another profile") + } + }) + } +} + +func TestTUIPeopleInferenceBackendRevokeAndDisableRejectStaleFingerprint(t *testing.T) { + for _, operation := range []string{"revoke", "disable"} { + t.Run(operation, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + var calls []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls = append(calls, r.Method+" "+r.URL.Path) + w.Header().Set("Content-Type", "application/json") + w.Header().Set("ETag", `"config-new"`) + _, _ = io.WriteString(w, `{"profiles":[{"name":"codex-profile","fingerprint":"fp-new"}],"configured_fingerprint":"fp-new","configured_enabled":true}`) + })) + t.Cleanup(server.Close) + backend := newTUISettingsBackend(newTUISettingsDaemonClient(t, server)) + var err error + if operation == "revoke" { + _, err = backend.RevokePeopleInferenceConsent(context.Background(), "codex-profile", "fp-old") + } else { + _, err = backend.DisablePeopleInference(context.Background(), "fp-old") + } + require.Error(err) + assert.Contains(err.Error(), "changed") + assert.Equal([]string{"GET /api/v1/settings/people-inference"}, calls) + }) + } +} + +func TestTUIPeopleInferenceBackendRevokeAndDisablePreserveConfigConflict(t *testing.T) { + for _, operation := range []string{"revoke", "disable"} { + t.Run(operation, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.Method == http.MethodGet { + w.Header().Set("ETag", `"config-old"`) + _, _ = io.WriteString(w, `{"profiles":[{"name":"codex-profile","fingerprint":"fp-1"}],"configured_fingerprint":"fp-1","configured_enabled":true}`) + return + } + assert.Equal(`"config-old"`, r.Header.Get("If-Match")) + w.WriteHeader(http.StatusPreconditionFailed) + _, _ = io.WriteString(w, `{"error":"settings_conflict"}`) + })) + t.Cleanup(server.Close) + backend := newTUISettingsBackend(newTUISettingsDaemonClient(t, server)) + var err error + if operation == "revoke" { + _, err = backend.RevokePeopleInferenceConsent(context.Background(), "codex-profile", "fp-1") + } else { + _, err = backend.DisablePeopleInference(context.Background(), "fp-1") + } + var conflict *tui.SettingsConflictError + require.ErrorAs(err, &conflict) + assert.Equal(tui.SettingsConflictConfig, conflict.Scope) + }) + } +} + +func TestTUIPeopleInferenceBackendCheckAndConsentConflicts(t *testing.T) { + for _, test := range []struct { + name string + operation string + status int + body string + want string + conflict bool + }{ + {name: "check config conflict", operation: "check", status: 412, body: `{"error":"settings_conflict"}`, conflict: true}, + {name: "consent config conflict", operation: "consent", status: 412, body: `{"error":"settings_conflict"}`, conflict: true}, + {name: "consent changed disclosure", operation: "consent", status: 409, body: `{"error":"consent_disclosure_changed"}`, want: "disclosure changed"}, + {name: "consent check missing", operation: "consent", status: 409, body: `{"error":"check_required"}`, want: "synthetic check"}, + {name: "check daemon error", operation: "check", status: 500, body: `{"error":"settings_read_failed","message":"people inference configuration is unavailable"}`, want: "500"}, + {name: "consent daemon error", operation: "consent", status: 500, body: `{"error":"settings_read_failed","message":"people inference configuration is unavailable"}`, want: "500"}, + } { + t.Run(test.name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.Method == http.MethodGet { + w.Header().Set("ETag", `"config-1"`) + _, _ = io.WriteString(w, `{"profiles":[{"name":"codex-profile","fingerprint":"fp-1","checked":true}]}`) + return + } + assert.Equal(`"config-1"`, r.Header.Get("If-Match")) + w.WriteHeader(test.status) + _, _ = io.WriteString(w, test.body) + })) + t.Cleanup(server.Close) + backend := newTUISettingsBackend(newTUISettingsDaemonClient(t, server)) + var err error + if test.operation == "check" { + _, err = backend.CheckCodexProfile(context.Background(), "codex-profile") + } else { + err = backend.ConsentCodexProfile(context.Background(), "codex-profile", "fp-1") + } + require.Error(err) + if test.conflict { + var conflict *tui.SettingsConflictError + require.ErrorAs(err, &conflict) + assert.Equal(tui.SettingsConflictConfig, conflict.Scope) + } else { + assert.Contains(err.Error(), test.want) + } + }) + } +} + +func TestTUIPeopleInferenceBackendSelectPreservesConfigConflict(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.Method + " " + r.URL.Path { + case "GET /api/v1/settings/people-inference": + w.Header().Set("ETag", `"config-old"`) + _, _ = io.WriteString(w, `{"profiles":[],"pending_restart":false}`) + case "POST /api/v1/settings/people-inference/select": + assert.Equal(`"config-old"`, r.Header.Get("If-Match")) + var body map[string]any + if !assert.NoError(json.NewDecoder(r.Body).Decode(&body)) { + http.Error(w, "invalid request", http.StatusBadRequest) + return + } + assert.Equal("codex-profile", body["name"]) + w.WriteHeader(http.StatusPreconditionFailed) + _, _ = io.WriteString(w, `{}`) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(server.Close) + backend := newTUISettingsBackend(newTUISettingsDaemonClient(t, server)) + + err := backend.SelectCodexProfile(context.Background(), "codex-profile") + var conflict *tui.SettingsConflictError + require.ErrorAs(err, &conflict) + assert.Equal(tui.SettingsConflictConfig, conflict.Scope) +} + +func TestTUIPeopleInferenceBackendSelectReportsDaemonGate(t *testing.T) { + for _, test := range []struct { + code string + want string + }{ + {code: "check_required", want: "run an exact synthetic check before selecting"}, + {code: "consent_required", want: "grant exact people inference consent before selecting"}, + } { + t.Run(test.code, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.Method == http.MethodGet { + w.Header().Set("ETag", `"config-1"`) + _, _ = io.WriteString(w, `{"profiles":[]}`) + return + } + w.WriteHeader(http.StatusConflict) + _, _ = io.WriteString(w, `{"error":"`+test.code+`"}`) + })) + t.Cleanup(server.Close) + backend := newTUISettingsBackend(newTUISettingsDaemonClient(t, server)) + + err := backend.SelectCodexProfile(context.Background(), "codex-profile") + require.Error(err) + assert.Equal(test.want, err.Error()) + }) + } +} + +func TestTUIPeopleInferenceBackendRejectsServerFailure(t *testing.T) { + for _, operation := range []string{"status", "select"} { + t.Run(operation, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if operation != "status" && r.Method == http.MethodGet { + w.Header().Set("ETag", `"config-1"`) + _, _ = io.WriteString(w, `{"profiles":[]}`) + return + } + w.WriteHeader(http.StatusInternalServerError) + _, _ = io.WriteString(w, `{"error":"settings_read_failed","message":"people inference configuration is unavailable"}`) + })) + t.Cleanup(server.Close) + backend := newTUISettingsBackend(newTUISettingsDaemonClient(t, server)) + var err error + switch operation { + case "status": + _, err = backend.LoadPeopleInferenceStatus(context.Background()) + case "select": + err = backend.SelectCodexProfile(context.Background(), "codex-profile") + } + require.Error(err) + assert.Contains(err.Error(), "500") + assert.Contains(err.Error(), "people inference configuration is unavailable") + }) + } +} + +func TestTUICodexAdapterReportsLoginFailure(t *testing.T) { + for _, state := range []string{"failed", "cancelled"} { + t.Run(state, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"state":"`+state+`"}`) + })) + t.Cleanup(server.Close) + backend := newTUISettingsBackend(newTUISettingsDaemonClient(t, server)) + poll, err := backend.PollCodexLogin(t.Context(), "session-1") + require.ErrorContains(t, err, state) + assert.True(t, poll.Failed) + }) + } +} + +func TestTUICodexAdapterStopsPollingMissingSession(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusNotFound) + _, _ = io.WriteString(w, `{"error":"codex_login_not_found","message":"Codex device login was not found"}`) + })) + t.Cleanup(server.Close) + backend := newTUISettingsBackend(newTUISettingsDaemonClient(t, server)) + poll, err := backend.PollCodexLogin(t.Context(), "session-1") + require.ErrorContains(t, err, "Codex device login was not found") + assert.True(t, poll.Failed) +} diff --git a/cmd/msgvault/cmd/tui_test.go b/cmd/msgvault/cmd/tui_test.go index 4bafbf691..2d7f640d9 100644 --- a/cmd/msgvault/cmd/tui_test.go +++ b/cmd/msgvault/cmd/tui_test.go @@ -25,17 +25,19 @@ import ( ) func TestOpenTUIEngineUsesConfiguredRemoteHTTP(t *testing.T) { + cfg := lifecycleTestConfig(t.TempDir()) + assert := assert.New(t) require := require.New(t) var requests atomic.Int32 srv := httptest.NewServer(tuiAccountsHandler(&requests, "remote@example.com")) t.Cleanup(srv.Close) - withTUIConfig(t, lifecycleTestConfig(t.TempDir())) + ctx := withTUIConfig(t, cfg) cfg.Remote.URL = srv.URL cfg.Remote.AllowInsecure = true - backend, err := openTUIBackend(context.Background()) + backend, err := openTUIBackend(ctx) require.NoError( err, "openTUIBackend") @@ -67,7 +69,7 @@ func TestOpenTUIEngineLocalFlagUsesLocalDaemonHTTP(t *testing.T) { localCfg.Remote.URL = "http://configured-daemonclient.example:8080" localCfg.Remote.AllowInsecure = true localCfg.Server.APIKey = "local-daemon-secret" - withTUIConfig(t, localCfg) + ctx := withTUIConfig(t, localCfg) forceLocalTUI = true var requests atomic.Int32 @@ -99,7 +101,7 @@ func TestOpenTUIEngineLocalFlagUsesLocalDaemonHTTP(t *testing.T) { require.NoError( err, "write runtime") - backend, err := openTUIBackend(context.Background()) + backend, err := openTUIBackend(ctx) require.NoError( err, "openTUIBackend") @@ -121,19 +123,14 @@ func TestOpenTUIEngineLocalFlagUsesLocalDaemonHTTP(t *testing.T) { assert.Equal(int32(1), requests.Load()) } -func withTUIConfig(t *testing.T, c *config.Config) { +func withTUIConfig(t *testing.T, c *config.Config) context.Context { t.Helper() - oldCfg := cfg - oldUseLocal := useLocal oldForceLocalTUI := forceLocalTUI - cfg = c - useLocal = false forceLocalTUI = false t.Cleanup(func() { - cfg = oldCfg - useLocal = oldUseLocal forceLocalTUI = oldForceLocalTUI }) + return testInvocationContext(t.Context(), c, invocationOptions{}) } func tuiAccountsHandler(requests *atomic.Int32, email string) http.Handler { diff --git a/cmd/msgvault/cmd/unpack_attachments.go b/cmd/msgvault/cmd/unpack_attachments.go index ee89e8362..3fe5d3885 100644 --- a/cmd/msgvault/cmd/unpack_attachments.go +++ b/cmd/msgvault/cmd/unpack_attachments.go @@ -51,7 +51,12 @@ func refuseUnpackWithLiveDaemon(dataDir string) error { } func runUnpackAttachmentsLocal(cmd *cobra.Command) (runErr error) { - if IsRemoteMode() { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + if IsRemoteMode(state) { return errors.New( "unpack-attachments is local-only; run it on the archive host, " + "or pass --local to select this machine's local archive intentionally") @@ -71,7 +76,7 @@ func runUnpackAttachmentsLocal(cmd *cobra.Command) (runErr error) { if err := refuseUnpackWithLiveDaemon(cfg.Data.DataDir); err != nil { return err } - s, cleanup, err := openWritableStoreAndInit() + s, cleanup, err := openWritableStoreAndInitForInvocation(state) if err != nil { return err } diff --git a/cmd/msgvault/cmd/unpack_attachments_test.go b/cmd/msgvault/cmd/unpack_attachments_test.go index dc92dcb7a..a52bea21d 100644 --- a/cmd/msgvault/cmd/unpack_attachments_test.go +++ b/cmd/msgvault/cmd/unpack_attachments_test.go @@ -69,6 +69,9 @@ func TestRefuseUnpackWithLiveDaemon(t *testing.T) { } func TestRunUnpackAttachmentsLocalRejectsConfiguredRemote(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) savedCfg := cfg savedUseLocal := useLocal @@ -82,8 +85,11 @@ func TestRunUnpackAttachmentsLocalRejectsConfiguredRemote(t *testing.T) { Data: config.DataConfig{DataDir: dataDir}, Remote: config.RemoteConfig{URL: "https://vault.example.com"}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = false cmd := &cobra.Command{} + cmd.SetContext(testCtx) cmd.SetOut(io.Discard) err := runUnpackAttachmentsLocal(cmd) @@ -98,6 +104,9 @@ func TestRunUnpackAttachmentsLocalRejectsConfiguredRemote(t *testing.T) { } func TestRunUnpackAttachmentsLocalHoldsDaemonLeaseBeforePostgresStoreOpen(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) savedCfg := cfg savedUseLocal := useLocal @@ -113,6 +122,8 @@ func TestRunUnpackAttachmentsLocalHoldsDaemonLeaseBeforePostgresStoreOpen(t *tes DataDir: dataDir, DatabaseURL: "postgres://user:pass@example.com:5432/msgvault", }} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = false server := httptest.NewServer(daemon.NewPingHandler(daemon.PingHandlerOptions{ Service: daemonService, @@ -139,6 +150,7 @@ func TestRunUnpackAttachmentsLocalHoldsDaemonLeaseBeforePostgresStoreOpen(t *tes <-release } cmd := &cobra.Command{} + cmd.SetContext(testCtx) cmd.SetOut(io.Discard) errCh := make(chan error, 1) go func() { errCh <- runUnpackAttachmentsLocal(cmd) }() @@ -158,6 +170,9 @@ func TestRunUnpackAttachmentsLocalHoldsDaemonLeaseBeforePostgresStoreOpen(t *tes } func TestRunUnpackAttachmentsLocalReportsHeldDaemonLease(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) savedCfg := cfg savedUseLocal := useLocal @@ -170,11 +185,14 @@ func TestRunUnpackAttachmentsLocalReportsHeldDaemonLease(t *testing.T) { DataDir: dataDir, DatabaseURL: "postgres://user:pass@example.com:5432/msgvault", }} + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = false owner, err := tryAcquireDaemonOwnerLock(dataDir) require.NoError(err) t.Cleanup(func() { require.NoError(owner.Close()) }) cmd := &cobra.Command{} + cmd.SetContext(testCtx) cmd.SetOut(io.Discard) err = runUnpackAttachmentsLocal(cmd) diff --git a/cmd/msgvault/cmd/update.go b/cmd/msgvault/cmd/update.go index 4e6569779..38a5618a5 100644 --- a/cmd/msgvault/cmd/update.go +++ b/cmd/msgvault/cmd/update.go @@ -1,9 +1,11 @@ package cmd import ( + "context" "errors" "fmt" "io" + "log/slog" "os" "path/filepath" "runtime" @@ -105,11 +107,20 @@ official release over a dev build.`, if err := performUpdateWithDaemonLifecycle( info, progressFn, - loadDaemonConfigForUpdate, - stopLocalDaemonsForUpdate, + func() (*config.Config, error) { + inv := invocationFromCommand(cmd) + if inv == nil { + return nil, errors.New("configuration is unavailable") + } + options := inv.options + return loadDaemonConfigForUpdate(options) + }, + func(c *config.Config) (updateDaemonStopResult, error) { + return stopLocalDaemonsForUpdate(c, invocationFromCommand(cmd).logger) + }, update.PerformUpdate, func(c *config.Config, result updateDaemonStopResult) error { - return restartDaemonAfterUpdate(c, result, installExecutablePath) + return restartDaemonAfterUpdate(c, result, installExecutablePath, invocationFromCommand(cmd)) }, ); err != nil { return err @@ -176,8 +187,8 @@ func performUpdateWithDaemonLifecycle( return nil } -func loadDaemonConfigForUpdate() (*config.Config, error) { - c, err := config.Load(cfgFile, homeDir) +func loadDaemonConfigForUpdate(options invocationOptions) (*config.Config, error) { + c, err := config.Load(options.cfgFile, options.homeDir) if err != nil { return nil, err } @@ -187,7 +198,7 @@ func loadDaemonConfigForUpdate() (*config.Config, error) { return c, nil } -func stopLocalDaemonsForUpdate(c *config.Config) (updateDaemonStopResult, error) { +func stopLocalDaemonsForUpdate(c *config.Config, logger *slog.Logger) (updateDaemonStopResult, error) { var result updateDaemonStopResult if c == nil { return result, errors.New("nil config") @@ -198,7 +209,7 @@ func stopLocalDaemonsForUpdate(c *config.Config) (updateDaemonStopResult, error) } for _, rec := range records { rt := daemonRuntimeFromRecord(rec) - if err := stopDaemonRuntimeForUpgrade(*c, rt); err != nil { + if err := stopDaemonRuntimeForUpgrade(*c, rt, logger); err != nil { return result, err } result.Stopped = true @@ -206,11 +217,18 @@ func stopLocalDaemonsForUpdate(c *config.Config) (updateDaemonStopResult, error) return result, nil } -func restartDaemonAfterUpdate(c *config.Config, result updateDaemonStopResult, executablePath string) error { +func restartDaemonAfterUpdate(c *config.Config, result updateDaemonStopResult, executablePath string, state *invocation) error { if !result.Stopped { return nil } cmd := &cobra.Command{Use: "msgvault update daemon-restart"} + // A command that never runs through ExecuteC has no context, and the + // readiness wait dereferences the one it is given. + ctx := context.Background() + if state != nil { + ctx = withInvocation(ctx, state) + } + cmd.SetContext(ctx) cmd.SetOut(io.Discard) cmd.SetErr(io.Discard) return runServeStartWithOptions(cmd, c, backgroundServeStartOptions{ExecutablePath: executablePath}) diff --git a/cmd/msgvault/cmd/update_account_test.go b/cmd/msgvault/cmd/update_account_test.go index 67c245232..0b90d070a 100644 --- a/cmd/msgvault/cmd/update_account_test.go +++ b/cmd/msgvault/cmd/update_account_test.go @@ -16,6 +16,9 @@ import ( ) func TestUpdateAccountUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) dataDir := t.TempDir() @@ -63,7 +66,10 @@ func TestUpdateAccountUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { Data: config.DataConfig{DataDir: dataDir}, Remote: config.RemoteConfig{URL: "http://configured-daemonclient.invalid"}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true updateDisplayName = "Work" var stdout bytes.Buffer @@ -73,6 +79,7 @@ func TestUpdateAccountUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { Args: updateAccountCmd.Args, RunE: updateAccountCmd.RunE, } + cmd.SetContext(testCtx) cmd.SetOut(&stdout) cmd.SetErr(&stderr) cmd.SetArgs([]string{"alice@example.com"}) @@ -86,6 +93,9 @@ func TestUpdateAccountUsesLocalDaemonHTTPAndPreservesOutput(t *testing.T) { } func TestUpdateAccountSourceIDUsesTypedDaemonRequest(t *testing.T) { + cfg := testConfigValue() + useLocal := false + require := require.New(t) assert := assert.New(t) @@ -133,9 +143,13 @@ func TestUpdateAccountSourceIDUsesTypedDaemonRequest(t *testing.T) { Data: config.DataConfig{DataDir: dataDir}, Remote: config.RemoteConfig{URL: server.URL}, } + testCtx := testInvocationContext(t.Context(), cfg, invocationOptions{}) + _ = testCtx useLocal = true + invocationFromContext(testCtx).options.useLocal = true cmd := newUpdateAccountCmd() + cmd.SetContext(testCtx) cmd.SetArgs([]string{"--source-id", "42", "--display-name", "Work"}) require.NoError(cmd.Execute()) assert.Equal(int32(1), requests.Load()) diff --git a/cmd/msgvault/cmd/update_test.go b/cmd/msgvault/cmd/update_test.go index b6fcdc8e7..693e180ef 100644 --- a/cmd/msgvault/cmd/update_test.go +++ b/cmd/msgvault/cmd/update_test.go @@ -3,6 +3,7 @@ package cmd import ( "context" "errors" + "log/slog" "net" "os" "path/filepath" @@ -200,12 +201,12 @@ func TestStopLocalDaemonsForUpdateStopsLiveRuntimeRecords(t *testing.T) { require.NoError(err, "write runtime") var stoppedPID int - stubStopDaemonRuntimeForUpgrade(t, func(_ config.Config, rt *DaemonRuntime) error { + stubStopDaemonRuntimeForUpgrade(t, func(_ config.Config, rt *DaemonRuntime, _ *slog.Logger) error { stoppedPID = rt.Record.PID return nil }) - result, err := stopLocalDaemonsForUpdate(lifecycleTestConfig(dataDir)) + result, err := stopLocalDaemonsForUpdate(lifecycleTestConfig(dataDir), testDiscardLogger()) require.NoError(err, "stop local daemons") assert.True(result.Stopped, "stopped") @@ -244,8 +245,50 @@ func TestRestartDaemonAfterUpdateUsesInstalledExecutablePath(t *testing.T) { }, true, nil }) - err := restartDaemonAfterUpdate(cfg, updateDaemonStopResult{Stopped: true}, installedExe) + err := restartDaemonAfterUpdate(cfg, updateDaemonStopResult{Stopped: true}, installedExe, testInvocationWithConfig(cfg)) require.NoError(err, "restart daemon") assert.Equal(installedExe, gotExecutable, "restart executable") } + +// The restart builds its own command, and a cobra command only gets a context +// from ExecuteC. Without one, cmd.Context() is nil and the first context call +// in the readiness wait panics. +func TestRestartDaemonAfterUpdatePassesUsableContext(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + cfg := lifecycleTestConfig(t.TempDir()) + installedExe := filepath.Join(t.TempDir(), "bin", updateExecutableName()) + waitCh := make(chan error) + var gotCtx context.Context + + stubStartServeBackgroundProcess(t, func( + _ *config.Config, + _ backgroundServeStartOptions, + ) (*backgroundServeProcess, error) { + return &backgroundServeProcess{ + PID: 991, + LogPath: "/tmp/msgvault-serve.log", + Wait: waitCh, + }, nil + }) + stubWaitForBackgroundServeReady(t, func( + ctx context.Context, + _ string, + _ <-chan error, + _ time.Duration, + ) (*DaemonRuntime, bool, error) { + gotCtx = ctx + return &DaemonRuntime{ + Record: daemon.RuntimeRecord{PID: 991}, + Host: net.IPv4(127, 0, 0, 1).String(), + Port: 9091, + }, true, nil + }) + + err := restartDaemonAfterUpdate(cfg, updateDaemonStopResult{Stopped: true}, installedExe, testInvocationWithConfig(cfg)) + + require.NoError(err, "restart daemon") + require.NotNil(gotCtx, "readiness wait received a nil context") + assert.NoError(gotCtx.Err(), "context usable") +} diff --git a/cmd/msgvault/cmd/vector_test_helpers_test.go b/cmd/msgvault/cmd/vector_test_helpers_test.go index a59597c15..957d12a0c 100644 --- a/cmd/msgvault/cmd/vector_test_helpers_test.go +++ b/cmd/msgvault/cmd/vector_test_helpers_test.go @@ -1,18 +1,14 @@ package cmd import ( + "context" "testing" "go.kenn.io/msgvault/internal/config" ) -// withTestConfig swaps the package-level cfg for the duration of a test -// and restores the previous value on cleanup. Untagged (no build -// constraint) so both the vector-tagged precheck tests and the untagged -// background-init tests (Task 5) can share it. -func withTestConfig(t *testing.T, c *config.Config) { +// withTestConfig binds a configuration to the test invocation context. +func withTestConfig(t *testing.T, c *config.Config) context.Context { t.Helper() - prev := cfg - cfg = c - t.Cleanup(func() { cfg = prev }) + return testInvocationContext(t.Context(), c, invocationOptions{}) } diff --git a/cmd/msgvault/cmd/verify.go b/cmd/msgvault/cmd/verify.go index 39aa717f9..d2c387b14 100644 --- a/cmd/msgvault/cmd/verify.go +++ b/cmd/msgvault/cmd/verify.go @@ -104,9 +104,15 @@ Examples: } func runVerifyLocal(cmd *cobra.Command, args []string) error { + state := invocationFromCommand(cmd) + if state == nil || state.cfg == nil || state.logger == nil { + return errors.New("configuration is unavailable") + } + cfg := state.cfg + logger := state.logger email := args[0] - release, err := acquireDirectSQLiteWriteLock(cfg) + release, err := acquireDirectSQLiteWriteLock(cfg, state) if err != nil { return err } @@ -137,7 +143,7 @@ func runVerifyLocal(cmd *cobra.Command, args []string) error { if err := s.InitSchema(); err != nil { return fmt.Errorf("init schema: %w", err) } - if err := runStartupMigrations(s); err != nil { + if err := runStartupMigrationsContext(cmd.Context(), s, state); err != nil { return fmt.Errorf("startup migrations: %w", err) } @@ -194,7 +200,7 @@ func runVerifyLocal(cmd *cobra.Command, args []string) error { } if !cfg.OAuth.HasAnyConfig() { - return errOAuthNotConfigured() + return errOAuthNotConfigured(cfg) } // Set up context with cancellation @@ -230,7 +236,7 @@ func runVerifyLocal(cmd *cobra.Command, args []string) error { } oauthMgr, mgrErr := oauth.NewManager(clientSecretsPath, cfg.TokensDir(), logger) if mgrErr != nil { - return wrapOAuthError(fmt.Errorf("create oauth manager: %w", mgrErr)) + return wrapOAuthError(fmt.Errorf("create oauth manager: %w", mgrErr), cfg) } // Machine-readable mode must not enter an interactive OAuth // flow that writes prompts to stdout before the JSON object. diff --git a/cmd/msgvault/cmd/verify_http_test.go b/cmd/msgvault/cmd/verify_http_test.go index 19421a5cf..e3a3d64ca 100644 --- a/cmd/msgvault/cmd/verify_http_test.go +++ b/cmd/msgvault/cmd/verify_http_test.go @@ -32,12 +32,13 @@ func TestVerifyUsesConfiguredRemoteHTTPAndPreservesOutput(t *testing.T) { })) t.Cleanup(server.Close) - configureRemoteSyncTest(t, server.URL) + testCtx := configureRemoteSyncTest(t, server.URL) resetVerifyFlagsForTest(t) var stdout bytes.Buffer var stderr bytes.Buffer cmd := &cobra.Command{Use: verifyCmd.Use, Args: verifyCmd.Args, RunE: verifyCmd.RunE} + cmd.SetContext(testCtx) cmd.Flags().IntVar(&verifySampleSize, "sample", 100, "Number of messages to sample for MIME verification") cmd.Flags().BoolVar(&verifySkipDBCheck, "skip-db-check", false, "Skip SQLite integrity check") cmd.Flags().BoolVar(&verifyJSON, flagJSON, false, "Output as JSON") diff --git a/cmd/msgvault/cmd/work_tracker.go b/cmd/msgvault/cmd/work_tracker.go index 05c4e3ecc..c2e996c69 100644 --- a/cmd/msgvault/cmd/work_tracker.go +++ b/cmd/msgvault/cmd/work_tracker.go @@ -31,6 +31,10 @@ func (t labeledWorkTracker) BeginWorkContext(ctx context.Context) (func(), bool) return t.gate.BeginLabeledWorkContext(ctx, t.label) } +func (t labeledWorkTracker) BeginLabeledWorkContext(ctx context.Context, label string) (func(), bool) { + return t.gate.BeginLabeledWorkContext(ctx, label) +} + // ShouldYield reports whether an API request is queued behind this holder, // so resumable scheduled work steps aside instead of blocking it. func (t labeledWorkTracker) ShouldYield() bool { @@ -83,6 +87,10 @@ func (t combinedWorkTracker) ShouldYield() bool { } func (t combinedWorkTracker) BeginWorkContext(ctx context.Context) (func(), bool) { + return t.BeginLabeledWorkContext(ctx, "") +} + +func (t combinedWorkTracker) BeginLabeledWorkContext(ctx context.Context, label string) (func(), bool) { if ctx == nil { ctx = context.Background() } @@ -94,7 +102,13 @@ func (t combinedWorkTracker) BeginWorkContext(ctx context.Context) (func(), bool } return func() {}, false } - done, ok := tracker.BeginWorkContext(ctx) + var done func() + var ok bool + if labeled, supportsLabels := tracker.(scheduler.LabeledWorkTracker); supportsLabels && label != "" { + done, ok = labeled.BeginLabeledWorkContext(ctx, label) + } else { + done, ok = tracker.BeginWorkContext(ctx) + } if !ok { for _, v := range slices.Backward(dones) { v() diff --git a/cmd/msgvault/cmd/work_tracker_test.go b/cmd/msgvault/cmd/work_tracker_test.go index d950442e2..dcdb2ba22 100644 --- a/cmd/msgvault/cmd/work_tracker_test.go +++ b/cmd/msgvault/cmd/work_tracker_test.go @@ -4,9 +4,12 @@ import ( "context" "sync" "testing" + "testing/synctest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.kenn.io/msgvault/internal/api" + "go.kenn.io/msgvault/internal/scheduler" ) type fakeDaemonWorkTracker struct { @@ -106,3 +109,26 @@ func TestCombineWorkTrackersUnwindsWhenLaterTrackerRejects(t *testing.T) { assert.Equal(1, secondBegin, "second begin") assert.Equal(0, secondDone, "second done") } + +func TestServeSchedulerReportsActualGateHolder(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + gate := api.NewSerialOperationGate() + sched, media := newServeSchedulers(nil, testDiscardLogger(), &fakeDaemonWorkTracker{allow: true}, gate) + defer func() { <-sched.Stop().Done(); <-media.Stop().Done() }() + release := make(chan struct{}) + require.NoError(sched.AddJob(scheduler.Job{Name: "activity-projection", Schedule: "0 0 1 1 *", Run: func(context.Context) error { + <-release + return nil + }})) + require.NoError(sched.StartJob("activity-projection")) + synctest.Wait() + label, since, busy := gate.Holder() + assert.True(busy) + assert.False(since.IsZero()) + assert.Equal("activity-projection", label) + close(release) + synctest.Wait() + }) +} diff --git a/docs/README.md b/docs/README.md index 2b642815c..965629b61 100644 --- a/docs/README.md +++ b/docs/README.md @@ -66,6 +66,18 @@ and copies the static marketing site to `/`. `make docs-check` runs source validation, builds that actual layout, and checks the output and redirects. Use `make docs-serve` to inspect it at `http://127.0.0.1:8000`. +Every public page has a Markdown companion. The build copies documentation +sources to matching paths such as `/docs/setup.md` and `/docs/usage/searching.md`; +the documentation index is `/docs/index.md`. HTML pages link to their Markdown +companions with `rel="alternate"`. Keep `website/llms.txt` linked to every public +Markdown page. `make docs-check` verifies this coverage against the built site. + +Deploy with `make docs-deploy-staging` for a Vercel preview or +`make docs-deploy` for production. Both targets pull the matching Vercel +environment, build locally with access to `docs/` and `website/`, and upload +the built site. Running `vercel` directly from `docs/` uploads only that +directory, so the remote build cannot find the sibling `website/` sources. + Check a representative rendered page after changing structure, navigation, tables, diagrams, or HTML. Inspect links, headings, and mobile line wrapping. For prose changes, run the existing docs checks; do not add tests that search @@ -82,3 +94,16 @@ Follow the [diagram guide](diagrams/README.md) and the scripts in `docs/screenshots/` when changing media. The public Enron screenshot fixture has a narrow provenance and privacy-review exception in [AGENTS.md](../AGENTS.md); it does not authorize reuse in ordinary tests. + +Refresh the Web UI images with `make docs-web-screenshots`. The script imports +the pinned fixture into a temporary archive, builds and starts its own daemon, +and captures Everything and Relationships at 1920 × 1080. It keeps the normal +build toolchain and caches. Set `MSGVAULT_DOCS_SCREENSHOT_OUTPUT_DIR` to retain +captures outside the hydrated asset directory while reviewing them. + +The Web UI PNGs belong on `docs-assets`; the TUI SVGs and concept images belong +on `docs-generated-assets`. Inspect every changed image before updating its +branch. Capture `*-linux.png` with a Linux browser and `*-darwin.png` with a +macOS browser. `MSGVAULT_DOCS_SCREENSHOT_PLATFORM` chooses the filenames only; +it does not change the browser platform. Keep the fixture's provenance and +review record on `docs-fixtures` unchanged unless the fixture itself changes. diff --git a/docs/api-server.md b/docs/api-server.md index a93f4fdf5..c7bbc93fe 100644 --- a/docs/api-server.md +++ b/docs/api-server.md @@ -1,5 +1,5 @@ --- -last_edited: "2026-09-09" +last_edited: "2026-10-04" title: Web UI & API Server description: Daemon-served analytical Web UI and REST API for your msgvault archive, with optional background sync scheduling. --- @@ -24,15 +24,141 @@ browser login, secure remote deployment, search states, and keyboard controls. | Follow background work | `/api/v1/operations/runs` and `/api/v1/operations/status` | | Integrate an AI assistant | [MCP server](usage/chat.md) | -### API compatibility +### Calendar control + +`POST /api/v1/calendar/control` accepts one strict JSON object with `action` and +`account`. Event mutations also require `calendar_id`; availability requires +either `calendar_id` or `calendar_ids`. It uses owner authentication or a +delegated agent token. Unknown fields, duplicate keys, noncanonical field +casing, null fields, and multiple objects are rejected. The body limit is 1 MiB. + +| Field | Contract | +|---|---| +| `action` | `create`, `update`, `delete`, `move`, `respond`, `freebusy`, or `conflicts` | +| `account` | Configured `[[gcal]]` source name or OAuth account | +| `calendar_id` | Exact event target calendar ID, `primary`, or configured alias; availability uses it only when `calendar_ids` is empty | +| `event_id` | Required for update/delete/move/respond | +| `event` | Partial writable event fields: `summary`, `description`, `location`, `start`, `end`, `recurrence`, `attendees`, `reminders`; omitted update fields are preserved, explicit empty values clear | +| `add_attendees` | Guest email array for update, preserving existing attendees; excludes `event.attendees` replacement | +| `send_updates` | `none` by default, `all`, or `externalOnly` | +| `scope`, `original_start` | Recurring scope `single` (default), `all`, or `future` for update/delete; original occurrence start is RFC3339 or an all-day date | +| `destination` | Required calendar ID or alias for move | +| `response` | Required self RSVP: `accepted`, `declined`, or `tentative` | +| `time_min`, `time_max`, `calendar_ids`, `time_zone` | Availability range, selected calendars (at most 50), and IANA time zone; explicit `calendar_ids` are the only calendars validated and authorized | +| `dry_run`, `read_only` | Verify and plan without writes; reject all event mutations | +| `expected_plan_fingerprint` | Optional precondition from a prior dry run; return 409 `calendar_plan_changed` before writing if the current write plan, normalized OAuth account, or normalized notification mode differs | + +`start` and `end` use Google's `dateTime` (RFC3339) or `date` (all-day +`YYYY-MM-DD`), and optional `timeZone`. All-day end is exclusive. Guests use +`email` and optional `displayName`, `optional`, and `resource`; caller-supplied +RSVP state, organizer, and self flags are rejected. Reminders use `useDefault` +and `overrides` containing `method` (`popup`/`email`) and `minutes` (0–40320). + +The daemon requires an enabled source, explicit `write_calendars`, event-write +OAuth consent, and a live `owner` or `writer` accessRole. Guest changes also need +`invite_calendars`. Delegated grants match the exact `gcal` source identifier +`account-email/calendar-id`: `calendar.read` for availability, +`calendar.event.read` for provider-derived event details in delegated plans and +write receipts, `calendar.write` for changes, and additional `calendar.invite` +for guest changes. The authenticated account is never substituted for a requested +non-primary calendar. Availability with `calendar_ids` authorizes each listed +calendar and ignores `calendar_id`. + +The response contains `plan`, `writes`, resolved calendar/account IDs, and the +notification mode. Existing-event plan entries include `target.summary` and +`target.start` for owners and grants with `calendar.event.read`. +This preview metadata is covered by the plan fingerprint and is never sent as +part of a provider mutation. Partial provider writes include `outcome_code` +(`calendar_partial` or `calendar_outcome_unknown`); uncertain results also set +`outcome_unknown: true`. For availability with `calendar_ids`, `calendar_id` names +the first resolved selection and `freebusy.calendars` contains every selection. +Each completed write includes its returned event, +`message_id`, `archived`, and optional `archive_error`. A later remote failure +returns completed writes plus `error`. If a later provider write has an unknown +outcome, the HTTP 200 result also sets `outcome_unknown: true` and retains +receipts for completed writes. An archive failure retains the receipt of the +successful Google change. Reconcile these results before deciding whether +another mutation is safe; do not replay an uncertain mutation based only on its +response. Provider mutations are sent once. Invalid requests return 400; +source/consent/grant/role denials return 403; missing events return 404. Local +daemon setup failures return 500 to owners and a generic 403 to delegated callers. +Provider failures without a partial result +return 502. An unknown outcome with no completed writes returns +`calendar_outcome_unknown`. Reconcile the current calendar state before taking +further action; do not replay the uncertain operation based only on this response. +The serialized operation gate protects provider mutations and their archive +writes, including delegated calls. Body decoding, authorization, availability, +and dry runs do not hold the gate or wait for a sync to release it. + +[Calendar usage](usage/calendar.md#control-events-unreleased) owns setup, +recurrence limits, notification behavior, and reconciliation instructions. +`POST /api/v1/cli/add-calendar/plan` also accepts `write=true` to plan opt-in +`calendar.events` consent while preserving existing Google scopes. + +## API compatibility The API publishes its generated OpenAPI contract at `/openapi.json`. `msgvault openapi` prints the checked-in contract without starting a daemon or opening an archive. OpenAPI `info.version` is the **API schema version**; -it is separate from the binary release version. The current schema is **2.25.0**. +it is separate from the binary release version. The current schema is **3.1.0**. Upgrade clients and daemon together across incompatible schema versions, including remote deployments. +Schema 3.0.0 removes the unguarded +`POST /api/v1/identity/match-candidates/{id}/accept` and `/reject` routes. +Use the corresponding `/review/accept` and `/review/reject` routes with a fresh +review token. Upgrade the CLI and daemon together; clients with an incompatible +schema fail before issuing archive requests. The HTTP prefix remains `/api/v1`. +This schema also adds consented identity scoring. See +[identity match review and scoring](#identity-match-review-and-scoring). + +Schema 3.1.0 adds unreleased [calendar event control](#calendar-control), +availability queries, and opt-in `write` on Calendar consent plans. + +Schema 2.35.0 adds `scope_escalation_source_type` (`gmail` or `msmail`) to +`POST /api/v1/cli/delete-staged/plan` responses that require a permission +upgrade. With `scope_escalation_account`, it lets the local CLI select the +authorization provider before starting the daemon worker. + +Schema 2.34.0 adds optional `display_name` to `POST /api/v1/people`. +Omitted or null seeds a new profile with the first nonblank observed name in +its linked participant cluster, ordered by participant ID. An explicit string +takes precedence; empty or whitespace-only strings leave the name unset. +Re-promotion preserves the saved name, including an edited or cleared value. + +Schema 2.33.0 adds `GET /api/v1/cli/message/original`, which returns one +message's original MIME with its account and conversation, and +`GET /api/v1/cli/message/thread`, which lists visible archived conversation +members in chronological order and reports which have original MIME stored. +The thread route accepts `all=true` to capture membership in one response; +otherwise it uses `limit` and `offset` pagination. Hidden duplicates are excluded. +The original route accepts a positive `max_bytes` below 9223372036854775807 +to bound decoded MIME and its stored payload, allowing for compression overhead. +Exceeding the bound returns `413 original_message_too_large`. Omit `max_bytes` +for an unrestricted export. + +Schema 2.31.0 adds analytics query freshness metadata, accepted background +cache-build jobs, job status lookup, and restricted archive SQL for MCP. + +Schema 2.30.0 adds Kata availability and person agenda reads, creation, linking, +list placement, and unlinking. See [Kata configuration](configuration.md#integrationskata) +for setup and limits. + +Schema 2.29.0 adds `took_ms` and a required `timings` breakdown +(`query_embedding_ms`, `retrieval_ms`, and `hydration_ms`) to vector and hybrid +`/api/v1/search` responses. SQLite responses also include an `accelerator` +field identifying the retrieval path. + +Schema 2.27.0 adds deterministic meeting context export, archived action-item +listing, and duration metrics with exact direct or Explore scope. + +Schema 2.28.0 adds optional `text_enabled` and `visual_enabled` fields to +authenticated health responses. They report configured search lanes; request +handlers still check readiness when each search runs. + +Schema 2.26.0 adds optional `web_url` metadata to message result schemas. The +URL opens that message in the selected daemon's browser interface. + Schema 2.25.0 adds the CardDAV publication review flow: `GET /api/v1/carddav/publications/{person_id}/preview` returns the exact vCard the next write would send plus an approval token, and @@ -88,6 +214,48 @@ only the explicit publication preview route returns a raw vCard. See [release changes](changelog.md#upgrade-and-compatibility) for removed paths and the 1.x/2.x transition. +### Identity match review and scoring + +Review identity suggestions through `GET /api/v1/identity/match-candidates` +and `GET /api/v1/identity/match-candidates/{id}`. Responses include the +evidence, blockers, and a `review_token` for that exact snapshot. Accept or +reject with `POST /api/v1/identity/match-candidates/{id}/review/accept` or +`.../review/reject`, passing the token in the request body. If the evidence or +endpoints changed, the API returns `409 identity_match_review_stale`; fetch the +candidate again and make a fresh decision. Other conflicts retain their own +error codes, including `person_merge_required` when separate profiles need a +merge review. The former `.../{id}/accept` and `.../{id}/reject` routes are +removed; clients must use the reviewed routes and supply a token. + +Identity scoring runs only on request. `GET /api/v1/identity/scoring/status` +returns `ready`, `blocker`, credential and consent status, the exact provider +`disclosure`, its `disclosure_fingerprint`, and a `data_fields` description of +the raw identity fields sent. `POST /api/v1/identity/scoring/consent` accepts +`{"disclosure_fingerprint":""}` for the current disclosure. +`POST /api/v1/identity/scoring/revoke` accepts the same body to withdraw that +fingerprint's consent without changing configuration. + +`POST /api/v1/identity/scoring/run` accepts `{}` or `{"limit":20}`. The limit +defaults to the configured batch size and cannot exceed it. The daemon creates +review suggestions and checks local blockers before sending eligible pairs to +the provider. It journals the results and never accepts matches. + +A completed or partially completed run returns HTTP 200 with `results` and +`processed`. Each result has a candidate ID, review token, proposed action, +status, and blockers. If the batch stops early, the response also has an +`error` object with `code` and `message`; clients must inspect it even after +HTTP 200. Completed results remain available. Error codes include +`consent_required`, `scoring_scan_incomplete`, and `scoring_run_failed`. +Preflight failures use the normal non-200 error response. + +`GET /api/v1/identity/scoring/history` returns redacted judgments. It accepts +`candidate_id` (zero or omitted means all), `limit` (default 100, range 1–100), +and `before_id` for older entries. Pass a returned `next_before_id` as the next +request's `before_id`. The complete contracts are in `/openapi.json` from the +daemon or printed by `msgvault openapi`. See +[configuration](configuration.md#people-identity-scoring) and the +[people guide](usage/people.md#optional-identity-scoring) for setup and consent. + ### Archive and processing boundaries The API uses the same archive database and attachment store as other clients. @@ -110,8 +278,9 @@ returning `200` while created manifests return `201`. The HTTP listener, health endpoint, and API routing start before analytics cache maintenance. With `engine = "auto"`, aggregate requests initially use live SQL -while the cache is built or opened, then switch to DuckDB after success. A -failed automatic build or open keeps the daemon on live SQL. With +while the cache is built or opened, then switch to DuckDB after success. If no +usable cache can be opened, the daemon stays on live SQL. A failed automatic +refresh keeps the last usable publication. With `engine = "duckdb"`, analytics remain unavailable until the required cache is ready, so analytics routes return `503` during initialization; the daemon does not fall back to SQL. Cache-dependent routes also return a structured `503` @@ -162,7 +331,7 @@ is required. Three API-key authentication methods are supported: | API key header | `X-API-Key: ` | `X-API-Key: my-secret` | | Plain auth header | `Authorization: ` | `Authorization: my-secret` | -If no `api_key` is configured, authentication is not required regardless of bind address. The separate `allow_insecure` / security validation prevents starting without an API key on non-loopback addresses. +If no effective API key is configured, authentication is not required. Secure startup requires a key for non-loopback addresses. On unreleased `main`, `serve` creates and persists one when no credential source is configured. See [server credentials](configuration.md#server) for file and environment sources, persistence, and explicit insecure mode. ## Historical import jobs {#historical-import-jobs} @@ -244,6 +413,48 @@ There is no dedicated cancellation endpoint for these jobs. ## API Endpoints +### Query the analytics cache {#post-apiv1query} + +**Endpoint:** `POST /api/v1/query` + +Send one read-only SQL statement as `{"sql":"SELECT 1"}`. Set `fresh` to +`true` in the JSON body or as `?fresh=true` to request a background cache +refresh. Conflicting body and query values are rejected. +For SQLite archives, this endpoint queries published Parquet through DuckDB. +The `[analytics].engine` setting selects the engine for aggregate views; it +does not change this raw SQL endpoint. + +A `200` response contains `columns`, `rows`, and `row_count`. When the result +uses a committed Parquet publication, `cache` includes `generation` and +`published_at`, plus `stale_reason`, `pending_additions`, or `building` when +applicable. A usable stale publication remains queryable during +`min_rebuild_interval` and while a refresh runs. This includes messages deleted +since publication; see the [cache freshness policy](configuration.md#analytics). + +With `fresh=true`, the endpoint accepts a refresh with `202`, `job_id`, and +`status` instead of holding the request open. Automatic recovery of a missing +or incompatible cache also returns `202` when enabled. A fresh request checks +archive writes committed before the request. If a build is already running, +its response identifies a queued follow-up check. Poll +`GET /api/v1/cache-builds/{job_id}` for `queued`, `running`, `published`, or +`failed`. After `published`, repeat the SQL query with `fresh=false` to get rows. +`published` means the check completed successfully and the cache is usable; a +check that finds no changes preserves the existing generation. A `failed` job +includes an `error` message. + +The daemon retains the last 100 completed jobs plus running and queued jobs. +Job status is held in memory: an evicted job or a job from before a daemon +restart returns `404`. + +`POST /api/v1/query/archive` accepts the same request and response shapes. MCP +uses this endpoint to run SQL in a separate DuckDB instance restricted to the +analytics directory. Outside-file access, network access, and extension loading +are disabled. Both endpoints require daemon owner authentication; the original +`/query` endpoint retains privileged file access. An unavailable restricted +engine returns `503` without falling back to the privileged engine. + +--- + ### Curated person network {#get-apiv1peopleidnetwork} **Endpoint:** `GET /api/v1/people/{id}/network` @@ -489,6 +700,11 @@ build/open, `sql` for deliberate live SQL, `postgres` for PostgreSQL, and available during initialization; analytics routes return `503` until the required engine is ready. +Authenticated `GET /api/v1/health` also includes `api_schema_version`. Starting +with schema 2.28.0, its `vector` object can include `text_enabled` and +`visual_enabled`. These fields reflect configured lanes and remain true while a +lane initializes or fails. Public and delegated health responses omit them. + --- ### Archive statistics {#get-apiv1stats} @@ -499,6 +715,18 @@ Archive statistics. When vector search is configured on the server, the response also includes a `vector_search` sub-object describing the state of the index. +Stats answer within about 2 seconds while a sync or cache build loads the +archive: + +- If fresh counts are not ready by then, the response reuses the previous + counts and sets `"stale": true`, with `as_of` giving when they were + computed. The fresh counts replace them when they finish. +- Vector statistics get 3 seconds. After that, the response sets + `"vector_stats_unavailable": true` instead of failing. + +`GET /api/v1/cli/accounts` bounds its message counts the same way, reporting +`stale` and `as_of` at the top level. + **Response (vector search disabled):** ```json @@ -646,11 +874,13 @@ use for message-type filtering when you do not need full-text ranking. ``` The companion `GET /api/v1/messages/gmail-ids` endpoint returns matching Gmail -source message IDs for email workflows such as deletion staging. It honors a -subset of these parameters: `sender` / `sender_name`, `recipient` / +and Microsoft Graph mail (`msmail`) source message IDs in its `gmail_ids` field +for email workflows such as deletion staging. The route and field names stay +the same for both providers. It honors a subset of these parameters: +`sender` / `sender_name`, `recipient` / `recipient_name`, `domain`, `label`, `source_id`, `after` / `before`, and -`limit`. Results are always restricted to Gmail sources, exclude deleted -messages, and are ordered newest-first; the remaining `/messages/filter` +`limit`. Results are restricted to Gmail and Microsoft Graph mail sources, +exclude deleted messages, and are ordered newest-first; the remaining `/messages/filter` parameters (`message_type`, `conversation_id`, `attachments_only`, `hide_deleted`, `offset`, `sort`, `direction`) are ignored. @@ -1391,6 +1621,7 @@ query string can also carry `message_type:` / `message_type=` operators inside "state": "active" }, "took_ms": 84, + "timings": {"query_embedding_ms": 12, "retrieval_ms": 41, "hydration_ms": 31}, "results": [ { "id": 12345, @@ -1411,12 +1642,19 @@ query string can also carry `message_type:` / `message_type=` operators inside Vector and hybrid responses expose `returned` instead of `total` (ANN search does not have a meaningful total count), add a `generation` sub-object naming the index generation that answered -the query, and include `took_ms`. The top-level `results` array +the query, and include `took_ms` plus a `timings` breakdown +(`query_embedding_ms`, `retrieval_ms`, and `hydration_ms`). The top-level `results` array replaces `messages`. `pool_saturated` is true when a vector or BM25 candidate pool hit its configured cap (or pure vector search returned as many hits as requested), hinting that increasing the limit or narrowing the query may expose more relevant results. +SQLite responses include `accelerator`: `vec1_ivf_opq` for approximate retrieval, +`exact-filter` for an exhaustive search of a small filtered population, `exact` +for exhaustive retrieval, or `exact-fallback` when an accelerator error caused +an exhaustive retry. Accelerator errors are also logged as warnings. Requests +larger than the accelerator's candidate ceiling use exact retrieval. + When `explain=1`, each element of `results` carries an extra `score` object exposing the fused-score components: @@ -1507,8 +1745,6 @@ sync runs without triggering a sync. "messages_updated": 3, "errors_count": 1, "error_message": null, - "cursor_before": "745391", - "cursor_after": "745406", "skipped_count": 2, "item_errors": [ { @@ -1546,6 +1782,57 @@ with a run-level error. --- +### Meeting intelligence {#meeting-intelligence} + +These authenticated read operations require daemon API schema 2.27.0 or newer. +They read archived provider evidence without an AI call or upstream mutation. +See the [meeting guide](usage/meetings.md#export-context-and-read-follow-ups) +for the user workflow and coverage meanings. + +| Endpoint | Request and result | +|---|---| +| `POST /api/v1/meetings/context` | Exactly one of `message_ids` or an Explore `selection`; returns a context packet envelope | +| `POST /api/v1/meetings/actions` | Optional `scope` or `explore`, plus action filters; returns rows, coverage, count, and cursor | +| `POST /api/v1/meetings/metrics` | Optional `scope` or `explore`; returns totals, duration bases, monthly rows, and undated count | + +Context accepts 1–100 meetings, `format: "json"` or `"markdown"`, +`include_transcript` (default false), and `max_bytes` (default 131072, range +4096–1048576). The budget applies to the UTF-8 bytes of `content`, not the HTTP +envelope. Save `content` directly; `content_bytes`, `truncated`, and +`omitted_message_ids` describe that exact download. Mixed selections fail with +`selection_not_all_meetings`. + +A direct `scope` supports `message_ids`, `source_ids`, `participant_id` or +`participant_ids`, `person_id`, `domains`, `after`, `before`, and `deletion`. +Person and participant scopes are mutually exclusive. Different filter groups +intersect; values within one group are alternatives. An explicit +`message_ids: []` matches nothing. Omitted scope means all archived meetings. +Dates use RFC3339 timestamps: `after` is inclusive and `before` exclusive. +Deletion defaults to `any`; `active` and `deleted` refer to source deletion. +Locally deleted records never participate. + +```json +{ + "scope": {"domains": ["example.com"], "after": "2026-01-01T00:00:00Z", "before": "2026-03-01T00:00:00Z"}, + "status": "pending", + "assignee_email": "alex@example.com", + "limit": 50 +} +``` + +The actions request above also supports `query` (literal title/description +substring, at most 256 characters) and opaque `cursor`. `limit` defaults to 50 +and accepts 1–200. Status accepts `pending`, `completed`, `cancelled`, or +`unknown`; omission includes all. Pagination reads current archived snapshots, +so it is not a retained snapshot across edits. Coverage describes the selected +meetings even when action filters return no rows. + +An `explore` scope carries the complete `predicate`, `cache_revision`, +`search_provenance`, and `candidate_snapshot_id` where applicable. It is mutually +exclusive with direct `scope`. The server resolves the full matching population, +with a 10000-ID transfer ceiling. A stale authority requires reloading; an +oversized scope must be narrowed. Neither case widens the request. + ### Import a meeting {#post-apiv1importmeeting} **Endpoint:** `POST /api/v1/import/meeting` @@ -1589,8 +1876,20 @@ Timestamps must be RFC 3339 values with explicit offsets. A meeting must contain at least one non-empty `summary_markdown`, `summary_text`, `transcript`, or `transcript_segments` value; plain and segmented transcripts are mutually exclusive. Segment offsets must be finite, non-negative, and non-decreasing. +`meeting.action_items` accepts up to 1000 structured actions. Each needs a +nonblank title; optional fields preserve explicit assignee, source status, due +date, description, and source ID. An empty array means supported with no actions; +omission means unsupported; `null` is rejected. See the +[complete import example](usage/meetings.md#import-from-any-meeting-source). Unknown fields are rejected except within `meeting.metadata`. +Each organizer or attendee needs an `email`, a `phone`, or both. `phone` must be +international (a leading `+` or `00`) and is normalized to E.164. An optional +`id` (up to 200 characters) identifies the person within the import source; +msgvault links the person's email and phone through it, including across +meetings. See [how meetings connect to people](usage/meetings.md#how-meetings-connect-to-people). +Validation errors name the failing field without echoing its value. + --- ### OAuth token exchange {#post-apiv1authtokenemail} @@ -1706,6 +2005,29 @@ Scheduler state and per-account schedule details. } ``` +The daemon runs scheduled work one job at a time. While a sync waits for +another job to finish, its entry reports `"queued": true` and `"running": false`. +`"running": true` means it acquired the operation gate; `started_at` gives when +it began. The top-level `running` field reports whether the scheduler is active. +A schedule tick that fires during a run +sets `"pending": true`, and the scheduler runs the sync once more when the +current run ends. Resumable account syncs (Gmail, Teams, and Discord), Slack, and +Beeper support preemption: after holding the gate for a minute while others +are queued, they are asked to stop at their next safe point. If they are +still running five seconds later, the scheduler cancels their context. An +interrupted run goes back behind waiting jobs immediately; it does not wait +for another schedule tick. Activity projection also supports preemption: it +stops after its current batch, limits each pass to ten batches, and stops at +two minutes regardless. It saves its reconciliation progress. Attachment +packing and daily attachment maintenance stop after one minute and resume +behind waiting work. IMAP full passes do not support scheduled preemption. +Other jobs keep their own runtime budgets. Waiting API requests can still +interrupt scheduled work. `GET /api/v1/sources/status` reports the same +state for every scheduled source as `scheduler_queued`, `scheduler_pending`, +and `scheduler_started_at`. Compare queued state and the last successful sync +to detect a source that is waiting too long. Health's `operation.label` names +the job holding the gate, such as `activity-projection` or `attachment-pack`. + --- ### Preflight an analytical selection {#post-apiv1explorepreflight} @@ -1770,8 +2092,8 @@ explore contract is in the generated OpenAPI document (`/openapi.json`). ``` `count` includes all selected items after exclusions. `deletable_count` is the -Gmail subset that can be staged; the difference is the number of items staging -will skip. A chat conversation counts as one item. +Gmail and Microsoft Graph mail subset that can be staged; the difference is +the number of items staging will skip. A chat conversation counts as one item. `unavailable_actions` lists actions this selection does not support. A `stage_deletion` entry means nothing in the selection can be deleted from its @@ -2005,8 +2327,8 @@ the IDs are already an explicit, reviewed list: } ``` -IDs that do not resolve to live deletable Gmail messages with provider message -IDs are omitted, and the +IDs that do not resolve to live deletable Gmail or Microsoft Graph mail messages +with provider message IDs are omitted, and the response `message_count` reports the number of targets resolved by the daemon. A pending manifest is written and `201` returned: @@ -2070,8 +2392,8 @@ and `skipped_count`: `message_count` is the staged subset, `matched_count` the reviewed match set, and `skipped_count` the items no source supports deleting. Deletion covers -Gmail-source email, so a mixed selection stages its Gmail rows and reports the -rest as skipped rather than failing; only a selection with nothing deletable +Gmail and Microsoft Graph mail email, so a mixed selection stages those rows +and reports the rest as skipped rather than failing; only a selection with nothing deletable returns `409 selection_not_deletable`. Legacy Gmail rows with a blank `message_type` count as email. @@ -2205,11 +2527,11 @@ The same HTTP server backs configured remote CLI access and the local background The server is designed for local use: - **Loopback-only by default.** The default bind address is `127.0.0.1`, restricting access to the local machine. -- **API key required for non-loopback.** If you bind to a non-loopback address (e.g., `0.0.0.0`), the server requires `api_key` to be set and will refuse to start without it. +- **API key required for non-loopback.** Binding to a non-loopback address requires an effective key. On unreleased `main`, `serve` creates a persisted key when no credential source is configured; a selected invalid source fails startup. See [server credentials](configuration.md#server). - **Opt-in for insecure binding.** To bind to a non-loopback address without an API key (not recommended), set `allow_insecure = true`. !!! warning - Exposing the server on a network without authentication gives anyone on that network access to your entire email archive. Always set an `api_key` when binding to non-loopback addresses. + Exposing the server on a network without authentication gives anyone on that network access to your entire email archive. Keep authentication enabled when binding to non-loopback addresses. ## Configuration Reference @@ -2222,12 +2544,15 @@ All server settings go in the `[server]` section of `config.toml`. Account sched | `api_port` | `0` (auto-select) | Port the server listens on; `0` picks an open port at startup and clients discover it automatically. Set a fixed port for remote/NAS deployments. | | `bind_addr` | `127.0.0.1` | Bind address | | `api_key` | — | API key for authentication | +| `agent_access` | `false` | Enable restricted agent grants; requires an effective API key and a daemon restart after changes | | `allow_insecure` | `false` | Allow non-loopback binding without `api_key` | | `cors_origins` | `[]` | Allowed CORS origins | | `cors_credentials` | `false` | Allow credentials in CORS requests | | `cors_max_age` | `0` | CORS preflight cache duration in seconds (defaults to `86400` when `cors_origins` is set) | +| `trusted_proxies` | `[]` | IP addresses or CIDRs allowed to supply forwarded HTTPS and host headers | | `daemon_idle_timeout` | `20m` | Idle timeout for lifecycle-managed background daemons; set to `"0s"` to disable | | `daemon_auto_restart` | `newer` | Local daemon restart policy when the CLI finds a different daemon binary version: `newer`, `never`, or `always` | +| `daemon_auto_start` | `true` | Let CLI, TUI, and MCP commands start a local background daemon when none is running; set `false` when a supervisor runs `msgvault serve` | `daemon_idle_timeout` only affects daemons started by `msgvault daemon start` or auto-started by a CLI command. A foreground `msgvault serve` runs until interrupted. `MSGVAULT_DAEMON_IDLE_TIMEOUT` can override the configured timeout for lifecycle-managed background daemons. @@ -2238,8 +2563,8 @@ All server settings go in the `[server]` section of `config.toml`. Account sched | Key | Default | Description | |---|---|---| | `engine` | `auto` | Aggregate engine for Web UI, TUI, and aggregate HTTP views: `auto`, `sql`, or `duckdb` | -| `auto_build_cache` | `true` | Build stale or missing Parquet cache files during daemon startup and after scheduled syncs; `false` skips both automatic paths | -| `min_rebuild_interval` | `0s` | Minimum age of a usable cache before a scheduled sync may rebuild it; zero preserves rebuilding after each sync | +| `auto_build_cache` | `true` | Refresh a stale or missing cache at startup, after scheduled or manual syncs, and when a query finds it due; `false` skips automatic builds | +| `min_rebuild_interval` | `0s` | Minimum age of a usable cache before a sync, query, or daemon restart may queue an automatic rebuild | | `builder_memory_limit` | `2GB` | DuckDB memory limit for cache builds, such as `4GB` or `512MiB` | | `builder_threads` | min(CPUs, 2) | DuckDB threads for cache builds; zero keeps the default | | `builder_temp_limit` | `32GB` | Maximum spill-to-disk size for cache builds | @@ -2249,17 +2574,17 @@ All server settings go in the `[server]` section of `config.toml`. Account sched `engine = "sql"` forces live SQL for aggregate views. `engine = "duckdb"` requires a usable Parquet cache and keeps analytics unavailable until it is -ready; a build or open failure is fatal rather than a silent SQL fallback. +ready. Startup fails if no usable cache can be built or opened. A failed +automatic refresh keeps the last usable publication available. `auto_build_cache = false` leaves cache rebuilds to explicit -`msgvault build-cache` runs. These settings replace the TUI/MCP analytics flags -deprecated in 0.17.0; see [Configuration: analytics](/docs/configuration/#analytics). - -`min_rebuild_interval` limits only automatic post-sync rebuilds. Explicit -builds, startup maintenance, query-required builds, and unusable-cache recovery -remain immediate. On a continuously changing archive, Parquet analytics can lag -SQLite by approximately the interval plus cache build time. Cache builder memory -and temporary disk usage scale with archive size, so the interval can prevent -repeated archive-scale work on frequently synced archives. Changes under +`msgvault build-cache`, `query --fresh`, or sync `--build-cache` requests. +These settings replace the TUI/MCP analytics flags deprecated in 0.17.0; see [Configuration: analytics](/docs/configuration/#analytics). + +`min_rebuild_interval` applies to automatic refreshes requested by syncs, +queries, and daemon startup, including usable partial snapshots. Explicit +refreshes and unusable-cache recovery are not delayed by it. See +[Configuration: analytics](configuration.md#analytics) for the cache freshness +policy and deletion visibility. Changes under `[analytics]` take effect after the daemon restarts. ### `[[accounts]]` diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 6987cc7a2..560906e84 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -1,5 +1,5 @@ --- -last_edited: "2026-09-08" +last_edited: "2026-10-01" title: Architecture Overview description: How msgvault captures communications, preserves the archive, and serves people, search, and automation. --- @@ -68,6 +68,13 @@ message text for keyword search. Semantic search stores vectors in a separate SQLite index; PostgreSQL uses its own full-text search and optional pgvector. See [storage](storage.md) and [search ranking](search-ranking.md). +Opt-in [calendar control](../usage/calendar.md#control-events-unreleased) runs +through the daemon's serialized mutation gate. The daemon verifies source policy, +delegated grants, OAuth scope, and live calendar access before sending a provider +write. Successful changes use calendar sync's Store persistence path immediately, +without advancing sync cursors. Remote completion and archive completion are +reported separately so clients can reconcile a partial failure. + ## Responsibilities | Component | Owns | Main source locations | @@ -114,6 +121,16 @@ connect them; matching display names alone do not merge people. Profile facts retain evidence and resolution history. User pins and explicit merge or split actions have their own contracts. See [people and profiles](../usage/people.md). +Identity review decisions bind to the inspected evidence and endpoint snapshot. +Changed evidence or a candidate collapse requires a new review. Optional manual +identity scoring creates suggestions and journals proposals; it does not +accept them. The daemon checks local blockers before sending eligible pairs +to the fixed provider and checks consent at each provider attempt. Consent +changes wait for requests already admitted by that daemon. Provider I/O holds +no archive writer gate or database write transaction; local mutations retain +the daemon's normal coordination. See the +[review and scoring contract](../api-server.md#identity-match-review-and-scoring). + ### Each external operation has its own scope Provider sync, identity discovery, CardDAV publication, remote image downloads, diff --git a/docs/architecture/search-ranking.md b/docs/architecture/search-ranking.md index e492afb20..5e0fde654 100644 --- a/docs/architecture/search-ranking.md +++ b/docs/architecture/search-ranking.md @@ -43,11 +43,13 @@ weights with PostgreSQL `setweight` labels: |---|---| | Subject | `A` | | From address | `B` | -| Body, To, Cc | `D` | +| To, Cc | `C` | +| Body | `D` | -PostgreSQL's default weights are roughly `A=1.0`, `B=0.4`, and `D=0.1`, which -matches SQLite's 10:4:1 field priority. Unlike BM25, default `ts_rank()` does -not penalize long documents. +msgvault passes rank weights of `A=1.0`, `B=0.4`, `C=0.1`, and `D=0.1`, which +matches SQLite's 10:4:1 field priority while retaining distinct PostgreSQL +weight classes for recipients and body text. Unlike BM25, `ts_rank()` without +a normalization flag does not penalize long documents. ## Where Ordering Can Diverge diff --git a/docs/architecture/storage.md b/docs/architecture/storage.md index 6cb29e9ca..65815ff58 100644 --- a/docs/architecture/storage.md +++ b/docs/architecture/storage.md @@ -1,5 +1,5 @@ --- -last_edited: "2026-09-08" +last_edited: "2026-09-25" title: Data Storage description: Database schema, Parquet analytics cache, content-addressed attachments, and token storage. --- @@ -33,7 +33,7 @@ separately. SQLite is the default and stores the archive at `~/.msgvault/msgvaul | Column | Type | Description | |---|---|---| | `id` | INTEGER PK | Auto-increment | -| `source_type` | TEXT | Provider/import type, for example `gmail`, `imap`, `gcal`, `teams`, `discord`, `beeper`, `mbox`, `whatsapp`, `granola`, `circleback`, or `notion_meetings` | +| `source_type` | TEXT | Provider/import type, for example `gmail`, `imap`, `gcal`, `teams`, `discord`, `beeper`, `mbox`, `whatsapp`, `granola`, `plaud`, `circleback`, `notion_meetings`, or `muesli` | | `identifier` | TEXT | Provider-stable identifier such as an email address, phone number, or Discord guild ID | | `display_name` | TEXT | Account display name | | `sync_cursor` | TEXT | Sync cursor (Gmail history ID for Gmail accounts) | diff --git a/docs/changelog.md b/docs/changelog.md index 9b9ba9caa..2ea9bf273 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,5 @@ --- -last_edited: "2026-09-15" +last_edited: "2026-10-03" title: Changelog description: Release history for msgvault --- @@ -8,14 +8,271 @@ All notable changes to msgvault, grouped by release. ## Unreleased -Since **0.19.3**, msgvault has added richer people profiles, document and image -search, background operations, and more ways to import and maintain an archive. -This section covers work on `main` through **September 15, 2026**. These changes -are not included in 0.19.3; this is not a new release announcement. +- Rerunning `import-whatsapp` on an Apple `ChatStorage.sqlite` writes only new + and changed messages instead of rewriting the whole archive, and picks up + edits and senders that `LID.sqlite` resolves later. + +- [Calendar event control](usage/calendar.md#control-events-unreleased) adds + create, update, delete, move, self RSVP, and availability commands, plus HTTP + and MCP interfaces. Write consent and exact source permissions are opt-in; + guest notifications default to `none`. The daemon verifies calendar access + and archives successful changes immediately. +- Adding Calendar to a Gmail token recognizes Google's short and expanded + `email`/`profile` scope names, avoiding false missing-scope errors on re-consent. + +## 0.21.0 +2026-10-02 + +Find your way around the new Web UI, prepare mail and chat drafts, and bring +Microsoft Graph mail and Muesli meetings into your archive. This release also +adds reviewed identity tools for agents and reduces repeated sync and search work. + +[GitHub release](https://github.com/kenn-io/msgvault/releases/tag/v0.21.0) · +[All changes since 0.20.0](https://github.com/kenn-io/msgvault/compare/v0.20.0...v0.21.0) + +### Before upgrading + +- **Back up your archive.** The first startup updates the database. Affected + SQLite archives also need a one-time messages-table scan to fill missing + activity timestamps. See [backup and restore](usage/backup.md). +- **Upgrade clients and daemon together.** API schema **3.0.0** replaces + token-free identity decisions with reviewed decisions. Integrations must + fetch a fresh review token and use the `/review/accept` or `/review/reject` + route. HTTP MCP writes also need `--http-allow-writes`; confirmation-dependent + writes need MCP protocol **2026-07-28** or newer. See + [API compatibility](api-server.md#api-compatibility). +- **Check CardDAV API callers.** Omitting `connection` from + `POST /api/v1/carddav/sync` now syncs all enabled connections. If any are + enabled, HTTP 200 can include failures; inspect `status` and each result. + Send `{"connection":"default"}` for the former single-account behavior, + including manual sync of a disabled default account. See + [connection selection](usage/people-carddav.md#select-connections-through-the-api). +- **Renew audio upload consent.** Docbank audio processing now covers captured + sources beyond Beeper. Set `all_sources_upload_consent = true` after reviewing + that scope; the former `upload_consent` no longer permits uploads. Document + extraction with `include_inline = true` also needs fresh consent. See + [audio processing](usage/beeper.md#send-audio-to-docbank) and + [document indexing](usage/document-indexing.md). +- **Allow one full analytics rebuild.** Existing caches need a full rebuild + before later refreshes can reuse retained data. Existing message embeddings + remain compatible. Build the optional SQLite search accelerator with + `msgvault embeddings optimize`; it makes no provider calls. See + [analytics settings](configuration.md#analytics) and + [vector search](usage/vector-search.md). + +### New features + +- Navigate the Web UI through a grouped sidebar and search from any page. Press `/` to + focus global search. Everything and Files also let you save the current view directly + from their headers. +- Sync Microsoft 365 and Outlook.com mail through Microsoft Graph with + `add-o365 --graph` when IMAP is unavailable. Staged deletion supports these accounts, + including moving messages to Deleted Items and permanent deletion. Deletion requires + `Mail.ReadWrite` consent and that permission in the app registration. +- Sign in to Microsoft 365 and Teams without a local browser using + `msgvault add-o365 you@example.com --headless` or + `msgvault add-teams you@example.com --headless`. msgvault prints a Microsoft URL + and a code to use on another device. +- Create and manage plain-text Gmail reply drafts, compose fresh IMAP drafts, and + reply-all to archived mail. Enable drafts for each source. Delegated agents can read, + edit, and delete managed drafts within their granted account access. +- Forward archived email with its stored attachments using `msgvault draft-forward`. + msgvault creates an IMAP draft with your note and the original message quoted as text. + Missing attachments or an exceeded server upload limit prevent draft creation. +- Save draft replies to archived Slack, Teams, and Discord conversations with + `draft-compose --conversation`. Review, edit, or delete them in msgvault. msgvault + never sends these replies. +- Place draft replies directly in Beeper Desktop's message box through the draft + commands after enabling `[[beeper.drafts]]`. msgvault rejects writes when it finds + unrelated text already there. Text typed between its check and write can still be + lost. +- Export a conversation's visible emails as numbered `.eml` files with + `msgvault export-eml --thread -o DIR`. MCP clients can list threads and download + stored emails and attachments without shell access. PST email exports contain + reconstructed MIME. +- Import iMazing Messages CSV exports with `import-imazing-csv`, including available + attachments. Supply `--me` and use `--timezone` when the export's timezone differs + from your local zone. +- Archive completed Muesli meetings, notes, transcripts, and later edits with + `add-muesli` and `sync-muesli`. msgvault must run on the Mac containing Muesli. Grant + the daemon Full Disk Access to read Contacts; meetings still sync without Contacts + access. +- Link meeting attendees to existing people through email or phone evidence. Names alone + never establish a match, and conflicting ownership goes to review. +- Export selected meetings as JSON or Markdown context with archive citations and + optional transcripts. Query recorded action items and meeting activity through the Web + UI, CLI, HTTP, or read-only MCP tools. Existing archives gain this data without a + provider resync. +- Show live Kata todos on a person's page through the opt-in `[integrations.kata]` + connection. Create and unlink tasks in the Web UI, or link existing tasks through the + CLI and API. Edit and complete tasks in Kata. This requires Kata API schema 0.21.0 or + later. +- Connect multiple named CardDAV accounts, each with its own credentials, schedule, and + sync history. Select one with `--connection` or in Settings. `sync-carddav` without a + connection syncs all enabled connections and exits nonzero on partial failure. Contact + publication still uses one target. +- Connect CardDAV servers that require Digest authentication or an approved private + HTTPS destination. Configure `trusted_origin` and `trusted_addresses` locally before + adding a private destination. +- Accelerate SQLite semantic, hybrid, and similar-message searches with an approximate + index and exact reranking. Run `msgvault embeddings optimize` to build or resume it + from stored embeddings without provider calls. Set + `[vector.search].sqlite_accelerator = "exact"` for exhaustive search. +- Review identity suggestions, which propose linking archived identities to people, + through the CLI or MCP. MCP also supports person merges and CardDAV writes. Enable + each capability with its corresponding `--allow-identity-decisions`, + `--allow-person-merges`, or `--allow-carddav-writes` flag. Each confirmation requires + user approval. +- Score identity suggestions with the optional Jev service. Scoring is disabled by + default and requires configuration and consent. `msgvault person scoring status` lists + the names, email addresses, phone numbers, identifiers, scopes, and matching evidence + sent to `api.typesafe.ai`. Scores remain advisory and never accept matches or merge + people. +- Configure people inference with OpenAI, OpenRouter, or Venice presets and explicit + model, privacy, and consent choices. Setup no longer enables inference just because an + OpenAI key exists. Codex sign-in and inference remain unavailable in this release. +- Process stored WAV and MP3 recordings from captured sources through Docbank, + preserving supplied transcripts or using a configured speech-recognition profile. + Enable `all_sources_upload_consent = true`. Existing Beeper audio users must consent + again; the former `upload_consent` setting no longer permits uploads. +- Select Slack direct conversations independently with `[slack].dms` and `group_dms`, + or override either setting for one sync using `--dms` and `--group-dms`. Skipped + conversations retain their incremental progress. +- Query archive analytics through the advanced MCP `query_sql` tool on supported SQLite + daemons. +- Compare search rankings with Jev reranking using `msgvault eval --rerank-jev`. This + sends queries and bounded message text to TypeSafe using `TYPESAFE_API_KEY`. Results + include quality, latency, usage, and cost measurements, with incomplete usage marked. + +Start with the [Web UI](web-ui.md), [source guide](guides/sources.md), +[draft commands](cli-reference.md#draft-reply), or [MCP tools](usage/chat.md). + +### Improvements + +- Keep querying published analytics while a cache refresh runs in the background. + `--fresh` requests include writes committed before the request. CLI queries wait when + a build is required; HTTP and MCP callers receive a build job to poll. +- Refresh analytics with less repeated work. New messages, label changes, and + identity-link changes reuse retained cache data where possible, including when sync + overlaps a build. Existing caches require one full rebuild after upgrade. +- Keep scheduled syncs moving under load. Background maintenance yields to queued work + and resumes saved progress across restarts. Scheduled activity and attachment + maintenance use one-minute budgets; scheduled Beeper imports use three-minute budgets. + Status distinguishes queued jobs from running jobs. +- Reduce query work for full-text, semantic, hybrid, and address-filter searches. Large + People and Domain groupings also avoid the memory exhaustion reproduced with a 512 MB + limit. +- Reduce CPU time spent preparing messages for embedding builds. Existing vectors remain + compatible, and no rebuild is required. +- Show readable chat snippets in CLI search tables and fit columns to the terminal + width. Redirected output preserves full sender and subject or snippet text. Slow + searches show elapsed time without implying that search waits for concurrent daemon + work. +- Check full-text index coverage without reading stored message content. Searches show + an incomplete-results warning when the index needs rebuilding or is rebuilding. +- Read person records through separate scroll areas and organized sections. Populated + attributes appear near the top, empty fields stay hidden until requested, and linked + identities explain their service and match origin. +- Show readable statuses and errors across Sources, Operations, Deletions, and Settings. + Settings remembers its category and applies saved theme or density changes to the open + tab unless a Display override is active. +- Finish `msgvault setup` without Google credentials by pressing Enter at the credential + prompt. The wizard then shows IMAP, Microsoft 365, and file-import examples. +- Control implicit daemon startup with `[server].daemon_auto_start = false` when a + supervisor manages the daemon. CLI commands use a running daemon or return an + actionable error. +- Discover MCP tools without waiting for archive statistics. Upgrade clients and daemon + together to API schema 3.0.0, which replaces token-free identity decisions with + reviewed decisions. HTTP MCP writes also require `--http-allow-writes`, and + confirmation-dependent writes require MCP protocol 2026-07-28 or newer. +- Diagnose document extraction failures through specific build errors and status + details. Inline attachments remain excluded by default; enabling + `attachments.documents.scope.include_inline` requires fresh consent. + +- Show better size estimates for Beeper, Slack, and Teams messages. New imports + count body bytes and supported attachment bytes; older rows keep their recorded + size until reimported or backfilled. + +### Bug fixes + +- Preserve PST message identifiers and repair reply chains after importing all folders. + To repair an earlier import, rerun the same PST with the same account identifier and + `--no-resume`. Existing messages remain counted as skipped. +- Complete IMAP syncs for supported legacy and damaged Message-IDs without replacing + archived messages or repeatedly enumerating folders. DavMail messages with empty + header responses receive a full-header retry to recover identities and labels. +- Skip unchanged partial Apple Mail messages during repeated `import-emlx` runs. Only + newly restored attachments trigger re-ingestion, and the restored count reports + attachments added during that run. +- Keep invalid Apple Messages dates from blocking analytics builds. Rerun + `msgvault import-imessage` without date filters or a message limit to correct an + earlier import, retaining any original `--db-path` and `--me` options. Undated + messages remain archived. +- Build analytics caches despite invalid UTF-8, late-occurring quoted fields, or + pre-1970 messages. Damaged display text uses replacement characters, while damaged + identities export as unknown. Repair an older damaged cache with + `msgvault build-cache --full-rebuild`. +- Return complete API JSON when archived subjects, snippets, or names contain invalid + UTF-8. +- Complete activity updates for affected upgraded SQLite archives with missing message + timestamps. The first startup after upgrade scans the messages table once to fill + those timestamps. +- Recover Gmail syncs when quota pauses exceed an individual request timeout. Quota + retries honor `Retry-After`, cancellation, and the caller's deadline. +- Stop repeatedly downloading Beeper media that the source reports as permanently + unavailable. Sync and backfill summaries report those files, while temporary failures + remain eligible for retry. +- Complete repeated person enrichment lookups without failing on reused citations or + duplicate values. Scheduled lookups stop repeating terminal failures, and disabling a + profile retains queued work for re-enablement. +- Preserve `--no-default-identity` choices during scheduled sync and re-authorization. +- Give newly promoted people an observed display name when one is available. Contacts + without names also receive generated full names so vCard publication succeeds. +- Filter Directory by Email, Chat, Meeting, or Other through the Primary channel menu. + Directory media and file filters also apply. +- Keep cached Files listings usable when a provider resync retains the same attachments. + Old attachment links no longer trigger endless page requests. +- Preserve files created at a SQLite backup target while the backup is running. msgvault + returns `backup target already exists` instead of overwriting the competing file. +- Avoid a panic when updating msgvault and restarting a running daemon. +- Load the full MCP tool catalog in strict clients through the corrected Saved View + schema. +- Complete initial and full Google Contacts syncs. Failed CardDAV requests report + the upstream status; response excerpts are available at DEBUG level. + +If an older version removed queued enrichment work while a profile was disabled, +re-enabling it does not restore that work. Request a +[manual lookup](cli-reference.md#person-enrichment) with a new idempotency key, +or wait for an identity change or claim expiry to queue another lookup. + +### Acknowledgements + +Thanks to everyone who contributed to this release: + +- [@aaronwolen](https://github.com/aaronwolen) for fixing quoted fields in analytics caches and initial Google Contacts sync. +- [@eliemada](https://github.com/eliemada) for preserving legacy IMAP Message-IDs. +- [@exactmike](https://github.com/exactmike) for Microsoft mail sync, deletion, and headless sign-in. +- [@franklintra](https://github.com/franklintra) for recovering empty IMAP header responses. +- [@fucx](https://github.com/fucx) for fixing daemon restart during updates. +- [@mariusvniekerk](https://github.com/mariusvniekerk) for Web pane dividers, vector requests, retries, and backup protection. +- [@metcalfc](https://github.com/metcalfc) for recovering Gmail syncs after quota throttling. +- [@rodboev](https://github.com/rodboev) for mail and chat drafts, audio processing, and archive maintenance. +- [@salmonumbrella](https://github.com/salmonumbrella) for people and meeting workflows, reviewed agent tools, analytics, and scheduling. +- [@shntnu](https://github.com/shntnu) for Slack DM selection and narrower people-fact migration checks. +- [@spf13](https://github.com/spf13) for generating full names for nameless contacts. +- [@wesm](https://github.com/wesm) for Web navigation, search, sync, activity, and documentation. -macOS analytics cache builds use the existing CSV export without the unusable -SQLite extension download and warning. +--- + +## 0.20.0 +2026-09-22 + +Manage people and relationships, search inside attachments, and bring more of +your communications into the archive. Review the upgrade notes before updating +an existing installation. +[GitHub release](https://github.com/kenn-io/msgvault/releases/tag/v0.20.0) · +[All changes since 0.19.3](https://github.com/kenn-io/msgvault/compare/v0.19.3...v0.20.0) ### Upgrade and compatibility @@ -25,7 +282,7 @@ SQLite extension download and warning. users need their own database backup; see [backend limits](architecture/postgresql.md). - **Upgrade clients and daemon together.** The API crossed the 1.x/2.x - compatibility boundary; the current schema is **2.25.0**. Analytical + compatibility boundary; the current schema is **2.26.0**. Analytical `/api/v1/people/*` routes moved to `/api/v1/participants/*`. Durable profiles moved from `/api/v1/persons/*` to `/api/v1/people/*`. The old paths were removed. Clients reject incompatible daemons; authenticated `/api/v1/health` @@ -67,203 +324,109 @@ SQLite extension download and warning. Keyword search and basic browsing do not require a model provider. Optional features can be enabled after the archive is usable. -### People and relationships - -- Maintain structured names, contact details, addresses, dates, categories, - private notes, organizations, employment, and relationships with start and end - dates. Merge or split duplicate profiles, inspect the history, and reverse - supported merges. -- Browse saved people by last contact in Web Directory, `person directory`, and - MCP's `list_directory_people`. Filter by dates and contact details, choose an - order, and page through results. `person list` retains its existing use. -- Find a person by remembered profile facts with semantic search. Find their - files across attachment metadata, document text, and the visual index. -- Keep the evidence behind profile facts, pin corrections, and repair derived - values when their inputs change. Curated names now appear in analytics, search - results, and exported authors. `export-messages --person-id` selects messages - through the person's bound participants. -- Maintain tracked profiles with consented provider sweeps. Profiles support - OpenAI Chat, OpenAI Responses, Anthropic Messages, and Gemini. Codex - app-server configuration exists, but no executable is approved by its release - gate. The CLI can add, update, check, select, and reverify profiles. -- Configure separate Exa or SixtyFour enrichment with exact consent, request - limits, and suppression controls. -- Generate a **“Last time we talked”** brief for an enrolled person and inspect - its cited sources and earlier versions. Briefs use supported chat and text - messages; email, meetings, documents, and your replies are excluded. CLI, TUI, - Web Directory, and API manage briefs; MCP reads saved versions. -- Review Beeper identity candidates across sources. Strong provider or Beeper - identifiers can link automatically; same-service usernames require review, and - conflicting bindings remain separate. -- Import CardDAV contacts and publish selected profiles with conflict review and - lossless vCard handling. Connect **Google Contacts** through OAuth from Web - Settings or the CLI. Contact sync now works with iCloud servers that reject - full URLs in requests for individual cards. -- Record `how_we_met` and up to 280 characters in seeded text attributes. - Multiline Notes retain their separate behavior. - -See [people](usage/people.md), [profile automation](usage/people-automation.md), -[enrichment](usage/people-enrichment.md), [briefs](usage/people-briefs.md), -[Beeper identities](usage/beeper.md#review-identities-across-sources), and -[CardDAV](usage/people-carddav.md). - -### Search and attachments - -- Search Beeper chats and meeting transcripts with embeddings that include - nearby messages. Build embeddings for selected accounts or collections and - configure model-specific task prefixes. Changing scope requires a rebuild; - hybrid search can still find out-of-scope messages through keyword matching. -- Extract attachment text with the shared Docbank engine and search it by - keyword, meaning, or both. CSV files can opt into local PDF conversion while - retaining their original file identity and extraction history. - PowerPoint PPTX attachments can be indexed from their original bytes after a - fresh capability probe proves Docbank's local slide bound. -- Search image and video content through a separately configured visual - provider, after its capabilities have been checked. -- Find mailing-list traffic with `list:` or `list-id:` and browse Lists - grouping. `repair-list-ids` previews an offline backfill; `--apply` writes it. - Use `conversation_id:` to select one local conversation. -- Choose whether search includes, excludes, or shows only messages deleted from - their source. Their local content remains available until purged. -- Preserve remote email images for offline reading after explicit opt-in, with a - separate backfill for existing mail. Stored MIME and HTML remain unchanged. - Fetching these images can activate tracking. - -See [searching](usage/searching.md), [vector search](usage/vector-search.md), -[document indexing](usage/document-indexing.md), and -[remote images](usage/remote-images.md). - -### Browser, terminal, and integrations - -- Use Web Directory to edit profiles, relationships, and employment, review - identity matches and facts, merge or split profiles, publish CardDAV contacts, - and inspect curated networks and person attachment galleries. -- Monitor sync, extraction, embeddings, enrichment, and CardDAV in Web - Operations. Filter run history, inspect failures, and run the actions the - daemon offers. CardDAV history survives daemon restarts. -- Edit Web Settings in sections with visible numeric limits, switches for - settings that can be off, and schedule presets with time zones and validation. - Track unsaved changes, discard them, and see when a restart is needed. - Credentials can be replaced or removed without revealing the complete stored - value. Host-managed settings stay read-only in the browser. -- Read messages beside the results or below them, resize the preview, and keep - the layout between visits. In dark mode, HTML email uses app colors; switch to - the sender's original colors for the open message when needed. -- Browse People and attachments in the TUI, use semantic search and Emacs-style - navigation, and scope Email by named collections. Multi-source collections - offer Fast search only; empty collections match nothing. Email, Texts, and - Meetings keep independent source selectors. -- Download, open, or export TUI attachments as a ZIP. Press `s` in email detail - to save the archived original as `.eml` in the client's current directory. - Existing files are preserved, and the original email must be archived. -- Share Saved Views between the browser, API, and MCP assistants. Read tools can - list, inspect, and run views. Creation, editing, and deletion are write tools; - HTTP requires `--http-allow-writes`. Stored definitions are validated against - the version-1 vocabulary, and revision checks prevent stale edits. -- Read person profiles, Notes, relationships, and files through MCP. General - profile reads exclude sensitive attributes and private Notes; Notes have a - separate explicit read tool. Profile writes require additional opt-in. -- Discover running HTTP MCP endpoints with `mcp status --json`, including actual - ports, backend URLs, and private token-file paths. Status does not start a - daemon or print token contents. MCP also supports protocol `2026-07-28`, - publishes object-root output schemas, and accepts parameterless inbound - messages. -- Use `setup providers` for consented provider defaults and `setup status` to - understand readiness. Sensitive profile inference requires a separate - `--allow-sensitive` opt-in. -- Browse the new website's product overview, archive lifecycle guide, and - operating documentation under `/docs/`. - -See [Web UI](web-ui.md), [TUI](usage/tui.md), -[recommended configuration](usage/recommended-configuration.md), and -[MCP](usage/chat.md). - -### Sync, imports, and maintenance - -- Sync Notion AI Meeting Notes with available transcripts, verified attendees, - changed-note refresh, and bounded retries for late transcripts. -- Import Slackdump directories or ZIPs, EML files and `.mailbox` trees, and - Maildir or Maildir++ archives. Maildir imports retain folder and flag labels - and recognize previously archived messages after filename changes. -- Import Google Groups Takeout MBOX or ZIP exports. Apple Mail imports preserve - RFC Message-ID and link unambiguous replies within a source. Re-import and - rebuild the cache to fill missing IDs in existing Apple Mail archives; see - [message identifier recovery](usage/importing.md#message-identifiers-and-replies). -- Import Apple WhatsApp ChatStorage text, including URL messages, with available - participant and push names. Contact-number matching requires country codes; - missing group-participant tables no longer block otherwise usable exports. -- Preserve Google Voice voicemail audio supplied in Takeout exports. Missing or - unreadable recordings remain visible as failed attachment records. Discord - voice messages retain their duration and waveform metadata. -- Run bounded historical Gmail or IMAP imports as background jobs through the - API, with durable status and checkpoint-based resumption. -- Keep IMAP sync incremental with QRESYNC where available and avoid refetching - unchanged folders without it. Retry connection failures within bounds, - preserve valid mailbox state after incomplete responses, and treat messages - removed during a fetch as handled. `repair-labels` rebuilds labels from stored - folder membership without contacting the provider. -- Create an IMAP reply draft with `draft-reply` after an operator grants access - to a specific source and Drafts folder. The server must support UIDPLUS. - Msgvault stores an archived copy; it never sends the email. -- Add restricted agent grants. Owner-only `agent-token issue/list/revoke` - commands manage in-memory tokens with `draft.create` permission for named - sources. Agents use `--agent-url` and `--agent-token-file` to run only - `draft-reply`; grants expire when revoked or when the daemon restarts. - Enable `[server] agent_access = true` with a non-empty `api_key`; see - [agent-token](cli-reference.md#agent-token) and - [configuration](configuration.md#server). -- Refresh the archived body, recipients, and attachments when a trusted outgoing - IMAP copy is edited or moves from Drafts to Sent. Ordinary received-mail and - All Mail copies cannot replace that content. Historical rows that already lost - their old location are not repaired automatically. See [IMAP](usage/imap.md). -- Authorize Gmail with read-only access using `add-account --readonly`. Existing - write grants require the documented revoke-and-reauthorize steps. Google - Calendar registration now also works directly with Workspace service accounts; - delegation and disabled-API errors stop without lengthy retries. -- Archive Teams self-chat and avoid replaying the last message at each - incremental boundary. Preserve Beeper transcript metadata, resume backfilled - history, store plain text, distinguish link previews, and repair older - classifications from archived payloads. Calendar snippets retain valid UTF-8. -- Select exact sources with `--source-id` where supported. Deletion manifests - preserve source type and identifier. Stage by query or explicit message IDs, - preview counts, and skip ineligible query matches. Eligible messages must - belong to one exact source; TUI and MCP reject cross-source staging. -- Keep messages and attachments after permanent source deletion. Use the - separate SQLite `gc` command to purge source-deleted rows and unreferenced - blobs. Deduplication requires the reviewed plan to remain valid and prefers - attachment-complete survivors only under its documented equivalence rules. -- Repair source identity, sender attribution, Gmail snapshots, and derived - metadata. Gmail sync reconciles expired history, retries failed fetches from - the previous completed run, verifies cached account identities, and bounds - request and OAuth refresh waits. Scheduler errors show copyable - reauthorization commands. -- Recover more malformed MIME messages and report importer read errors. Bound - large analytical listings, rebuild caches less often, maintain SQLite planner - statistics, resolve cached recipient addresses, and prune orphan embeddings. - Tune DuckDB query memory, threads, and disk spill separately from cache-build - limits. - -See [sources](guides/sources.md), [imports](usage/importing.md), -[text messages](usage/text-messages.md), [Calendar](usage/calendar.md), -[OAuth](guides/oauth-setup.md), [deletion](usage/deletion.md), -[deduplication](usage/deduplication.md), and -[analytics configuration](configuration.md#analytics). - -### Building and maintaining msgvault - -- Build with Go 1.27 and the documented Bun, Node.js, and native SQLite - prerequisites. Nix flake packaging has been removed. -- Use Docker Bake to export Linux AMD64 and ARM64 images as OCI archives. Image - builds check database initialization, DuckDB queries, and the embedded Web UI. - Repository release-publishing workflows and tag/changelog scripts have been - removed; local builds and installers remain available. -- Generate the browser API client with Orval and pinned OpenAPI tools. Local - SQLite test scheduling scales to available CPU and memory; PostgreSQL - configurations remain separately covered. Synthetic media examples and - relevance judgments prepare future retrieval evaluation; they do not add audio - transcription or a runnable evaluation workflow. - -See [Development](development.md) for build and check commands. +### New features + +- Manage people in the new Web Directory and TUI People browser. Profiles bring together contact details, custom fields, private Notes, messages, meetings, files, and change history. +- Record organizations, current and past employment, and dated relationships between people. Review uncertain identity matches, merge duplicate profiles, and split mistaken merges while retaining history. +- Browse people by last contact through `msgvault person directory` and MCP. Relationship calendars show contact activity over time. +- Keep tracked profiles current with optional model-assisted updates from archived conversations. Provider setup requires explicit consent and budgets; saved evidence explains profile changes, and pinned facts preserve manual choices. +- Enrich profiles through Exa or SixtyFour using explicitly permitted identity details. External enrichment stays off by default and does not send message bodies or private attributes. +- Catch up with a "Last time we talked" brief that cites archived messages. Briefs require separate enrollment for each person and currently summarize their supported chat and text messages, excluding email, meetings, and your replies. +- Find people by meaning with `msgvault person search`. This requires separate consent for profile embeddings and search queries. +- Sync contacts bidirectionally with a CardDAV address book, including Google Contacts OAuth setup. Publication is explicit per person, conflicts remain reviewable, and profile edits preserve unrelated vCard properties. +- Find a person's files across linked accounts and conversations through the Web Files view, `msgvault person files`, or MCP. Search attachment details, extracted text, or visual content, with links back to the original messages. +- Search text extracted from PDF, CSV, and PPTX attachments. Extraction requires a successful provider capability check, explicit consent, and a manual build. CSV conversion is opt-in, and PPTX requires a verified slide limit. +- Search indexed attachment text by meaning or combine keyword and semantic results. Document embeddings and query uploads require separate consent; default document searches remain local keyword searches. +- Find images and supported MP4 attachments with text or image queries. Visual search requires provider capability checks and consent before archive uploads. +- Preserve conversation context in chat and meeting search with optional Voyage contextual embeddings. The existing embedding method remains the default. +- Filter messages with `list:`, `list-id:`, and `conversation_id:`. Mailing-list views are available in the Web UI and TUI; run `msgvault repair-list-ids --apply` and rebuild the cache to backfill existing mail. +- Compare keyword, semantic, and hybrid search quality with `msgvault eval`, using your own queries and relevance ratings. Reports include ranking quality and query timings in table or JSON form. +- Import EML mailbox trees and Maildir archives directly with `import-eml` and `import-maildir`. Both preserve folder labels and support repeat imports without duplicating archived messages. +- Import Google Groups Takeout MBOX files and ZIPs with `import-mbox --source-type google-groups`. Imports retain group labels, threads, and attachments without treating the group address as your identity. +- Import Slackdump Standard and Mattermost export directories or ZIPs without a live Slack token. +- Import Apple WhatsApp `ChatStorage.sqlite` text conversations, including supported participant names. Media, reactions, and reply relationships remain outside this importer. +- Sync Notion AI Meeting Notes into meeting views and search. Sync retains summaries, notes, and transcripts, retries pending transcripts, and reports Notion's partial 50-meeting discovery window. +- Keep Google Voice Takeout voicemail recordings with their archived messages. Stored audio remains available after the Takeout files move. +- Create, retrieve, edit, and delete plain-text IMAP reply drafts through the CLI. Enable drafts for each source and configure its mailbox first; sending remains in your mail client. `draft-recover` resumes identified interrupted edits or deletions from saved receipts without appending another copy. +- Give an agent account-scoped permission to create reply drafts with `msgvault agent-token`. Delegation requires explicit server configuration and an owner API key. Tokens remain valid until revoked or the daemon restarts. +- Archive remote email images for offline reading with `[sync] archive_remote_images = true`. The feature is off by default because downloads can activate tracking pixels. Backfill existing email separately with `msgvault archive-remote-images --allow-tracking`. +- Authorize Gmail with read-only access using `msgvault add-account --readonly`. Existing write grants require revocation and reauthorization; adding the flag does not narrow an existing grant. +- Start bounded Gmail and IMAP historical imports through the API and retrieve persistent progress and completion status. +- Stage Gmail deletions from search criteria or explicit IDs with `msgvault stage-delete`, including `--dry-run`. Mixed selections stage eligible Gmail messages and report skipped items; each batch remains tied to one source. +- Inspect background work in the new Operations workspace. Filter run history and open outcomes, recovery guidance, and available actions. +- Manage daemon, source, search, and attachment settings in the Web UI or press `,` in the TUI. Web settings include schedule presets, time zones, numeric limits, and masked API-key hints. +- Configure optional search and people providers with `msgvault setup providers`. Use `msgvault setup status` to see missing credentials, checks, consent, and next steps. +- Let MCP clients discover, run, and manage Saved Views, the saved queries created in the Web UI. New people tools expose profiles, recent contacts, saved briefs, and person-specific file search. +- Discover running HTTP MCP listeners with `msgvault mcp status --json`, including their ports and token-file paths without printing tokens. +- Reclaim SQLite space from messages deleted at their source with `msgvault gc`, which requires confirmation and creates a backup by default. Remove embeddings left by hard-deleted messages with `msgvault embeddings prune` without paying for a full embedding rebuild. + +### Improvements + +- Sync large IMAP accounts incrementally with or without QRESYNC. Unchanged folders avoid full scans, and full enumerations update only changed records. In one 96,000-message Microsoft 365 benchmark, syncing after one message moved drops from 1 hour 46 minutes to 24 seconds. +- Scope embedding builds to selected accounts or named collections. Configure `document_prefix` and `query_prefix` for models that require task instructions; changing either requires `msgvault embeddings build --full-rebuild --yes`. +- Control chat attachment downloads by account, conversation size, and file size. Beeper, Slack, Discord, and Teams now skip downloads in conversations with more than 20 participants by default; set `media_max_participants = 0` to remove that cap. +- Retain larger individual attachments by default in Beeper, Slack, and Teams, with the limit rising to 250 MiB. Discord stays at 50 MiB. Existing files remain stored until you explicitly run `purge-excluded-media`. +- Read HTML email with dark colors and a resizable right-side preview in Everything. A per-message "Use original colors" control restores the sender's presentation. +- Share shorter Web UI links while preserving filters and selected items. Back and Forward retain the full session state, and shared links retain their search mode. +- Open messages in the browser from CLI JSON and MCP results through the new `web_url` field. +- Browse, download, and open individual attachments from the TUI with `e`. Press `s` in email detail to save an `.eml` file on the client machine without overwriting existing files. +- Reuse TUI searches with Up and Down, navigate with `Ctrl+n` and `Ctrl+p`, and select named collections with `A`. Semantic search is available when vector search is enabled. +- Stage the current TUI selection with `d`, or resolve every filter or search match with `D` before confirmation. Set `[data].export_dir` to choose the destination for attachment exports. +- Use curated person names in search, relationship analytics, completion, and message exports. Existing analytics caches require one full rebuild; later name changes refresh derived data without rebuilding message shards. +- Tune daemon query resources with `[analytics] query_memory_limit`, `query_threads`, and `query_temp_limit`. Cache builds now allow up to 32 GB of temporary disk use by default, and `min_rebuild_interval` limits automatic rebuild frequency. +- Select an exact source with `--source-id` for sync, account maintenance, and staged deletion when account names are ambiguous. +- Search retained source-deleted mail with `--deletion-scope deleted` or `any` in keyword searches. Message output includes deletion timestamps, and archived raw email remains exportable. +- Enable remote deletion persistently in the invoking CLI's configuration with `[deletion] remote_enabled = true`. The daemon's own configuration does not grant consent to a remote caller. +- Connect MCP clients using protocol 2026-07-28 with sessionless HTTP discovery. HTTP remains read-only unless explicitly enabled. Clients consuming `aggregate` or `search_by_domains` must read `structuredContent.data`. +- Update API integrations to use `/api/v1/participants` for observed-contact analytics and `/api/v1/people` for saved profiles. The old people-analytics routes and `/api/v1/persons` routes are removed. +- Install through the supported scripts, Homebrew, or release binaries instead of Nix, which is no longer supported. Source builds require Go 1.27.0. + +### Bug fixes + +- Preserve the local message, raw email, metadata, and attachments when permanently deleting mail from Gmail. +- Reconcile Gmail source deletions after history expires without removing archived content. Incremental sync also retries failed fetches from the latest completed run recorded as incremental. +- Reject cached Gmail credentials that belong to a different mailbox during `add-account`. Gmail and Google OAuth requests now have bounded waits, and authentication errors include complete recovery commands. +- Audit and repair mismatched Gmail snapshots from archives written between February and May 2026 with `repair-message --audit` and `repair-message `. +- Pick up new Teams chat messages after the first sync and archive self-chats. Existing accounts need no migration or reauthorization. +- Store readable Beeper message text instead of HTML markup and check daily for newly available older history. Existing archives repair derived text on the next sync or through `repair-derived --source-type beeper`. +- Preserve Beeper transcript metadata and Discord voice-message flags and waveforms. Existing Discord records support repair with `repair-derived --source-type discord`. +- Retry transient IMAP connection failures and reconnect after dropped connections during label collection. Confirmed mid-sync message removals no longer discard completed progress. +- Refresh archived IMAP content when an edited outgoing message moves to a trusted Sent folder. Servers without advertised folder roles need an account-specific `sync.trusted_imap_sent_mailboxes` setting. +- Recover cached top-level attachments from Apple Mail partial messages without duplicating messages on re-import. Nested attachment restoration remains unsupported. +- Preserve Apple Mail Message-ID values and reply links. Re-import with the same source identifier and `--no-resume`, then run `build-cache --full-rebuild` to repair missing IDs and expose them in SQL and JSON. +- Report unreadable Apple Mail directories instead of treating them as empty archives, including Full Disk Access guidance for macOS permission errors. +- Import PST archives without counting search folders or empty attachment tables as errors. +- Recover usable email text and attachments with malformed content types. When broken MIME structure prevents full parsing, intact headers still preserve identification and threading. +- Prefer more complete duplicate copies when earlier source and eligibility rules tie. Deduplication previews no longer write derived metadata before confirmation. +- Find recipient addresses in older archives through `message_recipients.email_address`. Use `envelope_address` when you need the original header address; the next cache rebuild adds both. +- Recover Web views automatically after initial cache preparation and reduce memory use for ordinary Everything and Files listings. Query failures explain memory or disk limits, and Files offers Retry instead of displaying an unavailable count as zero. +- Keep the daemon discoverable after VM clock changes, preventing a second daemon startup from failing on the existing archive lock. +- Build macOS analytics caches without repeated DuckDB extension code-signature warnings. Account removal also completes unique-attachment cleanup before rebuilding the cache. +- Preserve valid UTF-8 in calendar previews and repair affected historical snippets with `repair-encoding`. +- Register calendars with service-account apps using domain-wide delegation. Missing Calendar permissions or a disabled API report errors without waiting through the full retry budget. +- Import Android WhatsApp databases without the optional group-participant table and confirm the source identity after recoverable import errors. Contact matching requires explicit country codes. +- Repair missing sender details, account-owner attribution, and IMAP labels with `repair-senders`, `repair-identity`, and `repair-labels`. Sender and label repairs preview changes by default. + +### Acknowledgements + +Thanks to everyone who contributed to this release: + +- [@cpcloud](https://github.com/cpcloud) for making identity-index tests independent of platform memory limits. +- [@css521](https://github.com/css521) for handling MCP messages without parameters. +- [@danshapiro](https://github.com/danshapiro) for preserving UTF-8 calendar previews and repairing older snippets. +- [@elviskahoro](https://github.com/elviskahoro) for adding account-scoped embedding builds. +- [@exactmike](https://github.com/exactmike) for improving incremental IMAP and Teams sync, reconnecting IMAP sessions, and repairing labels. +- [@fucx](https://github.com/fucx) for recovering cached attachments from Apple Mail partial messages. +- [@hansn74](https://github.com/hansn74) for adding identity repair, configurable analytics resources, and Calendar service-account support. +- [@jesserobbins](https://github.com/jesserobbins) for fixing CardDAV contact sync with iCloud. +- [@mariusvniekerk](https://github.com/mariusvniekerk) for improving Web Settings and email previews, adding Google Contacts OAuth and browser message links, and maintaining build and API tooling. +- [@mikemikimike](https://github.com/mikemikimike) for preserving the local archive during permanent Gmail deletion. +- [@rodboev](https://github.com/rodboev) for adding managed IMAP drafts and delegated access, recent-contact browsing, CSV and PPTX indexing, and import and repair improvements. +- [@salmonumbrella](https://github.com/salmonumbrella) for building people profiles, relationships, briefs, CardDAV sync, contextual search, and archive browsing and maintenance workflows. +- [@shntnu](https://github.com/shntnu) for adding Apple WhatsApp text imports and updating contributor guidance. +- [@sweenzor](https://github.com/sweenzor) for repairing Beeper message text, older-history sync, and link-preview handling. +- [@ValentinViennot](https://github.com/ValentinViennot) for making Saved Views available to MCP clients. +- [@wesm](https://github.com/wesm) for adding document and visual search, person file retrieval, search evaluation, and improvements to analytics, imports, and documentation. --- diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 0e974a814..fa3de2b9a 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -1,25 +1,83 @@ --- -last_edited: "2026-09-15" +last_edited: "2026-10-05" title: CLI Reference description: Complete command reference for all msgvault commands. --- Find a command by task below, or use `msgvault COMMAND --help` for the flags in your installed binary. This reference follows current `main`; see -[the changelog](changelog.md#unreleased) for the release boundary. +[the 0.21.0 changelog](changelog.md#0210) for features and upgrade notes. | Task | Commands and guides | |---|---| | Add and sync a source | [Choose a source](guides/sources.md), [sync](#sync), [sync-full](#sync-full) | -| Import local exports | [import-eml](#import-eml), [import-mbox](#import-mbox), [import-maildir](#import-maildir), [import-emlx](#import-emlx), [import-pst](#import-pst), [import-slackdump](#import-slackdump), [text imports](usage/text-messages.md) | -| Search and browse | [search](#search), [tui](#tui), [show-message](#show-message), [documents](#documents), [embeddings](#embeddings) | +| Import local exports | [import-eml](#import-eml), [import-mbox](#import-mbox), [import-maildir](#import-maildir), [import-emlx](#import-emlx), [import-pst](#import-pst), [import-slackdump](#import-slackdump), [import-imazing-csv](#import-imazing-csv), [text imports](usage/text-messages.md) | +| Search and browse | [search](#search), [tui](#tui), [show-message](#show-message), [documents](#documents), [embeddings](#embeddings), [multimodal](#multimodal), [eval](#eval) | | Maintain people and contacts | [person](#person), [people guide](usage/people.md), [CardDAV](usage/people-carddav.md) | | Organize accounts | [identity](#identity), [collection](#collection), [update-account](#update-account) | -| Export | [export-messages](#export-messages), [export-eml](#export-eml), [export-attachments](#export-attachments) | +| Read meeting evidence | [meetings](#meetings), [meeting workflow](usage/meetings.md) | +| Export | [export-messages](#export-messages), [export-eml](#export-eml), [export-attachments](#export-attachments), [create-subset](#create-subset) | | Review and remove mail | [stage-delete](#stage-delete), [delete-staged](#delete-staged), [deduplicate](#deduplicate), [gc](#gc) | | Back up and manage attachment storage | [backup](#backup), [pack-attachments](#pack-attachments), [purge-excluded-media](#purge-excluded-media) | | Repair older records | [repair-identity](#repair-identity), [repair-senders](#repair-senders), [repair-message](#repair-message), [repair-derived](#repair-derived), [repair-labels](#repair-labels), [repair-list-ids](#repair-list-ids), [repair-dates](#repair-dates) | -| Operate or integrate | [setup](#setup), [daemon](#daemon), [serve](#serve), [mcp](#mcp), [query](#query), [openapi](#openapi), [agent-token](#agent-token) | +| Operate or integrate | [setup](#setup), [credentials](#credentials), [daemon](#daemon), [serve](#serve), [activity](#activity), [mcp](#mcp), [query](#query), [openapi](#openapi), [agent-token](#agent-token) | + +## meetings + +Read archived meeting evidence through the selected daemon. Requires API schema +2.27.0 or newer. See the [meeting workflow](usage/meetings.md) for coverage and +source limits. + +```bash +msgvault meetings context --id 42 --id 43 --format markdown --output context.md +msgvault meetings actions --status pending --assignee alex@example.com --json +msgvault meetings metrics --domain example.com --after 2026-01-01 --json +``` + +### meetings context + +| Flag | Contract | +|---|---| +| `--id` | Required meeting message IDs, repeatable; at most 100 | +| `--format` | `markdown` (default) or `json` | +| `--include-transcript` | Include archived transcript evidence; off by default | +| `--max-bytes` | UTF-8 content budget, default 131072; range 4096–1048576 | +| `--output`, `-o` | Output file; omitted or `-` writes the exact packet content to stdout | + +Mixed or invalid selections fail. Truncated packets identify omitted content +and meetings. + +### meetings actions and meetings metrics + +These commands share the following scope flags. Values within a filter group +are alternatives; different groups intersect. + +| Flag | Contract | +|---|---| +| `--id` | Meeting message ID, repeatable | +| `--source-id` | Source ID, repeatable | +| `--domain` | Exact participant domain, repeatable | +| `--participant-id` | Exact participant ID, repeatable | +| `--person-id` | Durable person ID; mutually exclusive with `--participant-id` | +| `--after`, `--before` | `YYYY-MM-DD` at UTC midnight; after inclusive, before exclusive | +| `--deletion` | Source deletion state: `any` (default), `active`, or `deleted` | +| `--json` | Structured response including coverage or duration bases | + +ID lists and domain lists accept at most 100 values. IDs must be positive +JavaScript-safe integers. Locally deleted records are always excluded. + +`meetings actions` also accepts: + +| Flag | Contract | +|---|---| +| `--status` | `pending`, `completed`, `cancelled`, or `unknown`; default all | +| `--assignee` | Exact assignee email | +| `--query` | Literal title or description substring, at most 256 characters | +| `--limit` | Action rows per page, default 50; range 1–200 | +| `--cursor` | Opaque continuation cursor from the previous page | + +Actions reflect the current archived source snapshot. Keep filters unchanged +when continuing a page; refresh from page one to see newer evidence. ## Global Flags @@ -46,9 +104,15 @@ in your installed binary. This reference follows current `main`; see Commands that access archive state keep their usual stdout/stderr output while using the same API path as remote access: 1. If `[remote].url` is configured and `--local` is not passed, the CLI talks to that remote server. -2. Otherwise, archive-access commands discover or start the local background daemon and talk to it over HTTP. +2. Otherwise, archive-access commands discover or start the local background daemon and talk to it over HTTP. With `[server].daemon_auto_start = false`, they use a daemon that is already running or starting and never start one. 3. `--local` selects the local daemon even when `[remote].url` is configured; it is not a request to open SQLite in the CLI process. -4. When `--agent-url` and `--agent-token-file` are both supplied, the CLI connects to that remote daemon as a restricted delegated caller using the token from the file. Only `draft-reply` is available in this mode. Owner configuration (`--config`, `--home`, `--local`) is rejected, and the token is never written to logs or argv. The token is transmitted in the `X-Msgvault-Agent-Token` request header; this header is not modeled in the generated OpenAPI clients — it is a transport detail that the CLI handles internally. +4. With both `--agent-url` and `--agent-token-file`, the CLI connects to a + remote daemon as a restricted caller. `draft-reply`, `draft-compose`, + `draft-get`, `draft-edit`, `draft-delete`, and `draft-recover` are + available in this mode. The CLI rejects owner + configuration (`--config`, `--home`, `--local`) and never writes the token + to logs or argv. It sends the token in the `X-Msgvault-Agent-Token` header; + generated OpenAPI clients do not model this transport detail. This makes local and remote msgvault behavior the same from the CLI's point of view and avoids opening a large SQLite database from foreground CLI processes. @@ -85,10 +149,27 @@ msgvault add-account --oauth-app | `--force` | Delete existing token and re-authorize | | `--readonly` | Request Gmail read-only access instead of read + write. Refused if the account already holds write access — see [OAuth Setup](/docs/guides/oauth-setup/#read-only-access) | | `--display-name` | Set a display name for the account | -| `--no-default-identity` | Do not auto-confirm the email address as this account's "me" identity | +| `--no-default-identity` | Do not auto-confirm the email address as this account's "me" identity. Saved across syncs and re-authorization; only explicit `--no-default-identity=false` clears the choice. See [saved identity choice](#saved-default-identity-choice) | If `[oauth].service_account_key` or `[oauth.apps.].service_account_key` is configured, `add-account` authorizes via Google service account domain-wide delegation instead of browser OAuth. Service-account accounts do not use `--headless`, `--force`, or `--readonly`; their scope comes from the domain-wide delegation grant in the Admin Console. +### Saved default identity choice + +`add-account` (including service accounts), `add-imap`, `add-o365` (IMAP and +`--graph`), and `add-teams` save `--no-default-identity` for later syncs. +Omitting the flag when adding or re-authorizing an existing account keeps its +saved choice. Use `--no-default-identity=false` to re-enable automatic defaults. +New accounts confirm their default identity unless the flag is set. + +The flag does not remove existing identities or prevent the one-time legacy +`[identity]` migration from adding configured addresses. If the saved sync +configuration cannot be read, automatic identity confirmation logs a warning +and skips the write. + +Removing the last confirmed identity also saves the no-default choice for +future syncs. To restore automatic confirmation, run the source's add command +with `--no-default-identity=false`. + --- ## add-imap @@ -116,7 +197,7 @@ It tests the connection before saving credentials. | `--port` | `993` | IMAP server port (993 for TLS, 143 for STARTTLS/plain) | | `--starttls` | `false` | Use STARTTLS instead of implicit TLS | | `--no-tls` | `false` | Disable TLS entirely (plaintext, not recommended) | -| `--no-default-identity` | `false` | Do not auto-confirm the username as this account's "me" identity | +| `--no-default-identity` | `false` | Do not auto-confirm the username as this account's "me" identity. Saved across syncs and re-authorization; only explicit `--no-default-identity=false` clears the choice. See [saved identity choice](#saved-default-identity-choice) | Credentials are stored in `tokens/imap_.json` with restricted file permissions (0600). Use app-specific passwords when your provider supports them. @@ -128,14 +209,23 @@ After adding an account, sync it with `msgvault sync-full`. IMAP accounts use th ## draft-reply -Create one reply draft from an archived IMAP message. The daemon requires a -confirmed `--from` identity and an operator grant in `[[imap.drafts]]`. +Create one reply draft from an archived message to an authorized IMAP +destination, or reply within its original Gmail account. The daemon requires +the matching operator grant. `--from` is optional when exactly one confirmed +identity is eligible. ```bash -msgvault draft-reply --from
--body -msgvault draft-reply --from
--body= --json +msgvault draft-reply --body +msgvault draft-reply --all --account --from
--body +msgvault draft-reply --source-id 42 --body= --json ``` +Use `--all` to include the parent sender, To, and Cc recipients. Msgvault +deduplicates them in order, removes the destination account's confirmed +identities, and never copies the parent's Bcc recipients. Use `--account` or +`--source-id` to choose a destination. An offline parent requires one of those +selectors; msgvault never treats imported provenance as provider credentials. + The daemon appends the composed message to the configured literal mailbox with the `\Draft` flag, then stores the local message and its `(mailbox, uidvalidity, uid)` receipt. The server must advertise UIDPLUS. An accepted APPEND without a @@ -159,10 +249,279 @@ not grant access or change the mailbox. The host policy applies per source. For owner callers (API key, browser session, or keyless loopback), any owner caller that can reach the daemon can create drafts on a granted source. A delegated caller authenticated with a restricted agent token additionally requires that -the target source appear in the token's grant; see [agent-token](#agent-token). +the target source and selected sender appear in the token's frozen grant; see +[agent-token](#agent-token). A draft `From` choice is local header selection. +It does not prove provider send-as permission. Draft creation never sends mail. A later sync reconciles the saved membership when the mailbox's UIDVALIDITY changes. Draft creation never moves an IMAP cursor. +For Gmail, enable drafting per source in the daemon host's `config.toml` and +restart the daemon: + +```toml +[[gmail.drafts]] +source_id = 42 +enabled = true +``` + +Gmail accepts draft writes with `gmail.modify`, `mail.google.com`, or +`gmail.compose`. Creation also lists send-as entries. That read accepts +`gmail.settings.basic`, `gmail.modify`, `gmail.readonly`, or +`mail.google.com`. The daemon checks saved scope information when available; +for older tokens without it, Gmail decides whether to allow the request. +The daemon accepts `--from` when it is the primary address or an accepted +alias. Service-account sources use the scopes in their delegated assertion. + +Gmail writes use one provider request. A 429 or rate-limit 403 is a rejection; +retry the command after the limit clears. A transport failure, response-read +failure, or 5xx returns `remote_unknown`. For an uncertain creation, inspect +Gmail using the reported RFC822 Message-ID before creating another draft. +Managed edits and deletes reconcile on retry as described below. The archive +stores the Gmail draft and its `DRAFT` label after a confirmed response. +`draft-reply` never sends mail. + +--- + +## draft-compose + +Create a new plain-text IMAP draft on one selected live source. Provide at least +one `--to`, `--cc`, or `--bcc` value. The flags are repeatable and keep their +envelope roles. + +```bash +msgvault draft-compose --account you@example.com \ + --from you@example.com --to recipient@example.com \ + --subject 'Project update' --body 'Draft text' +msgvault draft-compose --source-id 42 --to recipient@example.com \ + --bcc private@example.com --json +``` + +The source selector is required. The daemon applies the same host draft policy, +confirmed identity check, delegated sender grant, UIDPLUS requirement, and +structured provider outcomes as `draft-reply`. It stores the Bcc envelope in +the draft so the mail application can use it. It never sends the message or +validates provider send-as rights. + +### Beeper chat drafts + +For a Beeper source, `draft-compose` leaves a text draft in the composer of an +existing chat. `--to` takes the Beeper chat ID, and `--body` the text; no other +fields apply. Enable it per source in the daemon host's `config.toml`, the same +way as Gmail, and keep Beeper Desktop running: + +```toml +[[beeper.drafts]] +source_id = 42 +enabled = true +``` + +```bash +msgvault draft-compose --source-id 42 --to '!room:beeper.local' --body 'Draft text' +``` + +The chat's composer must be empty. Each chat has at most one managed draft; +`draft_exists` reports the existing one. `draft-get`, `draft-edit`, and +`draft-delete` work on the returned draft ID, and `draft-delete` clears the +composer. The edit body must be nonblank. Delegated tokens need `draft.create` on the +Beeper source to create; `draft-get` accepts `draft.edit` or `draft.delete`, +`draft-edit` needs `draft.edit`, and `draft-delete` needs `draft.delete`. +Delete-only agents receive draft metadata without committed or pending text, +including in error responses. A creator's `draft_exists` result carries the +draft ID without its text. `draft-get` shows the draft as +Beeper last reported it, which can differ from the text sent because Beeper +formats it. Edit and delete read the composer first and return `draft_conflict` +when someone changed it in Beeper. Beeper has no conditional write, so text typed +between that read and the write can still be cleared. A write Beeper refuses before anything changed +returns `provider_rejected`; a new draft whose text Beeper refused is +discarded, so the chat is free for another `draft-compose`. A write with an +unknown outcome returns `remote_unknown` and keeps the draft's pending +operation; the next edit or delete settles it from what Beeper shows. An edit +that finds an earlier delete settled, or a new draft's text never delivered, +returns `draft_discarded`. When Beeper shows text msgvault cannot +attribute, the command returns `pending_operation`; clear the composer in +Beeper and retry. `not_supported` means the installed Beeper Desktop does not +report chat drafts, `chat_not_found` that Beeper has no chat with that ID, +`provider_unavailable` that Beeper Desktop or its saved token cannot be +reached, and `provider_identity_mismatch` that Beeper answered +for another chat or account. + +--- + +## draft-forward + +Create an IMAP draft that forwards an archived message and reuses its retained +attachment files. Choose exactly one destination with `--account` or +`--source-id`, and provide at least one explicit `--to`, `--cc`, or `--bcc`. +The parent message's Bcc recipients are never copied. +This command requires owner access; agent tokens cannot invoke it. + +```bash +msgvault draft-forward --account you@example.com \ + --from you@example.com --to recipient@example.com --body 'Please review' +msgvault draft-forward --source-id 42 \ + --cc team@example.com --json +``` + +Every nonempty attachment must have a retained, readable file in the archive's +catalog, even when its bytes remain in the original message. Forwarding reuses +those catalog references; it does not restore skipped or missing files from raw +MIME. Empty attachment parts need no stored file. + +A pending, skipped, failed, unavailable, missing, unreadable, or unsupported +occurrence stops the command before `APPEND`. It never creates a partial draft. +The IMAP source must have an enabled `[[imap.drafts]]` entry, and the selected +sender must be a confirmed identity on that source. + +The draft has two inline plain-text parts: your editable note, then a +`---------- Forwarded message ----------` separator with the original headers +and text. The original attachments follow, retaining their names and +`Content-ID` values when present. Attached emails (`message/rfc822`) keep their +bytes with 8-bit encoding. An attached email that needs binary encoding, such as +one containing NUL bytes, bare line breaks, or lines longer than 998 bytes, is +refused. Unknown MIME dispositions are treated as attachments. + +The draft has a new Date and Message-ID. Its subject has one `Fwd:` prefix; +existing `Fw:` and `Fwd:` prefixes are removed. The `X-Msgvault-Forward: 1` +header identifies the layout for note editing. A mail client that preserves +custom headers may include it when you send the message. msgvault only creates +the draft. + +When the server supplies a numeric `APPENDLIMIT`, msgvault checks the encoded +size before uploading; otherwise the server's `APPEND` response decides. This +upload check does not limit the memory used to build the draft. + +With `--json`, attachment preflight refusals emit a JSON object on stderr with +`status: "attachment_preflight_failed"` and a `problems` array. Each problem has +a `reason`, plus `filename`, `part_key`, and `detail` when available: + +| Reason | Meaning | +|---|---| +| `missing_catalog_reference` | An original attachment has no matching catalog row. | +| `attachment_pending`, `attachment_skipped`, `attachment_failed`, `attachment_unavailable` | The catalog records a file that is not stored. `detail` includes its skip reason when available. | +| `unreadable_file` | Opening, reading, verifying, or closing a stored file failed. | +| `catalog_size_mismatch` | The stored file's length differs from the original MIME part. | +| `unrepresented_attachment` | A catalog row has no corresponding original attachment. | +| `attachment_reader_unavailable` | The daemon cannot access the attachment store. | +| `unrepresentable_attachment` | An attachment's metadata or bytes cannot be forwarded, or the generated draft cannot retain its catalog reference. | + +## draft-get, draft-edit, draft-delete, and draft-recover + +Read, edit, or delete a managed draft created by `draft-reply`, +`draft-compose`, or `draft-forward`: + +```bash +msgvault draft-get [--json] +msgvault draft-edit --revision --body [--json] +msgvault draft-delete --revision [--json] +msgvault draft-recover --revision [--json] +``` + +The creation result supplies the opaque `draft_id` and initial revision. + +- `--revision` is required for edit, delete, and recover; use the current + positive revision. +- `--body` is required for edit; `--body=` sets an empty plain-text body. +- `--json` emits one JSON result. + +`draft-edit --body` replaces the note in a draft created by `draft-forward` and +keeps the quoted text and attachments. Other drafts must be `text/plain`; any +other multipart draft returns `invalid_draft` before the provider draft changes. + +`draft-get` reads retained archive content, including discarded drafts, without +connecting to a provider or requiring the source's draft mutation grant. For +IMAP drafts, edit and delete continue to require the configured `[[imap.drafts]]` +policy. For Gmail drafts, edit and delete require the configured +`[[gmail.drafts]]` policy and one of `gmail.modify`, `mail.google.com`, or +`gmail.compose`. They do not list send-as entries. Delete removes the provider +draft and retains its archived content. Gmail edit and delete inspect the +current provider message ID before making a change. A Gmail web edit advances +the local revision and returns `changed_externally`; the next mutation must name +that revision. These commands never send mail. + +Retry a pending Gmail edit or delete with its current revision. The daemon +checks Gmail before writing again. If the original message is unchanged, it +clears the pending claim and runs the requested operation. If Gmail holds a +recorded replacement, the daemon finishes publishing it locally and returns +`recovered` with `revision_mismatch`; review the new revision before retrying. +Other provider edits are adopted with `changed_externally`. A pending delete +can finish locally when Gmail confirms absence, or retry deletion when the +original draft is still present. Confirmed deletions with unfinished local +cleanup retry that cleanup without another provider request. If Gmail no longer +has a draft during an edit retry, the command returns `provider_absent` and +keeps the candidate content. Use `draft-delete` to finish discarding it. + +Recovery applies to IMAP drafts only. `draft-recover` refuses a Gmail draft ID +with `not_supported` for the owner and `not_permitted` for delegated tokens. +The delegated refusal does not reveal whether the draft exists. Gmail +reconciliation uses edit and delete retries. +Delegated tokens with `draft.create` can create Gmail reply drafts. +For a Gmail or IMAP draft, delegated `draft-get` accepts `draft.create`, +`draft.edit`, or `draft.delete`. `draft-edit` requires `draft.edit`, and +`draft-delete` requires `draft.delete`. Each command requires the source's exact +type and identifier and the draft's archived From sender in the grant. The +frozen sender selection applies to get, edit, and delete, including when the +owner issued the token with `--sender`. If Gmail reports an external edit, +the daemon checks the adopted draft's sender again before returning it. + +A grant with only `draft.delete` receives lifecycle metadata, including the +revision, from get and delete responses. These responses omit `content`, +`raw_mime`, and `candidate_content` in JSON and human-readable output, including +pending and refused deletes. A matching `draft.create` or `draft.edit` grant +allows content reads. Delegated recovery continues to return metadata only. + +A missing command permission returns +HTTP 400 `command_not_allowed`. A permitted command targeting another source or +an unknown draft ID streams `not_permitted` before revision checks, draft +policy, source locking, or any provider request. `draft-send-as` remains +owner-only. +For IMAP drafts, recovery resumes a pending operation from recorded receipts. It can publish a known replacement or finish +confirmed removal without APPEND. Delegated recovery requires +`draft.edit` for an edit or active repeat and `draft.delete` for a delete or +discarded repeat, scoped to the source in the grant. An active draft with no +pending operation requires `draft.edit`, including after a delete was aborted +before writing. Recovery output keeps the saved `pending_code`. +Refused results use `refusal_code`; pending cleanup results describe the current +provider result in `observation.code`. See +[Manage a created draft](usage/imap.md#manage-a-created-draft) for revision, +provider checks, retention, retry behavior, and recovery limits. + +### Local chat drafts + +Slack, Teams, and Discord conversations get drafts that live only in msgvault. +They never appear in the provider's composer and no command makes a provider +request. + +```bash +msgvault draft-compose --conversation --body [--reply-to ] [--json] +msgvault draft-get --conversation [--json] +``` + +The conversation's source owns the draft. `--reply-to` names an archived +message in the same conversation. `draft-get --conversation` lists that +conversation's drafts oldest first, so a lost draft ID can be found again; +`--json` returns one array. `draft-get`, `draft-edit`, and `draft-delete` take +the returned `chat-draft-` ID and report `location` as `msgvault`. A stale +revision returns `revision_mismatch` and leaves the draft unchanged. +`draft-recover` returns `not_supported`. Delegated tokens use the same +permissions as mail drafts, scoped to the conversation's source, except that +listing and retrieval need `draft.edit` or `draft.delete`. Removing the source or +conversation deletes its local drafts. + +## draft-send-as + +List Gmail send-as identities for an owner-invoked Gmail account: + +```bash +msgvault draft-send-as [--json] +``` + +Gmail accepts this read with `gmail.settings.basic`, `gmail.modify`, +`gmail.readonly`, or `mail.google.com`. Saved scope information is checked +when available. The command reports the address, display name, primary and +default flags, verification status, and whether the +address is a confirmed msgvault identity. Delegated agent tokens cannot run +this command. It does not require `[[gmail.drafts]]` and never changes Gmail. + --- ## list-folders @@ -196,9 +555,13 @@ Requires a `[microsoft]` section with `client_id` in `config.toml`. See the [OAu | Flag | Default | Description | |---|---|---| | `--tenant` | `common` | Azure AD tenant ID (restricts which accounts can authorize) | -| `--no-default-identity` | `false` | Do not auto-confirm the email address as this account's "me" identity | +| `--headless` | `false` | Sign in with a device code instead of a local browser | +| `--no-default-identity` | `false` | Do not auto-confirm the email address as this account's "me" identity. Saved across syncs and re-authorization; only explicit `--no-default-identity=false` clears the choice. See [saved identity choice](#saved-default-identity-choice) | +| `--graph` | `false` | Sync through the Microsoft Graph mail API instead of IMAP. Creates an `msmail` account. Needs the `Mail.Read` permission. `delete-staged` asks for `Mail.ReadWrite` on first use | -After adding the account, sync it with `msgvault sync-full`. +After adding the account, sync it with `msgvault sync-full`. For a `--graph` +account, use `msgvault sync`. See +[Microsoft Graph mail sync](/docs/guides/oauth-setup/#microsoft-graph-mail-sync). --- @@ -220,7 +583,8 @@ the Microsoft IMAP token used by `add-o365`. Requires `[microsoft].client_id` in | Flag | Default | Description | |---|---|---| | `--tenant` | `common` | Azure AD tenant ID to use for authorization | -| `--no-default-identity` | `false` | Do not auto-confirm the email address as this source's "me" identity | +| `--headless` | `false` | Sign in with a device code instead of a local browser | +| `--no-default-identity` | `false` | Do not auto-confirm the email address as this source's "me" identity. Saved across syncs and re-authorization; only explicit `--no-default-identity=false` clears the choice. See [saved identity choice](#saved-default-identity-choice) | After adding the account, sync it with `msgvault sync-teams`. @@ -275,6 +639,8 @@ msgvault sync-full [account] [flags] | `--folder NAME` | Scan this IMAP folder (repeatable) | | `--skip-folder NAME` | Skip this IMAP folder (repeatable) | | `--source-id ID` | Sync exactly one source by numeric ID; mutually exclusive with the account argument | +| `--build-cache` | Queue a cache refresh after the sync, even inside `min_rebuild_interval` | +| `--no-build-cache` | Skip the post-sync cache refresh | | `--verbose` | Detailed progress output | An account token can select more than one matching source. Use `--source-id` @@ -306,11 +672,22 @@ msgvault sync [account] [flags] | `--folder NAME` | Scan this IMAP folder (repeatable) | | `--skip-folder NAME` | Skip this IMAP folder (repeatable) | | `--source-id ID` | Sync exactly one source by numeric ID; mutually exclusive with the account argument | +| `--build-cache` | Queue a cache refresh after the sync, even inside `min_rebuild_interval` | +| `--no-build-cache` | Skip the post-sync cache refresh | The CLI sends the incremental sync request to the configured remote server or local daemon and streams the daemon's stdout/stderr back to the terminal. The daemon serializes this work with other archive mutations. +The same cache flags apply to message-writing `sync-*` commands. By default, +manual syncs leave a usable stale cache in place until the minimum rebuild +interval expires. The daemon owns any refresh after the sync command returns. +The two flags are mutually exclusive. + +For `sync` and `sync-full`, either cache flag requires daemon API schema 2.31.0 +or newer. The CLI reports an upgrade error before starting sync against an +older daemon. + Folder filters are applied only to IMAP accounts. See [IMAP Folder Sync](/docs/usage/imap/) for examples and matching rules. @@ -478,6 +855,56 @@ privacy, retry behavior, and stored evidence. --- +## add-plaud + +Authorize and register a configured Plaud cloud account using browser OAuth. + +```bash +msgvault add-plaud [identifier] +``` + +With one configured `[[plaud]]` entry, omit the identifier. The browser callback +runs on the daemon host at `localhost:8091/callback/plaud`. A configured remote +refuses before proxying; use SSH with that port forwarded and run +`msgvault --local add-plaud ` on the daemon host. Credentials are +stored in `tokens/plaud_.json` and bound to the exact MCP endpoint. + +The live account email must equal configured `account_email` before source +registration. A source's confirmed owner cannot change under the same +identifier. Use a new identifier for another account. See +[Plaud configuration](configuration.md#plaud-sources). + +## sync-plaud + +Archive Plaud cloud recordings, complete transcripts, and every note tab. +Cloud Sync and upstream transcription must already be enabled. Audio is not +downloaded; no changes propagate to Plaud. + +```bash +msgvault sync-plaud [identifier] +msgvault sync-plaud work --limit 20 +msgvault sync-plaud work --full --after 2025-01-01 +msgvault sync-plaud work --probe +``` + +| Flag | Description | +|---|---| +| `--limit n` | Hydrate at most n recordings; newest first, then rotate through least recently attempted. Failed recordings retry on their next turn. 0 is unlimited; negatives fail | +| `--full` | Force archive repair, preserving stable source and file IDs | +| `--after YYYY-MM-DD` | Filter recording dates locally; implies `--full` and retains rotation progress | +| `--probe` | Print tool names, input schemas, and first-page counts without personal content or archive writes; requires an identifier when multiple accounts are configured | +| `--build-cache` | Refresh analytics cache after sync | +| `--no-build-cache` | Skip analytics cache refresh; mutually exclusive with `--build-cache` | + +Without an identifier, sync every configured Plaud entry. Every run validates +the live account owner and checks complete content for edits. Sync refuses an +unregistered source before authentication; run `add-plaud` first. + +Missing pending transcripts and notes preserve previous evidence. Deleted +recordings remain archived. Failed and canceled runs save rotation progress and +refresh committed changes before returning the error. See the +[meeting guide](usage/meetings.md#plaud) for pagination and consistency limits. + ## add-circleback Authorize a configured Circleback account using browser OAuth (their MCP @@ -541,6 +968,50 @@ See [Meeting Transcripts](/docs/usage/meetings/) for setup and what gets stored. --- +## add-muesli + +Check a configured local Muesli database and register it as a meeting source. + +```bash +msgvault add-muesli [identifier] +``` + +The matching `[[muesli]]` entry requires `account_email`; `db_path` defaults +to `~/Library/Application Support/Muesli/muesli.db`. With one entry, the +identifier may be omitted. The command fails unless the file opens read-only +as a Muesli database on the daemon's host. + +--- + +## sync-muesli + +Archive completed meetings from a local Muesli database. + +```bash +msgvault sync-muesli [identifier] +msgvault sync-muesli mac --limit 5 +msgvault sync-muesli --after 2026-01-01 +msgvault sync-muesli --full +``` + +Every run reads the whole database read-only and updates changed meetings in +place; unchanged meetings are skipped. Attendees are resolved through Apple +Contacts unless `contacts = false`; the summary reports whether Contacts was +`complete`, `partial`, `unavailable`, or `off`. Meetings still recording or processing +wait for a later run, and meetings deleted in Muesli stay archived. With no +identifier, every configured `[[muesli]]` source is synced. + +| Flag | Default | Description | +|---|---|---| +| `--limit` | `0` | Maximum meetings processed per run (`0` = unlimited) | +| `--after` | — | Only meetings that start on or after this UTC date (`YYYY-MM-DD`) | +| `--full` | `false` | Rewrite every archived meeting, even unchanged ones, to refresh attribution | + +See [Meeting Transcripts](/docs/usage/meetings/#muesli) for setup and what gets +stored. + +--- + ## archive-remote-images Download remote `` images from existing email for offline viewing. @@ -691,10 +1162,10 @@ msgvault backfill-beeper-media --account signal Register a [Slack workspace](/docs/usage/slack/) as a `slack` source. Requires a user token (`xoxp-…`) from an internal Slack app you create (see the usage -guide for the two-minute setup and scope list). The token is validated with -`auth.test` plus a `search.messages` probe (thread-reply archiving needs the -`search:read` scope, so an under-scoped token fails here rather than on -every future sync) and stored at `tokens/slack__.json`. +guide for setup and scope choices). The token is validated with `auth.test` +and stored at `tokens/slack__.json`. Public-channel-only +tokens need no search, file, or reaction permissions; sync revisits history +for thread replies when `search:read` is absent. ```bash msgvault add-slack @@ -736,24 +1207,31 @@ msgvault import-slackdump --me U0123456789 --limit 100 /path/to/export.zip ## sync-slack -Sync Slack conversations — channels you are a member of, group DMs, and 1:1 -DMs — for registered workspaces. The first run backfills full history and is +Sync Slack conversations for registered workspaces. Public-channel-only tokens +can archive all public channels, including unjoined ones. Broader tokens archive +your channel memberships, group DMs, and 1:1 DMs. The first run backfills full history and is resumable; later runs are incremental and sweep for thread replies created since the last run (any thread age). Per-workspace failures do not stop the run: remaining workspaces still sync and the command exits non-zero listing the failures. The `[slack]` -config `channels`/`exclude_channels` filters select which channels sync. See +config `channels`/`exclude_channels` filters select which channels sync. The +`private_channels`, `dms`, and `group_dms` settings independently select whether +private channels, DMs, and group DMs sync. See [Slack](/docs/usage/slack/). ```bash msgvault sync-slack msgvault sync-slack T0123456789 msgvault sync-slack --full +msgvault sync-slack --private-channels=false --dms=false --group-dms=false ``` | Flag | Default | Description | |---|---|---| | `--limit` | `0` | Max messages of work per conversation this run, thread replies included; the reply sweep gets the same budget workspace-wide (0 = no limit; every phase resumes next run so standing limited schedules converge; only the maintenance rescan is skipped) | +| `--dms` | configured | Include one-to-one DMs for this run (`true` or `false`) | | `--full` | `false` | Start (or continue) a repair session: re-fetch every message, upserting in place (catches old thread replies and edits). Interrupted or --limit-scoped repairs resume across later runs of any kind until complete | +| `--group-dms` | configured | Include group DMs for this run (`true` or `false`) | +| `--private-channels` | configured | Include private channels for this run (`true` or `false`) | | `--no-threads` | `false` | Skip thread-reply fetching for this run (a later threaded run pays the debt automatically) | | `--maintenance` | `false` | Repair edits/reaction changes on recent messages (ignored by default after capture) | | `--no-media` | `false` | Skip file downloads for this run (files become pending markers; `backfill-slack-media` fetches them later) | @@ -784,6 +1262,7 @@ msgvault add-calendar [flags] | Flag | Description | |---|---| +| `--write` | Also request `calendar.events`, preserving existing Google scopes; source write permissions remain required | | `--oauth-app` | Named OAuth app to use | | `--headless` | Print token-copy instructions for a headless host instead of opening a browser | | `--all-calendars` | Include reader/freeBusyReader (subscribed, holiday) calendars | @@ -813,6 +1292,49 @@ msgvault sync-calendar [flags] --- +## calendar + +Control live Google Calendar events through the daemon. This is unreleased +functionality. Follow [Calendar event setup](usage/calendar.md#control-events-unreleased) +for consent and source permissions. + +```bash +msgvault calendar create --account --summary --from <start> --to <end> +msgvault calendar update <calendar-id> <event-id> --account <name|email> [field flags] +msgvault calendar delete <calendar-id> <event-id> --account <name|email> +msgvault calendar move <calendar-id> <event-id> <destination-calendar-id> --account <name|email> +msgvault calendar respond <calendar-id> <event-id> --account <name|email> --status accepted +msgvault calendar freebusy <calendar-id> --account <name|email> --from <start> --to <end> +msgvault calendar conflicts <calendar-id> --account <name|email> --from <start> --to <end> --calendars <ids> +``` + +| Flag | Applies to | Contract | +|---|---|---| +| `--account` | All | Required OAuth account or configured source name, separate from the target calendar | +| `--dry-run` | All | Verify live access and return the proposed writes without applying them; availability still reads Google | +| `--read-only` | All | Reject all mutations | +| `--json` | All | Print complete results and archive receipts; dry runs and availability always print JSON | +| `--send-updates` | Writes | `none` (default), `all`, or `externalOnly` | +| `--summary`, `--description`, `--location` | Create/update | Omitted update fields are preserved; explicit empty clears | +| `--from`, `--to` | Create/update/availability | RFC3339 with offset, or local `YYYY-MM-DDTHH:MM` with `--tz` | +| `--tz` | Create/update/availability | IANA time zone; event edits require a time bound | +| `--all-day` | Create/update | Date-only `YYYY-MM-DD`; end is exclusive | +| `--attendees` | Create/update | Replace comma-separated guest list; explicit empty clears | +| `--add-attendee` | Update | Add guest emails while preserving existing guests and RSVP state; cannot combine with `--attendees` | +| `--rrule` | Create/update | Repeatable RRULE; explicit empty clears recurrence | +| `--reminder` | Create/update | Repeatable `popup:minutes` or `email:minutes` (0–40320, at most five); `default` or `none` cannot combine with overrides | +| `--scope` | Update/delete/respond | `single` (default), `all`, or `future`; future supports update/delete only | +| `--original-start` | Update/delete/respond | Original occurrence start, RFC3339 or all-day date; required for single on a series ID and future edits | +| `--destination` | Move | Required calendar ID or configured alias; standalone events only | +| `--status` | Respond | Required self RSVP: `accepted`, `declined`, or `tentative` | +| `--calendars` | Availability | Selected IDs or aliases; default is the positional calendar; at most 50 | + +Commands return completed Google writes with archive message IDs. A partial +remote failure or an archive failure exits with an error after printing the +receipts. Reconcile the reported event; do not repeat a completed mutation. + +--- + ## import-eml Import RFC 5322 `.eml` files inside MailMate-style `.mailbox` directories. @@ -923,10 +1445,19 @@ msgvault import-emlx <identifier> <mail-dir> The mail directory should be an Apple Mail mailbox tree containing `.mbox` or `.imapmbox` directories, each with a `Messages/` subdirectory of `.emlx` files. You can also point directly at a single `.mbox` directory. Labels are derived from directory names. -Apple Mail's `N.partial.emlx` files are also imported: their message body is -complete even when uncached attachment parts are absent. If both `N.emlx` and +Apple Mail's `N.partial.emlx` files are also imported. Apple Mail keeps the +attachments of these messages outside the MIME payload, in a sibling +`Attachments/N/` directory. The importer restores cached attachments directly +inside the message's outer multipart, within the message size limit. +Attachments in nested parts, such as inside some forwarded messages, are not +restored. An attachment without a cached file stays absent; unreadable files +or directories produce a warning. Re-importing a partial message adds newly +cached attachments to the existing message without creating another copy. +When the archived message already holds every cached attachment, the rerun +skips it instead of rewriting it. If both `N.emlx` and `N.partial.emlx` exist, the complete `N.emlx` copy wins. The command summary -reports the number of partial files imported. +reports the number of partial files read and how many attachments the run +added to the archive. | Flag | Default | Description | |---|---|---| @@ -1013,6 +1544,39 @@ See [Text Messages](/docs/usage/text-messages/) for usage examples. --- +## import-imazing-csv + +Import iMessage and SMS history from an iMazing Messages CSV export. Pass the +export root containing `csv/` and optional `attachments/`, or pass its `csv/` +directory directly. + +```bash +msgvault import-imazing-csv ~/Downloads/messages-export \ + --me +14155550100 --timezone America/Los_Angeles +``` + +| Flag | Default | Description | +|---|---|---| +| `--me` | (required) | Your phone number or email address | +| `--timezone` | local IANA zone (required on Windows) | Timezone used for dates that do not include an offset | +| `--contacts` | — | vCard file used to fill empty participant names | + +The importer accepts comma, tab, and semicolon CSV files with named iMazing +headers. It is deterministic across reruns. Available referenced files are +stored up to 100 MiB each. Larger files are recorded as skipped; missing files +and ambiguous filenames remain visible as missing attachment occurrences. +These outcomes do not fail the import or prevent contact enrichment, and a +later rerun can fill missing files. Reply links are added only when +the exported reply text identifies exactly one earlier message. + +CSV exports do not share stable message IDs with `chat.db`. Importing the same +history through both commands can create cross-source duplicates. + +See [Text Messages](/docs/usage/text-messages/#import-imazing-csv) for the full +format and rerun behavior. + +--- + ## import-gvoice Import texts, calls, and voicemails from a Google Voice Takeout export. @@ -1219,8 +1783,21 @@ msgvault pack-attachments The daemon serializes packing against sync and backup operations. Reads remain available from loose, packed, or mixed storage, and the command is safe to rerun as new loose content arrives. Bounded packing also runs after successful -attachment-producing operations and during scheduled maintenance; this command -processes the complete eligible backlog immediately. +attachment-producing commands sent to the daemon. Scheduled syncs never pack +inline: when one writes new loose blobs, the daemon's `attachment-pack` job +(every 6 hours) packs them, and the daily `attachment-maintenance` job packs +and repacks. Each automatic pack pass has a 256 MiB soft raw-byte budget and +finishes its current blob before stopping for that byte limit. Scheduled packing +and daily attachment maintenance also stop after one minute, which lets queued +scheduled work run. Each scheduled pack pass verifies at most 128 +existing packed blobs or 32 MiB of their raw content, allowing one oversized +blob. Its durable hash cursor resumes verification after a restart, while new +loose blobs can be packed during each pass. Manual unpacking reads the full +catalog. Cancellation preserves committed packs. A pass with committed +progress resumes behind waiting work; runtime expiry without a checkpoint +records an error and waits for the next trigger. The first pack tick after a +restart also checks for blobs left by the previous daemon. +This command processes the complete eligible backlog immediately. With `[data].loose_attachments = true`, automatic packing is disabled and this command refuses to run. @@ -1292,6 +1869,22 @@ contain spaces. Repeating either alias requires every value to match. `--mode vector` and `--mode hybrid` require at least one free-text term in the query (filter-only queries use `--mode fts`). They do not support pagination (`--offset` is rejected) or non-active deletion scopes because the vector index covers active messages only. Bump `--limit` to retrieve a larger candidate pool instead. See [Searching](/docs/usage/searching/) for the operator reference and [Vector Search](/docs/usage/vector-search/) for semantic setup. +Search tables show the subject, or the message snippet when the subject is blank. +Redirected or piped output keeps the full sender and subject/snippet on one line, +with terminal controls removed and whitespace collapsed. Terminal tables fit the +available width by shortening sender and subject text, including Unicode characters +and emoji. Fixed IDs, dates, sizes and explanation scores stay complete; very narrow +terminals may wrap. An unknown message size appears as `-`. + +With `--json`, each result also includes `web_url` when the selected daemon can +provide a browser link for that message. + +The daemon checks full-text index completeness in the background. The CLI +warns that results may be incomplete when the daemon finds an index gap or +is rebuilding the index. The warning also applies while a rebuild waits for +other daemon work to finish. A completeness check alone prints no warning. +API clients can still observe `index_state="checking"` while that check runs. + --- ## repair-list-ids @@ -1306,6 +1899,11 @@ msgvault repair-list-ids [--apply] The default is a dry run and does not modify the archive. Pass `--apply` to write changed values and mark derived analytics stale for the normal rebuild path. +The analytics cache exports invalid UTF-8 List-IDs as NULL so mailing-list +filters cannot match unrelated messages through replacement characters. +Use `--apply` to recover these values when the stored raw MIME has an intact +List-Id header. This cannot recover a header that is itself damaged. + --- ## repair-labels @@ -1364,6 +1962,19 @@ review their disclosure and preflight. and bound to a stable index revision; restart pagination after a stale-cursor error. +In v0.21.0, build failures include available local causes alongside +the document hash and reason code. `documents status` lists up to 20 current +failed documents, including failed replacements in an active rebuild. JSON +output adds `status.failures` and `status.failures_exhausted`. Each failure has +`canonical_blob_hash`, `reason_code`, `detail`, and `state`; detail is a bounded +single line and may be empty for historical failures. Use `documents retry` and +then build again to obtain fresh diagnostics for an older failure. + +To include inline documents, set +`[attachments.documents.scope].include_inline = true` and record exact consent +again. The default is false. See the +[scope configuration](configuration.md#attachmentsdocuments) for the contract. + See [Document Attachment Indexing](/docs/usage/document-indexing/) for fixture generation, configuration, privacy boundaries, scheduling, and recovery. @@ -1427,9 +2038,36 @@ and validation contract. --- +## export-discord + +Export a bounded interval from one Discord guild using the older +`msgvault-discord-export/1` JSON envelope. + +```bash +msgvault export-discord <guild-id-or-name> \ + --start 2026-01-01T00:00:00Z \ + --end 2026-02-01T00:00:00Z +``` + +| Flag | Description | +|---|---| +| `--start <RFC3339>` | Inclusive lower bound (required) | +| `--end <RFC3339>` | Exclusive upper bound (required) | +| `--format json` | Output format; `json` is the only supported value | + +This compatibility command reads only the archive and does not contact +Discord. New integrations should use [`export-messages`](#export-messages), +which emits the provider-neutral `msgvault-message-export/1` JSONL schema. See +the [Discord export guide](usage/discord.md#export-a-bounded-history-window). + +--- + ## export-eml -Export a message as a `.eml` file. Accepts either a numeric database ID or a Gmail message ID. +Export a message as a `.eml` file. Accepts an internal message ID or a provider +message ID. A numeric reference selects a live internal ID first, then falls +back to a provider ID if that internal ID does not exist. This also applies +with `--thread`. ```bash msgvault export-eml <id> [flags] @@ -1437,7 +2075,15 @@ msgvault export-eml <id> [flags] | Flag | Description | |---|---| -| `-o`, `--output <path>` | Output file (default: `<gmail_id>.eml`, use `-` for stdout) | +| `-o`, `--output <path>` | Output file (default: `<source_message_id>.eml`, use `-` for stdout). With `--thread`, the output directory (default: current directory) | +| `--thread` | Write visible archived messages in the conversation that have stored MIME as `<n>-<source_message_id>.eml`, numbered oldest first. Reports skipped messages and when the account last synced. Needs a daemon with API schema 2.33.0 or newer | +| `--account <email>` | With `--thread`, the account that holds a provider message ID found in more than one account | + +Thread export captures its message list once. Messages added afterward belong +to the next export. It reports and skips messages that disappear or lose their +stored MIME before they are read; other read or write errors stop the command. +Hidden duplicate copies are excluded. The last-sync time describes account +freshness, not conversation completeness. --- @@ -1477,6 +2123,32 @@ Accepts internal numeric IDs or Gmail message IDs. See [Exporting Data](/docs/us --- +## create-subset + +Create a new SQLite archive containing the requested number of most recent +messages and the records they reference. The destination receives its own +`msgvault.db` and can be opened as a separate msgvault home. + +```bash +msgvault create-subset --output ./subset-vault --rows 1000 +MSGVAULT_HOME=./subset-vault msgvault tui +``` + +| Flag | Description | +|---|---| +| `-o`, `--output <directory>` | Destination directory (required) | +| `--rows <count>` | Number of most recent messages to copy; must be positive (required) | +| `--include-identity` | Copy complete identity clusters for included participants | +| `--include-attributes` | Copy current and historical person and organization attribute values | +| `--include-profiles` | Copy profiles, profile history and media, relationships, employment history, and referenced organizations | +| `--include-vcard-resources` | Copy complete native vCards and retired UID aliases; requires `--include-profiles` | + +The command is SQLite-only. The optional identity, attribute, profile, and +vCard flags can copy personal records that have no message in the subset; read +the command's warning before sharing its output. + +--- + ## export-token Export a browser-created OAuth refresh token to a remote msgvault instance. @@ -1536,6 +2208,58 @@ Use the [people guide](/docs/usage/people/) for the workflow. Observed contacts use participant IDs; saved profiles use person IDs. Each command below takes the ID named in its arguments. +### identity matches + +Review archive-derived identity suggestions with a token from the current +review snapshot: + +| Command | Purpose | +|---|---| +| `identity matches list [--state candidate] [--limit 100] [--offset 0] [--json]` | List candidate, accepted, rejected, conflict, or all match records | +| `identity matches show <id> [--json]` | Inspect evidence, blockers, and the current review token | +| `identity matches accept <id> --review-token <token> [--notes-file <path> | --notes-stdin] [--json]` | Accept the reviewed suggestion and apply its participant link | +| `identity matches reject <id> --review-token <token> [--notes-file <path> | --notes-stdin] [--json]` | Reject the reviewed suggestion while retaining its record | + +Always inspect the match before deciding. Evidence or endpoint changes make a +review token stale; fetch the match again and decide with its new token. A +successful acceptance records the decision and applies the link. Review notes +are private data. See [people and profiles](/docs/usage/people/#review-identity-matches) +for how to review candidates safely. + +### person scoring + +Run a bounded scoring batch to create identity review suggestions and journal +the results. Scoring never accepts matches. Configure the disabled-by-default +[`people.identity_scoring`](configuration.md#people-identity-scoring) settings, +inspect the provider disclosure and data fields, then consent to its fingerprint: + +```bash +msgvault person scoring status +msgvault person scoring consent <disclosure-fingerprint> +msgvault person scoring run --limit 20 +msgvault person scoring history --limit 20 +``` + +| Command | Purpose | +|---|---| +| `person scoring status [--json]` | Show readiness, credential availability, consent, blockers, and the provider disclosure | +| `person scoring consent <disclosure-fingerprint> [--json]` | Consent to the exact current disclosure | +| `person scoring revoke <disclosure-fingerprint> [--json]` | Withdraw that disclosure's consent without changing configuration | +| `person scoring run [--limit <n>] [--json]` | Score and journal one batch; local blockers prevent provider requests for those pairs | +| `person scoring history [--candidate-id <id>] [--limit <n>] [--before-id <id>] [--json]` | Read redacted judgments, optionally for one candidate or below a history cursor | + +`run --limit` defaults to the configured batch size and cannot exceed it. +History defaults to 100 rows and accepts limits from 1 through 100. Candidate +and cursor IDs must be nonnegative; zero means no filter. Use the returned +`next_before_id` with `--before-id` for older history. + +Each run result includes its candidate's review token, proposed action, and +blockers. If a batch stops early, completed results remain in the response and +history, and the command reports the batch error. Inspect the candidate's +current evidence before accepting a proposal. See the +[people guide](usage/people.md#optional-identity-scoring) for the consent and +provider boundaries. + ### person notes Read, replace, or append private Notes while retaining earlier values: @@ -1601,8 +2325,13 @@ commands accept `--limit` (1–200) and `--offset`. `sweep run --backstop` revisits older evidence in a bounded pass. Provider configuration edits run on the daemon host. Restart the daemon after -changing its active provider or policy. For setup, privacy controls, budgets, -and recovery, see [profile automation](/docs/usage/people-automation/). +changing its active provider or policy. Profile removal uses the running daemon +to revoke consent and clear checks for both the saved and running policies. If +the daemon is incompatible, stop it before removing a profile. With no daemon, +removal updates the local archive directly. + +For setup, privacy controls, budgets, and recovery, see +[profile automation](/docs/usage/people-automation/). [Provider status](#person-provider-status), [reverify](#person-provider-reverify), [set](#person-provider-set), and [briefs](#person-brief) have additional details below. @@ -1686,25 +2415,33 @@ for direction and date examples. ## add-carddav, sync-carddav, and carddav -Connect one CardDAV account, choose its address-book roles, and resolve sync +Connect CardDAV accounts, choose their address-book roles, and resolve sync conflicts. Publishing or resolving a conflict can change the external address book; see the [CardDAV guide](/docs/usage/people-carddav/). | Command | Purpose | |---|---| -| `add-carddav <base-url> <username> [--schedule <cron>] [--disabled]` | Discover and save an account; password is prompted or read from piped stdin | -| `add-carddav --google <email> [--oauth-app <name>] [--schedule <cron>] [--disabled]` | Connect Google Contacts using an authorized account token | +| `add-carddav <base-url> <username> [--connection <name>] [--schedule <cron>] [--disabled]` | Discover and save an account; password is prompted or read from piped stdin | +| `add-carddav --google <email> [--connection <name>] [--oauth-app <name>] [--schedule <cron>] [--disabled]` | Connect Google Contacts using an authorized account token | | `carddav authorize-google <email> [--oauth-app <name>] [--no-browser]` | Authorize Google Contacts in the browser, preserving existing Google permissions | -| `sync-carddav [--full]` | Synchronize the account; `--full` reconciles complete books | +| `sync-carddav [--connection <name>] [--full]` | Synchronize all enabled connections, or the selected connection; `--full` reconciles complete books | +| `carddav connections` | List connection names, enablement, runtime availability and orphaned accounts | | `person publish <person-id>` / `person unpublish <person-id>` | Publish a saved profile or remove its remote card | | `person publish <person-id> --preview` | Print the exact vCard and approval token as JSON without publishing | | `person publish <person-id> --approve <token>` | Approve reviewed changes; conflict previews also need explicit `keep_local` resolution | -| `carddav books` | List discovered books and their roles | +| `carddav books [--connection <name>]` | List discovered books, owning connection names and roles | | `carddav books set-role <book-id> [--write-target] [--subscribed] [--lookup-source]` | Replace all three roles; omitted flags become false | | `carddav conflicts list` | List unresolved conflicts | | `carddav conflicts show <conflict-id>` | Compare bounded base, local, and remote summaries | | `carddav conflicts resolve <conflict-id> <keep_local\|keep_remote>` | Choose the local or remote side explicitly | +Unqualified `add-carddav` saves `default`. Explicit sync selects one saved +connection, including a disabled connection for manual repair. Unqualified sync +with no enabled connection fails as unavailable. Aggregate sync prints counts +and safe failure codes per connection, and exits nonzero for partial or failed +outcomes. Book IDs, publication commands and conflict commands remain global; +network operations use the owning connection's credentials. + Directory and integrations can also use the [publication API](/docs/usage/people-carddav/#sync-and-publish-selected-people). CardDAV commands do not expose a general `--json` flag; role changes, @@ -1744,7 +2481,7 @@ msgvault identity import [<account>] [--source-id <id>] (--file <path> | --stdin | `--json` | `list`, `show`, `discover`, `import` | Output structured JSON; discovery also suppresses progress | | `--signal` | `add` | Evidence signal name (default `manual`) | | `--apply` | `discover` | After the complete preview scan, confirm strong evidence | -| `--provider` | `discover` | Include the source's configured `[[fastmail]]` alias inventory | +| `--provider` | `discover` | Include the authenticated Gmail profile or the source's configured `[[fastmail]]` alias inventory | | `--confirm <address>` | `discover` | Explicitly confirm one weak candidate; repeatable and requires `--apply` | | `--file <path>` / `--stdin` | `import` | Read a text or JSON identity list from exactly one input | | `--signal` | `import` | Evidence signal recorded for imported identities (default `manual`) | @@ -1784,10 +2521,13 @@ msgvault person attributes set <person-id> <slug> (--value <scalar> | --value-js msgvault person attributes clear <person-id> <slug> [flags] ``` -`promote` is idempotent. `set-display-name` preserves the profile's stable ID -and vCard UID. `delete` permanently retires that UID and removes the profile's -participant bindings. A person with active merge lineage cannot be deleted -until that lineage is fully split. +`promote` seeds a new profile with the first nonblank observed name in its +linked cluster, ordered by participant ID. It leaves the name empty when no +member has an observed name. Repeating promotion preserves the saved name, +including an edited or cleared value. `set-display-name` preserves the +profile's stable ID and vCard UID. `delete` permanently retires that UID and +removes the profile's participant bindings. A person with active merge lineage +cannot be deleted until that lineage is fully split. `merge` keeps the survivor's ID and vCard UID, moves the absorbed profile into it, and records a reversible merge packet. Profiles with active CardDAV @@ -1836,6 +2576,52 @@ shipped definitions and complete workflow. --- +## person agenda + +Read and organize a person's live Kata tasks. Configure +[`[integrations.kata]`](configuration.md#integrationskata) on the daemon first. +Tasks belong to one person and stay in the configured Kata project. + +```bash +msgvault person agenda list <person-id> [--json] +msgvault person agenda create <person-id> --title "Discuss the proposal" [flags] +msgvault person agenda link <person-id> <ref> [--list agenda] [--json] +msgvault person agenda edit <person-id> <ref> --list follow-up [--json] +msgvault person agenda unlink <person-id> <ref> [--json] +``` + +`<ref>` is a task reference returned by Kata or an agenda command. `create` +creates a task and links it to the person; `link` attaches an existing task. +`edit` changes only its list. `unlink` removes the person link without deleting +the task. Complete or reopen tasks, or edit their title, body, priority, and +labels, in Kata. + +| Flag | Commands | Default | Description | +|---|---|---|---| +| `--title` | `create` | — | Required task title | +| `--body` | `create` | — | Task body | +| `--priority` | `create` | — | Kata priority from 0 through 4 | +| `--label` | `create` | — | Kata label; repeat for multiple labels | +| `--list` | `create`, `link`, `edit` | `agenda` for create/link | List name; required for `edit` | +| `--idempotency-key` | `create` | Generated | Stable key to reuse when retrying the same creation | +| `--json` | All agenda commands | `false` | Print the response as JSON | + +`create` prints its generated retry key to stderr before sending the request. +To retry, pass that key with `--idempotency-key` and keep the task details the +same. You can also choose the key in advance: + +```bash +msgvault person agenda create 42 --title "Discuss the proposal" --idempotency-key proposal-discussion-1 +``` + +`list` returns up to 100 open tasks and recognizes the person's current vCard +UID and aliases. JSON includes `truncated` when more items remain; use Kata to +view the rest. Oversized Kata responses produce an explicit error. See the +[Kata configuration](configuration.md#integrationskata) for metadata and +response limits. + +--- + ## person directory Browse promoted people by last contact through the selected local or configured remote daemon. The default order is most recent first, `last_contact_desc`. Each page uses the daemon's default of 50 people. @@ -1864,6 +2650,64 @@ JSON contains a `people` array and optional `next_cursor`. Each person retains t --- +## person provider add + +Create and synthetically check a named people inference profile. Select a +built-in preset to bind its protocol, endpoint, and authentication scheme: + +```bash +msgvault person provider add primary --provider openrouter --model <model> \ + --credential-env OPENROUTER_API_KEY --retention-posture <assertion> \ + --training-posture <assertion> --source conversation_text \ + --source-since 2026-01-01 --allow-sensitive=true --yes +msgvault person provider consent primary --yes +msgvault person provider use primary +``` + +`--provider` accepts `openai`, `openrouter`, or `venice` and requires an explicit +model and privacy policy. It cannot be combined with `--custom` or +`--accept-catalog-prices`; conflicting protocol, endpoint, or auth overrides +are rejected. `--credential-env` reads only the named host variable. Alternatively, +`--api-key-stdin` reads a key from standard input, or an interactive terminal +prompts for it. A successful synthetic check does not grant consent or select +the profile. See [profile automation](usage/people-automation.md) for custom +protocol profiles and policy fields. + +## person provider enroll-codex + +Create, check, consent to, and select a new Codex profile through the daemon. +**Codex enrollment is unavailable in this release:** no Codex build is approved. +The daemon returns HTTP 503 before changing credentials or consent. This command +requires a terminal and never starts a noninteractive device login. + +```bash +msgvault person provider enroll-codex <new-name> \ + --retention-posture <assertion> --training-posture <assertion> \ + --source conversation_text --source-since 2026-01-01 --allow-sensitive=true +``` + +The gated flow prints a verification URL, user code, and local deadline, waits +for sign-in, then prompts for an available model and reasoning effort. Existing +profile names are rejected. After saving, a synthetic check and separate +consent are required before selection. Declining consent leaves the saved +profile unselected. Restart the daemon when the reported running profile differs. + +| Flag | Default | Description | +|---|---|---| +| `--model` | prompt | Codex model ID available to the signed-in account | +| `--reasoning-effort` | prompt | Supported effort for the chosen model | +| `--retention-posture`, `--training-posture` | required | Explicit operator assertions | +| `--source` | required | Repeatable: `conversation_text`, `meeting_text`, or `document_text` | +| `--source-since` | required | Earliest disclosed date, `YYYY-MM-DD` | +| `--source-until` | absent | Latest disclosed date, on or after `--source-since` | +| `--allow-sensitive` | explicit decision required | `true` permits archive evidence; `false` permits only synthetic checks | +| `--yes` | `false` | Confirm the displayed check disclosure and select the profile without a consent prompt | + +Host-side `person provider login` reauthenticates the selected existing Codex +profile; `person provider models` lists its models and reasoning efforts. Both +accept `--json` and remain unavailable behind the same release gate. See +[Codex configuration](configuration.md#codex-app-server-profiles). + ## person provider status Show the exact people inference provider policy and its check and consent state. @@ -2105,7 +2949,7 @@ msgvault build-cache [flags] | Flag | Description | |---|---| -| `--full-rebuild` | Discard existing cache and rebuild | +| `--full-rebuild` | Rebuild all cache files from scratch | The CLI sends the request over HTTP and streams the daemon's stdout/stderr back to the terminal. A local daemon runs the DuckDB export in an isolated child @@ -2114,7 +2958,33 @@ long-lived daemon process. With `[remote].url` configured, the remote daemon builds its own cache; use `--local` only to target this machine's local daemon. For automatic cache rebuilds after daemon-owned syncs, configure -`[analytics].auto_build_cache` in `config.toml`. +`[analytics].auto_build_cache` in `config.toml`. See +[analytics settings](configuration.md#analytics) for snapshot freshness, +builder memory, threads, and temporary disk limits. + +Build warnings count exported text values repaired with U+FFFD and identity +values exported as unknown. Invalid source message IDs, source conversation +IDs, and source identifiers export as empty strings; valid keys stay unchanged. +Invalid List-IDs export as NULL; see [List-Id recovery](#repair-list-ids). +The counts cover only datasets written by that build or refresh, on every +platform. A source value written to two output +columns counts twice; repeated identity comparisons do not add to the count. +See [encoding repair limits](#repair-encoding) for archive recovery. + +--- + +## activity + +Refresh contact activity and last-contact dates from archived messages. + +```bash +msgvault activity build +msgvault activity build --backstop +``` + +The normal build resumes from its watermark. `--backstop` rescans the complete +archive. `msgvault serve` also runs this projection on the schedule configured +under `[activity]`. --- @@ -2143,8 +3013,11 @@ msgvault embeddings <subcommand> [flags] | `build` | Build or update the index. Incremental by default; `--full-rebuild` starts a new generation. | | `resume` | Continue scan-and-fill embedding for the building or active generation. Incremental by default; `--backstop` also scans below the watermark. | | `list` | List index generations with their state, model, dimension, and pending count. | +| `optimize [generation-id]` | Build or resume the local SQLite search accelerator, or remove it with `--drop`. | +| `prune` | Remove embeddings for hard-deleted messages. | | `activate <generation-id>` | Activate a completed building generation, retiring the current active one. | | `retire <generation-id>` | Retire a generation. | +| `prune` | Remove embeddings whose messages were hard-deleted. | ### embeddings build @@ -2189,7 +3062,36 @@ msgvault embeddings resume --backstop msgvault embeddings list ``` -Print one row per index generation: ID, state (`building`, `active`, or `retired`), model, dimension, embedded message count, pending count, fingerprint, and the start, completion, and activation timestamps. +Print one row per index generation: ID, generation state, model, dimension, +coverage, accelerator state and row count, accelerator timestamps and last +error, fingerprint, and generation timestamps. + +### embeddings optimize + +```bash +msgvault embeddings optimize [generation-id] [--drop] +``` + +Build or resume the SQLite approximate-search accelerator from vectors already +stored for a generation. The active generation is used when the ID is omitted. +The command never calls the embedding provider. It is safe to interrupt and +rerun; the accelerator is not used for search until verification and atomic +publication complete. The daemon's operation gate pauses scheduled embedding +and other gated writes until the command finishes. Searches remain available. +PostgreSQL does not need this command. + +Use `--drop` to remove the accelerator and its build state, including for a +retired generation. Exact vectors remain intact. Freed database pages become +reusable; the database file does not shrink. Re-run optimization after substantial +archive growth to retrain its fixed bucket count. + +### embeddings prune + +```bash +msgvault embeddings prune +``` + +Remove stored message embeddings whose source messages were hard-deleted. ### embeddings activate @@ -2217,10 +3119,162 @@ Mark a generation as retired. Retiring the active generation requires `--force-a | `--yes` | Skip the confirmation prompt. | | `--force-active` | Allow retiring the generation that is currently active. | +### embeddings prune + +```bash +msgvault embeddings prune +``` + +Remove vector rows whose source messages no longer exist. The configured +vector backend must support orphan pruning. + `msgvault build-embeddings` remains as a deprecated alias for `msgvault embeddings build` (same `--full-rebuild` and `--yes` flags). --- +## multimodal + +Build, inspect, and search the optional visual attachment index. The workflow +requires `[vector.multimodal]` configuration, a probed capability manifest, +and explicit consent for hosted processing. See [Visual attachment +search](usage/vector-search.md#visual-attachment-search). + +| Subcommand | Purpose | +|---|---| +| `probe --seeds <dir> --out <file> --yes` | Send synthetic fixtures to the configured provider and write a capability manifest. `--fixtures` keeps the generated fixtures instead of using a temporary directory. | +| `build --yes` | Record consent for the configured capability profile and build the visual index. | +| `resume` | Continue a consented build. | +| `status [--json]` | Report generation state and coverage. | +| `retry --message <id> --hash <sha256>` | Retry one attachment occurrence. | +| `retire <generation-id> --yes` | Retire one generation and delete its vectors. Original attachments remain archived. | + +Search by text or by one local JPEG, PNG, or WebP image: + +```bash +msgvault multimodal search "a whiteboard timeline" +msgvault multimodal search --image ./reference.png +``` + +| Search flag | Default | Description | +|---|---|---| +| `--image <path>` | — | Use a local query image of at most 20 MiB instead of text | +| `--limit <count>` | `20` | Results to return; must be 1–100 | +| `--cursor <value>` | — | Continue from an opaque result cursor | +| `--sender-person <id>` | — | Attachments sent by one durable person | +| `--person <id>` | — | Attachments related to one durable person | +| `--participant <id>` | — | Attachments related to one observed participant | +| `--direction <value>` | — | `from_person`, `to_person`, or `group`; requires `--person` or `--participant` | +| `--source <id>` | — | Restrict to one source | +| `--message <id>` | — | Restrict to one owning message | +| `--filename <text>` | — | Case-insensitive filename substring | +| `--mime-prefix <text>` | — | Case-insensitive MIME prefix | +| `--after`, `--before` | — | `YYYY-MM-DD` sent-date bounds | +| `--json` | `false` | Emit JSON | + +Supply exactly one text query or `--image`. `--person` and `--participant` are +mutually exclusive. `--sender-person` cannot be combined with either of those +or with `--direction`. + +--- + +## eval + +Compare how well search modes find messages you have rated. The command runs +the same full-text, vector, and hybrid retrieval paths used by production +search. It reports ranking quality, recall, latency, configuration, and input +diagnostics. + +```text +# topics.tsv: qid<TAB>query<TAB>optional-category +q1 quarterly planning pointed +``` + +```text +# qrels.txt: qid iteration document-id relevance (1 or greater means relevant) +# Replace example-message-001 with a real source_message_id, not a local numeric ID. +q1 0 example-message-001 2 +``` + +```bash +msgvault eval \ + --topics topics.tsv \ + --qrels qrels.txt \ + --modes fts,vector,hybrid \ + --limit 100 +``` + +| Flag | Default | Description | +|---|---|---| +| `--topics <path>` | required | Tab-separated topics: `qid`, query, and optional category | +| `--qrels <path>` | required | Whitespace-separated judgments: `qid iteration docid relevance`; relevance of 1 or greater means relevant | +| `--modes <list>` | `fts,vector,hybrid` | Comma-separated modes to evaluate | +| `--doc-key <kind>` | `message` | Match judgments to `message` source IDs or `conversation` source IDs | +| `-n`, `--limit <count>` | `100` | Distinct documents retrieved per query | +| `--json` | `false` | Emit the report as JSON | +| `--rerank-jev <shapes>` | disabled | Opt in to `per-candidate`, `batched`, or both Jev shapes | +| `--rerank-top <count>` | `30` | Between 2 and 30 candidates; cannot exceed `--limit` | +| `--rerank-max-requests <count>` | `1000` | Maximum provider requests for the whole invocation | +| `--rerank-input-usd-per-million <amount>` | required when enabled | Input price supplied for this run | +| `--rerank-output-usd-per-million <amount>` | required when enabled | Output price supplied for this run | + +`eval` opens the archive selected by local configuration directly; it does not +use `[remote]`. Vector and hybrid evaluation currently require a SQLite archive, +an `sqlite_vec` build, enabled vector configuration, and a compatible active +generation. On PostgreSQL, run `--modes fts`. + +### Optional Jev reranking + +`--rerank-jev` adds an evaluation arm after retrieval. It sends the topic query +and up to 30 message candidates to TypeSafe's fixed Jev endpoint. The command +uses `TYPESAFE_API_KEY` from the process environment. Selecting a shape is the +per-invocation consent to send that archived text. The reranked arms accept +`--doc-key=message` because TREC judgments identify messages. Conversation +judgments remain available for the baseline run. + +The command prepares candidate text with the same body selection and cleanup +used for embeddings. It sends only the query and the cleaned subject/body. +Candidate text is capped at 2048 UTF-8 bytes and query text at 4096 bytes. +Msgvault calls TypeSafe through Docbank's `document/typesafe` client, which +sets the request and response size limits and runs at most eight calls at +once. Msgvault gives each ranking 40 seconds for all of its calls, enough for +four waves of 10-second calls at 30 candidates. A ranking's full elapsed time +contributes to the reported latency. The `per-candidate` and `batched` shapes +use the same retrieved messages in one invocation. + +Cost is the returned input and output tokens times the prices supplied on the +command line. Before opening the archive, the command refuses a run whose +worst-case request count, judged topics times modes times requests per +ranking, exceeds `--rerank-max-requests`. Msgvault has no local spend limit, +so before a live study, verify an account or order limit that TypeSafe +enforces by rejecting charges above the cap. A displayed balance or alert does +not establish that behavior. A response without token usage fails that arm. +The first provider failure ends reranking for the run. The report keeps the +baseline and the token subtotals observed before the failure, +`usage_complete=false` marks them as partial, and unknown cost prints `unknown` +in the table and `null` in JSON. Requests and tokens count completed rankings +only. The command returns a nonzero result. + +Example with placeholder prices: + +```bash +TYPESAFE_API_KEY=replace-me msgvault eval \ + --topics topics.tsv --qrels qrels.txt --modes fts,vector,hybrid \ + --limit 100 --rerank-jev per-candidate,batched --rerank-top 30 \ + --rerank-input-usd-per-million 1 \ + --rerank-output-usd-per-million 2 +``` + +Reranked p95 includes retrieval, shared candidate preparation, and that shape's +provider calls. A live TREC Legal 2010 study also needs a permitted matching +message collection, topics, qrels, source-id mapping, and the enforced account +cap. Consider a search integration only when a complete hybrid shape improves +Hit@10 by 0.05 absolute and has end-to-end p95 at or below 2000 ms under the +same corpus, topics, qrels, retrieval settings, and top N of 30. Report the +paired topic denominator and changes for both shapes. The local threaded +fixture checks wiring only. + +--- + ## cache-stats Show statistics about the analytics cache. @@ -2243,16 +3297,42 @@ Run arbitrary SQL against the Parquet analytics cache using an in-memory DuckDB msgvault query <sql> [flags] ``` -If the analytics cache is stale, it is automatically rebuilt before the query runs. +A usable stale cache remains queryable while refresh work runs. The JSON result +includes `cache.published_at`, and includes `stale_reason` and +`pending_additions` when known. If a new publication is required before rows +can be returned, the command reports the build job on stderr, waits for it, +then returns query results. A failed build or interrupted wait exits with an +error. `--fresh` waits for a check that includes archive writes committed before +the request, rebuilding if needed, before returning rows. | Flag | Default | Description | |---|---|---| | `--format` | `json` | Output format: `json`, `csv`, or `table` | +| `--fresh` | `false` | Wait for a freshness check and any required rebuild before returning rows | See [SQL Queries](/docs/usage/querying/) for available views and example queries. --- +## credentials + +On unreleased `main`, manage provider keys on the daemon host without the Web UI: + +```sh +msgvault credentials set <id> --from-file PATH [--endpoint URL] +msgvault credentials set <id> --stdin [--endpoint URL] +msgvault credentials list [--json] +msgvault credentials import-env +``` + +Supply exactly one input source. Supported IDs are `vector.embeddings`, +`vector.multimodal`, `people.enrichment/<name>`, and +`people.enrichment/suppression`. The configured provider supplies the default +endpoint. Suppression keys have no endpoint. Listing prints IDs and bound +origins, never key values. Import copies present configured environment keys +and preserves stored keys. See [stored credentials](configuration.md#stored-provider-credentials) +for input security, runtime precedence, and sweep-provider commands. + ## mcp ### Discover running HTTP listeners @@ -2280,12 +3360,22 @@ msgvault mcp [flags] |---|---|---| | `--force-sql` | `false` | Deprecated in 0.17.0; use `[analytics].engine = "sql"` in `config.toml` instead. See [Configuration: analytics](/docs/configuration/#analytics). | | `--no-sqlite-scanner` | `false` | Deprecated in 0.17.0; cache engine selection is daemon-managed. Use `[analytics].engine = "sql"` for live SQL. | -| `--http` | — | Serve MCP over StreamableHTTP on this address instead of stdio. Bare ports bind to loopback, e.g. `8080` becomes `127.0.0.1:8080`. Non-loopback addresses require `[server].api_key` or `--http-allow-insecure`. | -| `--http-allow-insecure` | `false` | Allow non-loopback HTTP binding without `[server].api_key`. A configured key is still enforced; without one, use only behind a trusted network boundary or authenticated reverse proxy. | -| `--http-allow-writes` | `false` | Expose Saved View management, attachment export, and deletion staging tools over StreamableHTTP. Enable only for trusted, authenticated clients. | +| `--http` | — | Serve MCP over StreamableHTTP on this address instead of stdio. Bare ports bind to loopback, e.g. `8080` becomes `127.0.0.1:8080`. Non-loopback addresses require an effective inbound key or `--http-allow-insecure`. | +| `--http-token-file` | — | On unreleased `main`, read an independent inbound bearer key from an owner-only file; takes priority over `--http-token-env`. Requires `--http`. | +| `--http-token-env` | — | On unreleased `main`, name the environment variable holding an independent inbound bearer key. Requires `--http`. | +| `--http-allow-insecure` | `false` | Allow non-loopback HTTP binding without an effective inbound key. A configured key is still enforced; without one, use only behind a trusted network boundary or authenticated reverse proxy. | +| `--http-allow-writes` | `false` | Expose Saved View management, attachment export, and deletion staging tools over StreamableHTTP. Calendar event mutations also require `--allow-calendar-writes`. Enable only for trusted, authenticated clients. | +| `--allow-calendar-writes` | `false` | Expose calendar event mutation tools. HTTP also requires `--http-allow-writes`; only enable for sessions where the user explicitly authorizes calendar writes. | See [MCP Server](/docs/usage/chat/) for configuration and tool reference. +For Kata person agendas, `get_person_agenda` reads the live open tasks and +returns the person's canonical vCard UID and aliases. Use Kata's MCP tools to +create or change tasks; msgvault's agenda tool is read-only. Use the returned +canonical UID as the scalar `msgvault.person` metadata value and +`msgvault.list` for the list name (`agenda` by default). See +[Kata configuration](configuration.md#integrationskata) for setup and limits. + --- ## skills @@ -2345,7 +3435,7 @@ msgvault daemon restart `start` launches the daemon in the background, `status` reports its recorded URL/PID/version/API schema/uptime, `stop` shuts it down, and `restart` performs a stop followed by a start. Starting a newer compatible binary replaces an older recorded daemon when `[server].daemon_auto_restart = "newer"`; incompatible running daemons are reported with a prompt to stop them first. -The lifecycle commands have no command-specific flags. All configuration (port, bind address, API key, CORS, account schedules, SyncTech SMS sources, background idle timeout, daemon restart policy, and vector embedding schedule) is read from your `config.toml`. See [Web UI & API Server](/docs/api-server/) for endpoint documentation, run `msgvault openapi`, or fetch `/openapi.json` from a running server for the generated OpenAPI contract. See [Configuration](/docs/configuration/#server) for config options. When vector search is enabled, the daemon can also run the embed worker on a cron and/or after every successful sync, see [Configuration: vector.embed.schedule](/docs/configuration/#vectorembedschedule). +The lifecycle commands have no command-specific flags. Configuration comes from `config.toml` with the [runtime environment overrides](configuration.md#environment-variables). See [Web UI & API Server](/docs/api-server/) for endpoint documentation, run `msgvault openapi`, or fetch `/openapi.json` from a running server for the generated OpenAPI contract. See [Configuration](/docs/configuration/#server) for config options. When vector search is enabled, the daemon can also run the embed worker on a cron and/or after every successful sync, see [Configuration: vector.embed.schedule](/docs/configuration/#vectorembedschedule). Background daemons started by `daemon start` or auto-started by a CLI command shut down after `[server].daemon_idle_timeout` with no requests. The default is `20m`; set it to `"0s"` to disable idle shutdown. `MSGVAULT_DAEMON_IDLE_TIMEOUT` can override the value for a lifecycle-managed background daemon. @@ -2365,11 +3455,21 @@ msgvault serve `msgvault serve` stays in the foreground until interrupted and is not idle-stopped. Use it for externally supervised, Docker, and NAS deployments; use `msgvault daemon` for local background lifecycle management. +On unreleased `main`, `--bind ADDRESS` and `--port PORT` override environment, +TOML, and defaults. A port of `0` selects an open port. `--bind iface:NAME` +resolves a named network interface at startup and fails before binding if it +cannot find a usable address. See [runtime configuration](configuration.md#environment-variables) +for environment-only deployment and persisted API keys. + +```sh +msgvault serve --bind 0.0.0.0 --port 8080 +``` + --- ## setup -Run the first-run setup wizard for OAuth and optional remote deployment. +Run the first-run setup wizard. It writes `config.toml`, optionally stores a Google OAuth credential (needed only for Gmail and Google Calendar; press Enter to skip), and optionally configures a remote deployment. ```bash msgvault setup @@ -2378,7 +3478,7 @@ msgvault setup If configured for a remote server, this command generates `<MSGVAULT_HOME>/nas-bundle` with: - `config.toml` ready for container deployment -- `client_secret.json` +- `client_secret.json`, if a Google OAuth credential is configured - `docker-compose.yml` The wizard also stores remote URL/API key in `remote` config block so `export-token` can use it without extra flags. @@ -2389,9 +3489,11 @@ Configure optional search and people features from the available API keys. The command reads `VOYAGE_API_KEY`, `OPENAI_API_KEY`, and the document provider's configured key variable (default `MISTRAL_API_KEY`). For an unset text feature, it chooses Voyage contextual embeddings, then OpenAI embeddings, then an -available loopback Ollama model at `[chat].server`. For inference, it chooses -OpenAI, then the configured local Ollama chat model. These are setup choices; -the running daemon does not fall back to another provider after a failure. +available loopback Ollama model at `[chat].server`. Hosted people inference +requires an explicit `--provider` and `--model`; an OpenAI key alone does not +select it. Without an OpenAI key or explicit provider choice, setup can offer +the configured local Ollama chat model. The running daemon does not fall back +to another provider after a failure. Setup prints a plan, asks once per hosted provider, writes `config.toml`, and prints the remaining commands. It onboards a new people-sweep provider through @@ -2400,10 +3502,14 @@ consents remain separate. Visual search needs a valid probe manifest before setup enables it; document extraction remains manual. See [Recommended Configuration](usage/recommended-configuration.md). -The people sweep stays pending unless `--allow-sensitive` is supplied. This -permits sending sensitive archive excerpts to its inference provider and -inferring sensitive personal attributes. `--yes` alone does not grant this -permission. Vector lanes also stay pending when the binary lacks the backend +Local people-sweep setup stays pending unless `--allow-sensitive` is supplied. +An explicit `--provider` requires `--allow-sensitive=true|false` and explicit +`--retention-posture` and `--training-posture` assertions. Only `true` permits +sending sensitive archive excerpts and inferring sensitive personal attributes; +`false` permits the synthetic check but real sweeps cannot process evidence. +`--yes` alone does not grant this permission. If a sweep is already enabled, +`--provider` is rejected; use `person provider add` and `person provider use` +to choose another profile. Vector lanes also stay pending when the binary lacks the backend required by the configured database; setup prints rebuild guidance. Saved retention and training postures on disabled lanes are preserved unless @@ -2428,6 +3534,10 @@ msgvault setup providers --yes --document-retention zdr --document-training opte | Flag | Default | Description | | ---------------------- | ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--provider` | absent | Explicit people inference preset: `openai`, `openrouter`, or `venice`; requires model and privacy decisions | +| `--model` | absent | Required model ID for the selected preset | +| `--credential-env` | absent | Host environment-variable name for the preset key; mutually exclusive with `--api-key-stdin` | +| `--api-key-stdin` | `false` | Read the preset API key from standard input; otherwise a terminal can prompt for it | | `--yes` | `false` | Accept every provider disclosure without prompting. Required to apply a plan with hosted-provider prompts when stdin is not a terminal or `--json` is used. It does not grant the separate sensitive-data opt-in. | | `--allow-sensitive` | `false` | Allow the people sweep to send sensitive archive excerpts and infer sensitive personal attributes | | `--dry-run` | `false` | Print the plan, the disclosures, and the current lane report without writing | @@ -2473,6 +3583,9 @@ msgvault show-message <id> [flags] |---|---| | `--json` | Output as JSON | +JSON output includes `web_url` when the selected daemon can provide a browser +link for the message. + --- ## list-accounts @@ -2574,16 +3687,17 @@ schema version. The query resolves with the same search semantics as `msgvault search`, and `--dry-run` prints the set that staging would create. -Deletion staging covers Gmail-source email only. A search that also matches -chats, meetings, calendar entries, or mail from non-Gmail sources such as Apple -Mail imports stages the Gmail subset and reports how many items it skipped. Only +Deletion staging covers email from Gmail and Microsoft Graph mail (`msmail`) +sources only. A search that also matches chats, meetings, calendar entries, or +mail from other sources such as IMAP or Apple Mail imports stages the +deletable subset and reports how many items it skipped. Only a search with nothing deletable in it is refused. Legacy Gmail messages imported before message types existed carry a blank type and count as email, so `message_type:email` stages them too. In ID mode, the -daemon resolves live Gmail targets and source boundaries for the requested IDs; -IDs that do not resolve to live deletable Gmail messages with provider message +daemon resolves live targets and source boundaries for the requested IDs; +IDs that do not resolve to live deletable messages with provider message IDs are omitted, so the reported count is the number of targets resolved by the daemon; the CLI does not search, probe FTS readiness, call Explore or preflight, or require @@ -2640,8 +3754,10 @@ msgvault cancel-deletion --all ## delete-staged Execute staged remote deletions. Gmail and IMAP move messages to Trash by -default. `--permanent` uses Gmail batch deletion or IMAP UID EXPUNGE; the -IMAP permanent path requires UIDPLUS. Recovery from Trash depends on the +default, and Microsoft Graph mail moves them to Deleted Items. `--permanent` +uses Gmail batch deletion, IMAP UID EXPUNGE, or Graph `permanentDelete`; the +IMAP permanent path requires UIDPLUS. The first deletion for a Graph mail +account asks to upgrade its token to `Mail.ReadWrite`. Recovery from Trash depends on the provider. See [deletion behavior](usage/deletion.md). ```bash @@ -2651,7 +3767,8 @@ msgvault delete-staged [batch-id] [flags] | Flag | Description | |---|---| | `-y`, `--yes` | Skip confirmation prompt | -| `--permanent` | Permanently delete through Gmail batch deletion or IMAP UID EXPUNGE instead of moving to Trash | +| `--permanent` | Permanently delete through Gmail batch deletion, IMAP UID EXPUNGE, or Graph `permanentDelete` instead of moving to Trash | +| `--headless` | Use device-code sign-in for Microsoft Graph permission upgrades; open the printed URL on another device | | `--dry-run` | Show what would be deleted without deleting | | `-l`, `--list` | List staged deletion batches | | `--account` | Filter to one source by identifier or unique display name | @@ -2776,6 +3893,17 @@ and [backup](usage/backup.md) before applying. Fix UTF-8 encoding issues in existing messages through the configured remote server or local daemon. The command streams the daemon's stdout/stderr back to the terminal, and the daemon serializes the repair with other archive mutations. +For SQLite archives, it also rebuilds the analytics cache. +It reports invalid RFC 822 Message-ID values and leaves their original bytes +unchanged to avoid making distinct identifiers collide. The analytics cache +exports these IDs as NULL. Recover the original values separately from a +verified source before rebuilding the cache. + +It does not repair source message IDs, source identifiers, recorded +sender/recipient envelope addresses, account identity addresses, or participant +identifiers. Those fields need separate recovery from verified original +values; rebuilding the analytics cache alone +cannot recover them. ```bash msgvault repair-encoding @@ -2954,17 +4082,28 @@ can read, never pass it as a flag or environment variable. ```bash msgvault agent-token issue --label <name> \ --permissions draft.create \ - --source-ids <id>[,<id>...] + --source-ids <id>[,<id>...] \ + --sender <source-id>=<address> ``` | Flag | Description | |---|---| | `--label <name>` | (required) Human-readable name for the grant | -| `--permissions <perms>` | Comma-separated permission names to grant; accepted values: `draft.create` | +| `--permissions <perms>` | Comma-separated permissions: `calendar.read` for availability, `calendar.event.read` for provider-derived event details in delegated plans and write receipts, `calendar.write` for calendar mutations, and additional `calendar.invite` for guest changes; `draft.create` for `draft-reply`, `draft-compose`, and `draft-get`; `draft.edit` for `draft-get`, `draft-edit`, and `draft-recover`; `draft.delete` for `draft-get`, `draft-delete`, and `draft-recover` (see [managed drafts](#draft-get-draft-edit-draft-delete-and-draft-recover)) | | `--source-ids <ids>` | Comma-separated source IDs that the permissions apply to | +| `--sender <source-id>=<address>` | Restrict a source to one confirmed sender identity; repeat for multiple choices | The grant is valid until revoked or until the daemon restarts. +When `--sender` is omitted for a selected source, issuance snapshots every +currently confirmed valid mailbox identity. Sender selections are stored as +canonical mailbox keys and remain fixed until the token is revoked. Adding an +alias later does not expand an existing grant. A source with no selected sender +has no delegated draft sender authority. Calendar grants use exact calendar source +identities and do not require a draft sender selection. Delegated callers can run +`calendar` commands and `mcp` over stdio; the delegated MCP bridge exposes only +calendar tools. + The response includes the daemon address, the secret, and the granted source references. Pass `--agent-url <address>` and the file path to `--agent-token-file` when invoking delegated commands. The address comes from the issuing request. On a default local install it is an @@ -2982,7 +4121,8 @@ Agent-token commands return an error when `server.agent_access` is disabled. msgvault agent-token list ``` -Each row shows the grant ID, label, permissions, sources (as `id/type/identifier`), and creation time. +Each row shows the grant ID, label, permissions, sources (as +`id/type/identifier`), frozen sender keys, and creation time. ### agent-token revoke diff --git a/docs/configuration.md b/docs/configuration.md index fc9e2c7ab..484694ebd 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,5 +1,5 @@ --- -last_edited: "2026-09-15" +last_edited: "2026-10-05" title: Configuration description: Configuration file reference, environment variables, and file locations. --- @@ -54,12 +54,53 @@ values remain in effect. Provider keys alone do not enable processing. and separate query consent. - [Vector search](usage/vector-search.md): text, person, and visual indexes. +## People identity scoring + +Identity scoring is disabled by default. An operator starts each batch and +consents to the exact disclosure shown by `msgvault person scoring status`. +Scoring creates review suggestions and records judgments; it never accepts +matches or links participants. + +```toml +[people.identity_scoring] +enabled = false +model_id = "jev-1.13.0" +minimum_probability = 0.80 +credential_env = "MSGVAULT_JEV_API_KEY" +batch_size = 20 +retention_declaration = "provider retention policy accepted by the operator" +``` + +| Key | Default | Description | +|---|---|---| +| `enabled` | `false` | Enable consented identity scoring. Consent is still required. | +| `model_id` | `jev-1.13.0` | Fixed provider model identifier. | +| `minimum_probability` | `0.80` | Probability must be strictly greater than this threshold before local policy can propose acceptance. Accepted values are at least `0.80` and less than `1.00`. | +| `credential_env` | empty | Name of the environment variable holding the provider key. The key value is read from the daemon environment and is never stored in `config.toml`. Required when enabled. | +| `batch_size` | `20` | Default and maximum scoring batch size, from 1 through 100. | +| `retention_declaration` | empty | Operator's exact declaration of the provider retention policy. Required when enabled and included in the consent fingerprint. | + +The provider endpoint is fixed at `https://api.typesafe.ai/v1/systemone`, with +model `jev-1.13.0`. The disclosure binds that endpoint, model, packet schema, +retention declaration, policy version, and question version. A change to any +of them requires consent to the new fingerprint. `status` also prints the raw +identity fields and limits covered by the packet schema. + +`msgvault person scoring revoke <fingerprint>` withdraws consent for that +disclosure. It leaves the configuration enabled and retains prior judgments. +Set `enabled = false` to disable scoring in the configuration. See +[identity scoring](usage/people.md#optional-identity-scoring) for the workflow +and [the API reference](api-server.md#identity-match-review-and-scoring) for +endpoints. + ## People sweep inference People sweeps use one named protocol profile at a time. A profile records the exact endpoint, model, wire protocol, negotiated output mode, privacy posture, -and source scope. It is configuration, not a provider preset. Msgvault never -changes the active profile or switches providers automatically. +and source scope. Built-in OpenAI, OpenRouter, and Venice presets bind the +protocol, endpoint, and authentication scheme; you still choose the model and +privacy policy. Msgvault never changes the active profile or switches providers +automatically. ```toml [people.sweep] @@ -98,7 +139,8 @@ Codex app-server profiles section below. Onboarding negotiates and saves also save either `max_completion_tokens` or `max_tokens`; the other protocols use their defined token-limit field. -These are examples of protocol profiles, not built-in presets: +Other providers use explicit protocol profiles; OpenRouter, Venice, and OpenAI +also have built-in presets: | Example profile | Protocol | Typical profile choice | |---|---|---| @@ -121,7 +163,12 @@ their provider terms. Credentials are not stored in this TOML. `credential = "stored"` keeps a profile-specific secret under the private tokens directory and is supported on Linux and macOS only; `credential = "env"` stores only the selected -environment-variable name and works everywhere. +environment-variable name and works everywhere. Environment-variable names are +host-only settings: configure them through the CLI or TOML, not the Web UI. +On hosts without stored-key support, the Web UI hides profile enrollment and +key fields. Run [`msgvault person provider add`](cli-reference.md#person-provider-add) +with `--credential-env` on the daemon host, then reload the Web settings to +check and select the profile. `credential = "none"` is restricted to credentialless local or Codex paths. Changing a credential value does not change the profile fingerprint, but changing its source or reference does. @@ -149,7 +196,7 @@ same provider and schedule, with separate enrollment and interval controls. | Key | Default | Description | | ------------------------ | ------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `enabled` | `false` | Run the scheduled people sweep with the selected provider. | -| `provider` | `default` | Name of a table under `[people.sweep.providers]`. The initial profile has an OpenAI endpoint but no model; it is not a usable, consented provider. Setup creates and selects `openai` or `ollama`. | +| `provider` | `default` | Name of a table under `[people.sweep.providers]`. The initial profile has an OpenAI endpoint but no model; it is not a usable, consented provider. Setup can explicitly select `openai`, `openrouter`, or `venice`, or configure local `ollama`. | | `schedule` | `15 2 * * *` | Daily at 02:15 in the daemon's time zone. An omitted or empty value receives this default; use `enabled = false` to disable the sweep. | | `work_batch_size` | `25` | Tracked people considered in one worker batch. | | `historical_message_cap` | `2000` | Maximum archived messages considered when finding context for each profile field. | @@ -158,12 +205,14 @@ same provider and schedule, with separate enrollment and interval controls. | `evidence_max_items` | `200` | Item limit for an evidence packet. | | `backstop_interval` | `24h` | Interval before checking tracked people for changes missed by incremental work. | -`setup providers --allow-sensitive` uses `gpt-5.6-luna` with `medium` reasoning -when an OpenAI key is present, or the configured local Ollama chat model -otherwise. It preserves an existing active profile and never switches after a -request failure. Without `--allow-sensitive`, setup leaves inference pending: -the same profile flag controls both sensitive archive evidence and sensitive -attribute targets. +Hosted people inference requires an explicit `setup providers --provider +<openai|openrouter|venice> --model <model>` choice, a credential source, and +explicit retention, training, and sensitive-content decisions. An +`OPENAI_API_KEY` alone configures only eligible embedding lanes. With no OpenAI +key or explicit provider choice, setup can offer the configured loopback Ollama +chat model when `--allow-sensitive` is supplied. An enabled sweep is preserved; +`--provider` then fails with instructions to use `person provider add` and +`person provider use`. See [setup flags](cli-reference.md#setup-providers). ### `[people.sweep.budgets]` @@ -231,7 +280,17 @@ The `codex_app_server` protocol is not usable in this release. Its transport stays unavailable until the executable isolation gate releases a verified build, and until then every Codex operation fails closed with `codex app-server isolation is not released`. The profile shape is documented -here so the configuration is ready when the gate ships. +here so the configuration is ready when the gate ships. Codex sign-in and +model routes return HTTP 503 before changing credentials or consent. The +terminal-only [`person provider enroll-codex`](cli-reference.md#person-provider-enroll-codex) +command creates a new profile through the daemon; host-side `person provider +login` reauthenticates the selected existing Codex profile. Both remain gated. + +The Linux launcher disables Codex's local execution environment and shell +tools. The app server can manage its staged OAuth credential, but its command +and filesystem interfaces cannot access it. Command-execution requests or +events abort inference. Enabling a release still requires a real authenticated +structured-inference check through this launcher. `codex_app_server` profiles are also the one protocol `person provider add` cannot create: generic onboarding negotiates HTTP capabilities through an @@ -348,8 +407,21 @@ manifest explicitly and displays its upload and cost preflight before requiring `--yes`. When document indexing is enabled, the daemon's weekly reconciliation and local derivative cleanup remain automatic and make no provider requests. -`[attachments.documents.scope]` accepts `message_types`; an empty list includes -all supported standalone attachment sources. The first release requires +`[attachments.documents.scope]` accepts these fields: + +| Field | Default | Meaning | +|---|---|---| +| `message_types` | `[]` | Include all supported message sources, or restrict extraction to the listed types | +| `include_inline` | `false` | Also include inline attachments with an authorized document media type and authoritative role provenance | + +Inline scope support is available in v0.21.0. Some mail clients mark +ordinary document attachments as inline. Set `include_inline = true` to include +them; other roles remain excluded. This changes the consent fingerprint. Run +`msgvault documents consent-mistral --capabilities <manifest> --yes` again before +building. Selecting a standalone-only profile stops inline search results from +serving, including results extracted under an earlier profile. + +The first release requires `[attachments.documents.index].lexical = true` and `store_chunk_text = true`. Hosted document embeddings are not enabled by this configuration. @@ -391,10 +463,21 @@ provider behavior. When `service_account_key` is configured, `msgvault add-account <email>` validates the delegated Gmail profile and registers the account without storing a per-user refresh token. The service account key file must be owner-only on Unix-like systems, for example `chmod 600 /path/to/service-account.json`. -### `[carddav]` +### `[carddav]` and `[carddav_connections.<name>]` {#carddav} + +`[carddav]` is the `default` connection. Add named tables for other accounts; +each uses the keys below and has its own credential binding, discovery, +retry state and schedule. Names use 1–64 lowercase ASCII letters, digits, +underscores or hyphens, starting with a letter. `default` is reserved for +`[carddav]`. Connect through the [CardDAV account workflow](usage/people-carddav.md) so the -daemon validates discovery before saving these settings. +daemon validates discovery before saving these settings. The same `base_url` +and `username` cannot belong to two connections, including disabled connections. +For Google, account email is case insensitive and the OAuth app does not create +a separate CardDAV account. Config edits and account saves reject duplicates. +Removing a config table retains the account's archive data; see +[recovering an orphaned connection](usage/people-carddav.md#recover-a-connection-removed-from-config). | Key | Default | Description | |-----|---------|-------------| @@ -404,6 +487,29 @@ daemon validates discovery before saving these settings. | `username` | `""` | Server username or Google account email | | `schedule` | `""` | Cron schedule; empty disables scheduled sync | | `enabled` | `false` | Enable the configured connection | +| `trusted_origin` | `""` | Exact HTTPS origin approved for private access, including its port; a trailing `/` is accepted. Applies only when it matches the account URL's origin. | +| `trusted_addresses` | `[]` | Private IP addresses to dial for `trusted_origin`, without DNS. Accepts `10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`, `100.64.0.0/10`, and `fc00::/7`; rejects duplicates, IPv6 zones, loopback, and link-local addresses. | + +For example, a second connection uses: + +```toml +[carddav_connections.work] +base_url = "https://contacts.example.com/dav/" +username = "you@example.com" +enabled = true +schedule = "0 */6 * * *" +``` + +Use `add-carddav --connection work` or **Add connection** in Settings to save +its credential and discover books. Passwords are rejected in these TOML tables. +The default binding remains `tokens/carddav.json`; named bindings use +`tokens/carddav-connections/<name>/carddav.json`. Google authorizations are +shared by account email and OAuth app, separately from these connection bindings. +Scheduler job names are `carddav` for default and `carddav:<name>` otherwise. + +Set both trusted-destination keys together. See the +[private-server setup](usage/people-carddav.md#private-servers) for an example, +restart requirements, and behavior when the origin does not match. Passwords and Google tokens stay in the configured token directory, outside `config.toml`. See [Google Contacts setup](usage/people-carddav.md#google-contacts) @@ -539,7 +645,7 @@ Use `msgvault logs` to view and tail log files from the selected local or remote | Key | Default | Description | |---|---|---| -| `rate_limit_qps` | `5` | Gmail API requests per second | +| `rate_limit_qps` | `5` | Scales Gmail's local quota-unit refill rate: `5` allows 250 units/second; `3` allows 150. Gmail values above `5` are capped at `5`. Also sets Microsoft Teams Graph requests/second, without that cap, so lowering it slows Teams imports too. Reduce it if Gmail reports quota errors; Google's [project quotas](https://developers.google.com/workspace/gmail/api/reference/quota) can be lower than this local budget. | | `archive_remote_images` | `false` | Download remote email images during Gmail/IMAP sync and EML, EMLX, MBOX, and PST imports | | `trusted_imap_sent_mailboxes` | `{}` | Per-IMAP-account Sent-folder names (keyed by the ACCOUNT identifier from `msgvault list-accounts`) that enable edited-copy snapshot refresh for servers without advertised special-use roles | @@ -590,9 +696,11 @@ Settings for the Web UI and API server started by `msgvault serve`. The same HTT | Key | Default | Description | |---|---|---| | `api_port` | `0` (auto-select) | Port the server listens on; `0` picks an open port at startup and clients discover it automatically. Set a fixed port for remote/NAS deployments. | -| `bind_addr` | `127.0.0.1` | Bind address | +| `bind_addr` | `127.0.0.1` | Bind address, or `iface:NAME` to bind an address on a named interface | | `api_key` | — | API key for daemon/API authentication and bearer authentication on `msgvault mcp --http` | -| `agent_access` | `false` | Enable restricted agent grants; requires `api_key` to be non-empty. Read at daemon startup only; a `config.toml` edit takes effect only after a restart. | +| `api_key_file` | — | Owner-only file holding the API key | +| `api_key_env` | — | Name of an environment variable holding the API key | +| `agent_access` | `false` | Enable restricted agent grants; requires an effective API key. Read at daemon startup only; a `config.toml` edit takes effect only after a restart. | | `allow_insecure` | `false` | Allow non-loopback binding without `api_key` | | `cors_origins` | `[]` | Allowed CORS origins | | `cors_credentials` | `false` | Allow credentials in CORS requests | @@ -600,19 +708,75 @@ Settings for the Web UI and API server started by `msgvault serve`. The same HTT | `trusted_proxies` | `[]` | IP addresses or CIDRs allowed to supply forwarded HTTPS/host headers | | `daemon_idle_timeout` | `20m` | Idle timeout for lifecycle-managed background daemons; set to `"0s"` to disable | | `daemon_auto_restart` | `newer` | Local daemon restart policy when the CLI finds a different daemon binary version: `newer`, `never`, or `always` | +| `daemon_auto_start` | `true` | Let CLI, TUI, and MCP commands start a local background daemon when none is running; set `false` when a supervisor runs `msgvault serve` | `daemon_idle_timeout` applies only to background daemons started by `msgvault daemon start` or auto-started by a CLI command. Foreground `msgvault serve` keeps running until stopped. `MSGVAULT_DAEMON_IDLE_TIMEOUT` overrides the configured value for lifecycle-managed background daemons. +On unreleased `main`, flags and environment variables can configure the server +without `config.toml`. `serve --bind` and `serve --port` take priority over +environment variables, then TOML, then defaults. `iface:NAME` resolves at +startup, preferring a usable IPv4 address and otherwise IPv6. An unknown, down, +or unaddressed interface fails before opening a listener. Startup logs report +the bound address and whether the bind came from a flag, environment, a config +file, or the default. + +Config edits validate the saved settings without resolving network interfaces +or reading server credentials. These resources must be available when the +server starts; an unavailable interface or key does not block unrelated edits. + +Credentials use `api_key`, then `api_key_file`, then `api_key_env`. A selected +file or named variable that is missing, empty, or unsafe fails without trying +another source. Files must be regular, owned by the process user, at most +64 KiB, and readable only by that user (`0400` or `0600` on Unix). Symlinks are +rejected. Windows files require an owner-only ACL. File reads trim surrounding +whitespace and leave mounted permissions unchanged. Relative secret paths in +TOML or environment variables resolve beside `config.toml`, including before +the default file exists. With `--config`, they resolve beside the selected file. +Saving configuration preserves the original credential path strings, so relative +paths continue to work when the configuration directory moves. + +Container secret mounts must meet these same rules. Docker Swarm can set the +secret's `uid` to the process user and its `mode` to `0400`; see +[Swarm secret options](https://docs.docker.com/reference/cli/docker/service/create/#create-a-service-with-secrets---secret). +The default root-owned `0444` mount is rejected. With Docker Compose file +mounts, set the host file's ownership and mode before mounting it. +Kubernetes Secret volumes use symlinks and are not accepted directly. Use a +Secret-backed environment variable, or provide a regular owner-only file. +These restrictions apply to `api_key_file`, MCP's `--http-token-file`, and +`credentials set --from-file`. `credentials set --stdin` reads a stream and +can import a readable mounted secret through shell input redirection. + +When a secure non-loopback server has no configured credential source, it +creates `<data_dir>/tokens/server-api-key` with an unpredictable key and +owner-only permissions. Persist `data_dir` to retain the key across restarts. +It reuses that key on later starts, including loopback-only starts. The Web UI +then requires login on `127.0.0.1` too; use the key from that file. An invalid +existing key fails instead of being replaced. Local CLI clients discover the +persisted key, including when they start the daemon themselves. +`allow_insecure = true` skips this default key and retains the explicit +unauthenticated mode; explicitly configured keys are still enforced. Startup +logs name the credential file without printing its contents. + +Run local CLI commands as the daemon's operating-system user and provide the +same selected credential sources. A different user, including root through +`sudo` or `docker exec`, fails the key file's ownership check. For a container, +use `docker exec --user <daemon-uid> ...`. If `api_key_env` names a variable +provided only to the supervised daemon, also provide it to the CLI process; +the CLI does not inherit the daemon's environment. + `daemon_auto_restart = "newer"` replaces an older compatible local daemon with the current CLI binary. Use `"never"` when another supervisor owns the daemon lifecycle, or `"always"` to restart whenever the recorded daemon version differs. Remote servers are never auto-restarted by a CLI client. +`daemon_auto_start = false` is for installs where a supervisor such as launchd, systemd, or Docker runs `msgvault serve`. Local archive commands then use the daemon that is already running, wait for one that is still starting, and otherwise fail with an error instead of starting their own. They also never replace a running daemon, whatever `daemon_auto_restart` says, because the supervisor owns restarts. `msgvault daemon start`, `msgvault daemon restart`, and the restart after `msgvault update` still start a daemon when you run them. Commands routed to `[remote].url` are unaffected. + Browser sessions are additive to API-key authentication. Existing CLI and programmatic clients continue to send the configured key. For remote browser access, terminate TLS at a reverse proxy and list that proxy—not arbitrary clients—in `trusted_proxies`. See [Web UI](/docs/web-ui/) for the complete security model and the plain-HTTP warning. -For MCP Streamable HTTP, send `[server].api_key` as `Authorization: Bearer -<key>` on every `/mcp` request. This inbound credential is independent of +For MCP Streamable HTTP, send the effective `[server]` key, or the key selected +by `--http-token-file` or `--http-token-env`, as `Authorization: Bearer <key>` +on every `/mcp` request. This inbound credential is independent of `[remote].api_key`, which authenticates `msgvault mcp` when it connects to a remote daemon. @@ -638,7 +802,8 @@ effect only after restart, which also invalidates browser sessions. ### `[integrations.tasks]` -Optional provider-neutral task integration: +Optional provider-neutral integration for message-to-task links. Person agendas +use the separate Kata connection below. | Key | Default | Description | |---|---|---| @@ -652,6 +817,49 @@ the required idempotency and compare-and-swap capabilities; the UI distinguishes disabled, authentication required, incompatible, partial, stale, unavailable, and ready states. +### `[integrations.kata]` + +Optional live person agendas backed by Kata. Tasks stay in Kata; msgvault shows +their current state when you open a person's agenda. This integration is built +against Kata v0.18.0 and requires Kata API schema version 0.21.0 or later. + +| Key | Default | Description | +|---|---|---| +| `enabled` | `false` | Enable Kata person agendas | +| `endpoint` | — | Required when enabled: an explicit HTTPS URL, loopback HTTP URL, or Unix socket URL | +| `api_key` | — | Bearer credential sent by the daemon to Kata; Settings returns only its configured state and a masked hint | +| `default_project` | `msgvault` | Existing active Kata project used for person agendas | + +Create the project in Kata, then configure its endpoint and credential on the +machine running the msgvault daemon: + +```toml +[integrations.kata] +enabled = true +endpoint = "https://kata.example.com" +api_key = "replace-with-your-kata-api-key" +default_project = "msgvault" +``` + +Restart the msgvault daemon after saving. These settings are also editable in +Settings and take effect after restart. Changing the endpoint to a different +origin in Settings clears its saved key unless you provide a replacement key +in the same save. Remote plaintext HTTP is rejected. Kata does not use local +endpoint discovery, and `[integrations.tasks]` does not configure person agendas. + +Each Kata task can belong to one person. The scalar metadata value +`msgvault.person` is the person's canonical vCard UID, not their numeric +msgvault person ID. `msgvault.list` names its list and defaults to `agenda`. +Reads and unlink operations also recognize the person's UID aliases after a +merge. + +Agendas show open tasks only, with at most 100 returned items. A `truncated` +response means more remain; open Kata to see them. The transport also limits +each response to 1 MiB and reports an error when it exceeds that limit. +Create, link, move between lists, and unlink tasks through msgvault; edit task +content or priority, complete tasks, and reopen them in Kata. See +[person agenda commands](cli-reference.md#person-agenda). + ### `[analytics]` Settings for daemon-side aggregate query behavior. The Web UI, TUI, MCP server, and aggregate list commands use these settings through the local daemon or a configured remote server. @@ -659,30 +867,77 @@ Settings for daemon-side aggregate query behavior. The Web UI, TUI, MCP server, | Key | Default | Description | |---|---|---| | `engine` | `auto` | Aggregate engine: `auto` starts with live SQL and switches to DuckDB after cache maintenance succeeds; `sql` always uses live SQL; `duckdb` requires a usable Parquet cache | -| `auto_build_cache` | `true` | Build a stale or missing Parquet cache during daemon startup and after scheduled syncs; `false` skips both automatic paths | -| `min_rebuild_interval` | `0s` | Minimum age of a usable cache before a scheduled sync may rebuild it; zero preserves rebuilding after each sync | -| `builder_memory_limit` | `2GB` | DuckDB memory limit for cache builds, such as `4GB` or `512MiB` | +| `auto_build_cache` | `true` | Refresh a stale or missing Parquet cache automatically at startup, after scheduled or manual syncs, and when a query finds it due; `false` skips automatic builds. An explicit `query --fresh` or sync `--build-cache` can still request one | +| `min_rebuild_interval` | `0s` | Minimum age of a usable cache before a sync, query, or daemon restart may queue an automatic rebuild. Queries serve the committed snapshot during the interval | +| `builder_memory_limit` | `2GB` | DuckDB buffer-manager budget for cache builds, such as `4GB` or `512MiB`; total process memory can exceed it | | `builder_threads` | min(CPUs, 2) | DuckDB threads for cache builds; zero keeps the default | | `builder_temp_limit` | `32GB` | Maximum spill-to-disk size for cache builds | | `query_memory_limit` | `512MB` | DuckDB memory limit for daemon aggregate queries; raise it on a large archive | | `query_threads` | min(CPUs, 4) | DuckDB threads for daemon aggregate queries; zero keeps the default | | `query_temp_limit` | `2GB` | Maximum spill-to-disk size for daemon aggregate queries; a query that spills past it fails with a DuckDB out-of-memory error | +If a Web UI query runs out of memory or temporary disk space, its error names +the query-limit settings above. Try filters that narrow the results. On the +machine running msgvault, check available memory and free disk space before +raising these limits in `config.toml`. Restart the daemon to apply the change, +then retry the query. The limits cap resource use; they do not reserve memory +or disk space. Cache builds have separate `builder_*` limits. + +Start with the builder defaults, including at most two threads. More threads +can increase peak memory use. DuckDB's memory budget covers its buffer manager; +native allocations and Go memory add to the process total. A `24GB` budget +therefore does not guarantee that a build stays below 24 GB of resident memory. +Leave room for the daemon, syncs, other applications, and those allocations. +For a constrained host, lower `builder_memory_limit` and set +`builder_threads = 1` before raising the memory budget. See +[DuckDB's memory guidance](https://duckdb.org/docs/stable/guides/performance/oom). + +Builders spill temporary work under the cache staging directory, beside the +analytics cache. Check free space on that filesystem before increasing +`builder_temp_limit`. The default allows up to `32GB` of spill in addition to +the existing cache and the new generation being staged. When DuckDB's SQLite +scanner is unavailable, the CSV fallback also writes a temporary copy of the +source tables beside the database, or in the system temporary directory if that +fails. A larger disk budget can help a large archive finish with a smaller +memory budget; it does not reserve space. + The daemon starts HTTP health and API routing before analytics cache maintenance. With `engine = "duckdb"`, analytics remain unavailable until a -usable cache is ready; if the cache cannot be built or opened, `msgvault serve` -fails instead of silently falling back. With `auto_build_cache = false`, use -`msgvault build-cache` for explicit cache maintenance. Deprecated in 0.17.0: +usable cache is ready. If no usable cache can be built or opened, `msgvault serve` +fails instead of silently falling back. A failed automatic refresh keeps serving +the last usable publication. With `auto_build_cache = false`, use +`msgvault build-cache`, `query --fresh`, or sync `--build-cache` for explicit +cache maintenance. Deprecated in 0.17.0: per-command analytics flags such as `msgvault tui --force-sql`, `msgvault mcp --force-sql`, `msgvault tui --no-cache-build`, and `--no-sqlite-scanner` were replaced by this daemon-level section. Use `engine = "sql"` to force live SQL. -`min_rebuild_interval` limits only automatic post-sync rebuilds. A busy archive -can therefore serve Parquet analytics that lag SQLite by approximately the -configured interval plus cache build time. Explicit `msgvault build-cache` -requests, startup maintenance, query-required builds, and recovery of an -absent, interrupted, incompatible, or otherwise unusable cache are not delayed. +`min_rebuild_interval` limits automatic refreshes requested by syncs, queries, +and daemon startup. A restart or sync within the interval leaves the existing +publication in service and schedules a rebuild check when the interval ends. +A busy archive can therefore serve Parquet analytics that lag SQLite by +approximately the configured interval plus cache build time. Deleted messages +can remain visible in analytics query results until the next cache publication, +including while the interval has not elapsed and while a build runs. A zero +interval still allows stale rows during the build. Use `query --fresh` to wait +for analytics that include deletions committed before the request. + +Explicit refresh requests and recovery of an absent, interrupted, incompatible, +or otherwise unusable cache are not delayed. + +The daemon runs automatic rebuilds in the background, outside the scheduled +sync that requested them, so syncs keep their cadence while a build runs. A +sync that finishes during a build does not discard it: the build publishes +one consistent read snapshot with that snapshot's counters and message boundary. +The next build appends later messages and repairs journaled child-row changes. +Participant links can refresh relationship data while retaining existing +message shards, including when new messages arrive in the same sync. Changes +to baked message facts, account identities, deletions, and failed syncs still +require a full rebuild. Archives without the child-row repair journal use a +conservative full rebuild after an overlapping sync. The published snapshot +remains usable across daemon restarts, and automatic follow-up builds still +honor `min_rebuild_interval`. Cache build memory and temporary disk usage scale with archive size, so a minimum interval can prevent repeated archive-scale work when sources sync frequently. Changes under `[analytics]` take effect after the daemon restarts. @@ -707,10 +962,18 @@ When set, archive-access CLI commands use the remote server by default. Without |---|---|---| | `url` | — | Remote API base URL (e.g. `http://nas-ip:8080`) | | `api_key` | — | API key used by remote commands | +| `api_key_file` | — | Owner-only file holding the remote API key | +| `api_key_env` | — | Name of an environment variable holding the remote API key | | `allow_insecure` | `false` | Allow HTTP remote connections | Affected CLI commands include `search` (FTS mode), `query`, `show-message`, `stats`, `list-accounts`, `list-senders`, `list-domains`, `list-labels`, `identity` subcommands, `collection` subcommands, `export-eml`, `export-attachment`, `export-attachments`, and `tui`. +The same settings route `mcp` to a remote daemon. Secret precedence and file +requirements match [server credentials](#server). `--local` ignores the remote +destination and its secret sources. An unused destination's secret is not read. +Runtime keys from files or environment variables are never copied into saved +`api_key` fields by `setup` or `export-token`. + ### `[[accounts]]` Scheduled sync sources for the web server. Each `[[accounts]]` entry defines a @@ -768,6 +1031,9 @@ oauth_app = "" # optional named OAuth app calendars = [] # optional calendarId filter; empty = owner+writer schedule = "0 */6 * * *" # 5-field cron, no seconds enabled = true +write_calendars = [] # explicit IDs; empty denies every event write +invite_calendars = [] # subset allowed to change guests or notify them +# calendar_aliases = { team = "team@example.com" } ``` | Key | Default | Description | @@ -777,7 +1043,18 @@ enabled = true | `oauth_app` | — | Named Google OAuth app to use | | `calendars` | — | Specific calendar IDs to sync; empty syncs owned/writable calendars | | `schedule` | — | Cron expression used by `msgvault serve` | -| `enabled` | `false` | Whether the source is daemon-scheduled | +| `enabled` | `false` | Enable the source for scheduled sync and live control; scheduling also requires `schedule` | +| `write_calendars` | `[]` | Exact live calendar IDs allowed for event writes; empty denies all writes | +| `invite_calendars` | `[]` | Exact IDs allowed to change guests, respond, or notify them; writes still require `write_calendars` | +| `calendar_aliases` | `{}` | Names mapped to exact calendar IDs for live control; aliases do not expand permissions | + +Event control also requires [write consent](usage/calendar.md#control-events-unreleased). +`calendars` selects sync targets; it does not grant write authority. `email` selects +the OAuth token, while `write_calendars` selects the calendar that owns the event. +For example, `person@example.com` can create on `team@example.com` when Google +currently reports `owner` or `writer` for that calendar. `primary` resolves to the +live primary calendar ID before the daemon checks policy. List the actual ID in +both permission lists; neither list supports wildcards or alias names. ### `[beeper]` @@ -818,6 +1095,46 @@ max_media_mb = 250 # per-attachment download cap (MiB) | `max_media_mb` | `250` | Per-attachment download cap in MiB (over-cap media is recorded as a `size_cap` skip and retried only after the cap changes) | | `accounts_config` | — | Per-accountID `media` and `max_media_mb` overrides | +#### Send stored audio to Docbank + +The daemon can send stored WAV and MP3 audio from any captured source, including +messaging and email imports, to a separately running Docbank media service. The +service needs Docbank's media HTTP routes. See +[Send audio to Docbank](/docs/usage/beeper/#send-audio-to-docbank) for the +capture and processing rules. + +```toml +[integrations.docbank] +enabled = true +url = "http://127.0.0.1:8080" # your Docbank daemon; the port is an example +api_key_env = "DOCBANK_API_KEY" # daemon environment variable with the key +all_sources_upload_consent = true # allow audio from every captured source to leave msgvault +# asr_profile = "asr" # optional Docbank profile for audio without source text +``` + +| Key | Default | Description | +|---|---|---| +| `enabled` | `false` | Schedule the stored-media job in `msgvault serve` | +| `url` | — | Docbank base URL: HTTPS, or HTTP on a loopback address. User info, query strings and fragments are rejected | +| `api_key_env` | — | Name of the daemon environment variable that holds the Docbank API key. It is read for each request and sent as `X-Api-Key` | +| `api_key` | — | Inline Docbank API key; takes priority over file and environment sources | +| `api_key_file` | — | Owner-only Docbank key file, read for each request; takes priority over `api_key_env` | +| `all_sources_upload_consent` | `false` | Allow stored audio and explicit source transcripts from every captured source, including future providers, to be sent to `url`. Without it the job only records local state | +| `asr_profile` | — | Optional Docbank processing profile for stored audio without usable source text. An empty value retains audio without requesting processing. Msgvault rejects `supplied-transcript`, which Docbank reserves for supplied transcript input. | + +The former Beeper-only `upload_consent` setting no longer enables uploads. +Existing users must explicitly set `all_sources_upload_consent = true` to resume +sending audio, including Beeper recordings. + +The daemon reads these settings at startup, so restart it after a change. A new +`url` starts a separate delivery record; earlier rows stay. Disabling the route +stops the job and keeps its rows. A failed setup, such as an invalid `url`, +does the same and logs a warning. `all_sources_upload_consent` covers transport only; the +Docbank daemon's processing consent still decides whether a configured profile +may run. The route inspects stored CAS bytes, so MIME claims do not expand +Docbank's WAV and MP3 capability. Capture gaps and unsupported formats remain +typed local states. + ### `[slack]` Archive [Slack workspaces](/docs/usage/slack/). A single block covers every @@ -828,8 +1145,11 @@ workspace first with `msgvault add-slack`. [slack] enabled = true # gate for the daemon schedule schedule = "*/30 * * * *" # 5-field cron; empty = manual sync only -channels = [] # channel-name include filter (empty = all memberships) +channels = [] # channel-name include filter (empty = all available) exclude_channels = [] # channel names to skip, e.g. ["noise"] +private_channels = true # sync private channels +dms = true # sync one-to-one direct messages +group_dms = true # sync group direct messages media = true # download shared-file bytes media_scope = "all" # all, direct, or none media_max_participants = 20 # skip files from larger channels; 0 = no cap @@ -843,14 +1163,23 @@ max_media_mb = 250 # per-file download cap (MiB) |---|---|---| | `enabled` | `false` | Whether the daemon schedules Slack sync | | `schedule` | — | Cron expression used by `msgvault serve` | -| `channels` | all | Channel names to sync (include filter; DMs are never filtered) | +| `channels` | all | Channel names to sync (include filter; never applies to DMs or group DMs) | | `exclude_channels` | — | Channel names to skip (wins over `channels`) | +| `private_channels` | `true` | Sync private channels; `false` pauses them without removing archived messages or affecting DMs | +| `dms` | `true` | Sync one-to-one DMs; `false` pauses them without removing archived messages | +| `group_dms` | `true` | Sync group DMs; `false` pauses them without removing archived messages | | `media` | `true` | Download shared-file bytes (failed downloads retry via `backfill-slack-media`) | | `media_scope` | `all` | `all`, `direct` (DMs and group DMs only), or `none`; see [Media policy](#media-policy) | | `media_max_participants` | `20` | Skip files from conversations above this many members; `0` = no cap | | `max_media_mb` | `250` | Per-file download cap in MiB (over-cap files are recorded as a `size_cap` skip and retried only after the cap changes) | | `accounts_config` | — | Per-team-ID `media` and `max_media_mb` overrides | +For public channels only, set `private_channels`, `dms`, and `group_dms` to +`false`. These settings select what sync archives; the Slack token determines +what it can access. See [Slack permissions](/docs/usage/slack/#prerequisites) +for a token restricted to public channels. A restricted token lists all public +channels, including unjoined ones; broader tokens list your memberships. + ### `[teams]` Media policy for [Microsoft Teams](/docs/usage/teams/) chats and channels. Teams @@ -911,6 +1240,36 @@ the primary identity even if aliases already exist. Manage aliases with existing meeting attribution. A scheduled source must still be registered in the archive; removing it prevents the scheduler from silently recreating it. +### Plaud Sources + +Configure one top-level `[[plaud]]` entry per Plaud cloud account. Browser +OAuth stores credentials separately from this file. Enable Cloud Sync and +transcription in Plaud before syncing. See the +[meeting guide](usage/meetings.md#plaud) for setup and preservation rules. + +```toml +[[plaud]] +identifier = "work" +account_email = "you@example.com" +schedule = "30 */6 * * *" +enabled = true +``` + +| Key | Default | Description | +|---|---|---| +| `identifier` | `default` for one unnamed entry | Stable command, source, and token label; must be unique and contain no path separators, control characters, or surrounding whitespace | +| `account_email` | Required | Explicit account email; normalized to lowercase and checked against live Plaud identity | +| `endpoint` | `https://mcp.plaud.ai/mcp` | MCP resource endpoint override; changing it requires new authorization | +| `schedule` | — | Five-field cron expression used by `msgvault serve` | +| `enabled` | `false` | Whether a scheduled entry runs in the daemon | + +Authorize with `msgvault add-plaud <identifier>` on the daemon host. The callback +uses `localhost:8091/callback/plaud`; tokens use +`tokens/plaud_<identifier>.json`. An existing source retains its confirmed +owner even if configuration changes. Use a new identifier for another account. +A scheduled entry must be registered; source removal prevents sync from +recreating it automatically. + ### Circleback Sources Circleback meeting sync is configured with top-level `[[circleback]]` @@ -970,6 +1329,39 @@ scheduler from recreating it. See [Meeting Transcripts](/docs/usage/meetings/) f the 50-result discovery limit, attendee visibility, transcript retries, and stored data. +### Muesli Sources + +Muesli meeting sync uses one top-level `[[muesli]]` entry per Muesli database. +The daemon reads the database read-only on its own host, so msgvault must run +on the Mac where Muesli records. No credential is needed. + +```toml +[[muesli]] +identifier = "mac" # stable source label; defaults to "default" for one entry +account_email = "you@example.com" # required; you, the person who records +db_path = "~/Library/Application Support/Muesli/muesli.db" # optional; this is the default +phone_country_code = "1" # optional; convert national-format Contacts phones +schedule = "*/30 * * * *" # optional 5-field cron, no seconds +enabled = true +``` + +| Key | Default | Description | +|---|---|---| +| `identifier` | `default` (single entry) | Source name used by `sync-muesli <identifier>` and scheduler logs | +| `account_email` | (required) | Normalized primary identity; attributed as the organizer of every meeting | +| `db_path` | `~/Library/Application Support/Muesli/muesli.db` | Muesli database path; `~` expands, and a relative path resolves against the config directory when `--config` is used | +| `contacts` | `true` | Resolve attendees through Apple Contacts; needs Full Disk Access for the daemon | +| `contacts_path` | `~/Library/Application Support/AddressBook` | Apple Contacts data folder; expands like `db_path` | +| `phone_country_code` | — | Country calling code (1–3 digits, such as `"1"` or `"44"`) used for Contacts phone numbers typed without one; unset means only international numbers are used | +| `schedule` | — | Cron expression used by `msgvault serve` | +| `enabled` | `false` | Whether the source is daemon-scheduled | + +Run `msgvault add-muesli <identifier>` to check the database and register the +source before enabling a schedule. Run `msgvault sync-muesli <identifier> --full` +after identity changes to repair existing meeting attribution. Removing the +source prevents the scheduler from recreating it. See +[Meeting Transcripts](/docs/usage/meetings/#muesli) for what gets stored. + ### `[vector]` Top-level toggle and backend marker for semantic/hybrid search. SQLite vector search requires a build with `sqlite_vec` support (default via `make build`). PostgreSQL vector search requires a build with the `pgvector` tag and a PostgreSQL `[data].database_url`. See [Vector Search](/docs/usage/vector-search/) for prerequisites, initial embedding, and the full workflow. @@ -1020,6 +1412,38 @@ Changing a stored key for vector or multimodal (visual) embeddings requires a da restart, like the other `[vector]` settings. Person enrichment and sweep keys apply on the next run. +On unreleased `main`, a stored person-enrichment suppression key also takes +precedence over a custom `suppression_key_env`. Previously, a custom variable +won. Before upgrading an installation that has both, ensure the stored key +matches the variable's value. If existing suppression records were made with +a different key, enrichment stops with `ErrSuppressionKeyMismatch`. + +On unreleased `main`, the host owner can also install keys without the Web UI: + +```sh +msgvault credentials set vector.embeddings --from-file /run/secrets/embedding-key +msgvault credentials set vector.multimodal --stdin < /run/secrets/visual-key +msgvault credentials set people.enrichment/research --endpoint https://api.example.com/search --stdin +msgvault credentials set people.enrichment/suppression --stdin < /run/secrets/suppression-key +msgvault credentials list --json +msgvault credentials import-env +``` + +The CLI lists IDs and bound origins without values. File input follows the +[server secret-file rules](#server). Standard input is trimmed and limited to +64 KiB. Suppression keys must contain at least 32 bytes after trimming; +`set` and `import-env` reject shorter values before saving them. +`--endpoint` defaults to the configured provider endpoint; suppression +keys have no endpoint. `import-env` copies present configured vector, +multimodal, named enrichment, and suppression variables once, preserving +existing stored keys. These host commands do not start the daemon or change +provider consent. Run them on the daemon host; `--local` selects a local home +when remote access is configured. + +Sweep providers use their own named profile credentials. Use +`msgvault person provider add --api-key-stdin` or `--credential-env` for them; +the legacy `people.sweep` store ID is not consumed by current sweep runs. + The index generation fingerprint includes the model, dimension, document and query prefixes, preprocessing settings, `max_input_chars`, embedding policy, and scope. Changing those settings triggers a stale-index error on the next vector/hybrid query. For an existing account-scoped generation built with CLI flags, set matching `[vector.embed.scope].accounts` and restart the daemon; otherwise run `msgvault embeddings build --full-rebuild`. #### `[vector.preprocess]` @@ -1045,6 +1469,14 @@ Hybrid ranking parameters applied at query time. | `k_per_signal` | `100` | Candidate pool size drawn from each signal (BM25 or vector) before fusion. | | `subject_boost` | `2.0` | Multiplier applied when a query term matches a message's subject line. | | `max_page_size_hybrid` | `50` | Hard cap on `page_size` for vector/hybrid responses. Set to `0` to disable clamping. | +| `sqlite_accelerator` | `auto` | Use a ready SQLite approximate index. Set to `exact` to keep exhaustive vector search. PostgreSQL ignores this setting. | +| `ann_nprobe` | `8` | SQLite index partitions searched per query. Higher values trade latency for recall. | +| `ann_oversample` | `8` | Approximate candidates requested per result before exact reranking. Range: 1–128. | +| `ann_threads` | CPU count, max `128` | Native worker threads used by `msgvault embeddings optimize`. Range: 1–128. | + +Accelerator tuning does not change the embedding generation fingerprint. It +changes how stored vectors are searched or optimized, not how text is sent to +the embedding provider. #### `[vector.embed.scope]` @@ -1142,6 +1574,19 @@ deterministic source of "when did we last talk" for every person and runs hourly by default inside `msgvault serve`. `msgvault activity build` runs it by hand; `--backstop` rescans the whole archive. +Scheduled projection commits at most ten batches per pass. When other +scheduled work has waited for a minute, it stops after its current batch. A +pass always stops at two minutes. A pass with committed progress releases the +operation gate and resumes behind queued work without waiting for the next +cron tick. Reaching the two-minute limit before any batch commits records an +error and waits for the next scheduled or manual trigger, avoiding repeated +retries of the same batch. + +Identity reconciliation and timezone or `max_direct_counterparts` changes save +their progress in the archive. A new identity revision restarts identity +reconciliation; completed batches remain committed. Manual builds are not +limited to ten batches or two minutes. + | Key | Default | Description | |---|---|---| | `schedule` | `17 * * * *` | 5-field cron used by `msgvault serve`. Empty disables the scheduled job. | @@ -1174,19 +1619,63 @@ ownership and permission checks to the target directory. | Variable | Description | |---|---| | `MSGVAULT_HOME` | Base directory for all data (default: `~/.msgvault`) | -| `MSGVAULT_REMOTE_URL` | Remote URL for `export-token` (flag > env > config) | -| `MSGVAULT_REMOTE_API_KEY` | Remote API key for `export-token` (flag > env > config) | +| `MSGVAULT_BIND_ADDR` | Server bind address or `iface:NAME`; `serve --bind` wins | +| `MSGVAULT_API_PORT` | Server port from `0` to `65535`; `serve --port` wins | +| `MSGVAULT_API_KEY` | Inline server key for this process | +| `MSGVAULT_API_KEY_FILE` | Mounted server key file | +| `MSGVAULT_API_KEY_ENV` | Name of the environment variable holding the server key | +| `MSGVAULT_ALLOW_INSECURE` | Allow unauthenticated non-loopback serving | +| `MSGVAULT_BACKUP_REPO` | Backup repository path | +| `MSGVAULT_CORS_ORIGINS` | Comma-separated browser origins; empty clears the list | +| `MSGVAULT_CORS_CREDENTIALS` | Whether browser CORS requests may use credentials | +| `MSGVAULT_TRUSTED_PROXIES` | Comma-separated proxy IP addresses or CIDRs; empty clears the list | +| `MSGVAULT_REMOTE_URL` | Remote daemon URL for all commands with remote support | +| `MSGVAULT_REMOTE_API_KEY` | Inline remote key for this process; `export-token --api-key` wins | +| `MSGVAULT_REMOTE_API_KEY_FILE` | Mounted remote key file | +| `MSGVAULT_REMOTE_API_KEY_ENV` | Name of the environment variable holding the remote key | +| `MSGVAULT_REMOTE_ALLOW_INSECURE` | Allow plaintext HTTP to the remote daemon | + +These runtime controls are available on unreleased `main`. Environment values +override TOML. For each server or remote credential group, setting any of its +three environment variables selects that group's environment sources; within +the group, inline wins over file, then named environment. A supplied empty +credential variable is an error when that destination is used. Booleans accept +Go's `strconv.ParseBool` values, such as `true`, `false`, `1`, and `0`; empty or +invalid values fail. Origin and proxy lists trim whitespace and ignore empty +entries, including a trailing comma. Explicit `export-token --to`, `--api-key`, +and `--allow-insecure` choices are saved even when they match an environment +override; environment-only values are not saved. An explicit +`--allow-insecure=false` overrides the environment and saved configuration, +requires HTTPS, and saves `false` after a successful export. The setup wizard also saves +new choices that match an override; keeping existing settings leaves them +unchanged on disk. + +For example, a supervised daemon can start without a config file: + +```sh +MSGVAULT_HOME=/data MSGVAULT_BIND_ADDR=0.0.0.0 MSGVAULT_API_PORT=8080 msgvault serve +``` + +Persist `/data` across restarts so the archive and generated key survive. The +published stock image is `ghcr.io/kenn-io/msgvault`; it runs as UID/GID `1000` +and stores its home at `/data`. Use an image containing these unreleased +features once published. No startup hook or entrypoint wrapper is required. ## File Locations -All data lives under the msgvault home directory (`~/.msgvault` on macOS/Linux, `C:\Users\<you>\.msgvault` on Windows). The directory is created automatically on first use. +The default home is `~/.msgvault` on macOS/Linux and `C:\Users\<you>\.msgvault` +on Windows. It is created automatically; existing directory permissions are +left unchanged. Configuration stays under the home unless `--config` selects +another file. The data paths below use `[data].data_dir`, which defaults to the +home; `[log].dir` can override the log location. | File | Description | |---|---| -| `config.toml` | Configuration file | +| `<home>/config.toml` | Configuration file | | `msgvault.db` | SQLite database (system of record when PostgreSQL is not configured) | | `attachments/` | Content-addressed attachment files | -| `tokens/` | OAuth tokens per account | +| `tokens/` | OAuth tokens and stored provider credentials | +| `tokens/server-api-key` | Persisted daemon API key, reused on later loopback and non-loopback starts | | `logs/` | Structured log files (when [file logging](/docs/configuration/#log) is enabled) | | `analytics/` | Parquet cache files for Web UI and TUI analytical views | @@ -1267,6 +1756,7 @@ bind_addr = "127.0.0.1" api_key = "your-secret-key" daemon_idle_timeout = "20m" # background daemon idle timeout; "0s" disables daemon_auto_restart = "newer" # newer, never, or always +daemon_auto_start = true # false when a supervisor runs msgvault serve [analytics] # Daemon-side analytics engine for Web UI, TUI, and aggregate HTTP views: diff --git a/docs/development.md b/docs/development.md index 57e856560..c7034f721 100644 --- a/docs/development.md +++ b/docs/development.md @@ -28,15 +28,19 @@ make install ### Windows -Use the PowerShell build helper from the repository root. It provides the same -debug and release builds as the Make targets and selects the host architecture -automatically: +Use the PowerShell helper from the repository root to compile the Go binary. +It selects the host architecture automatically and embeds assets already in +`internal/web/dist`; it does not build the browser application. + +For a binary with the Web UI, first run `make web-embed` in an MSYS2 shell +with GNU Make, Bun, and Node.js available. This builds and validates the browser +assets. Then run the PowerShell helper: ```powershell -# Debug build (equivalent to make build) +# Debug Go build .\scripts\build.ps1 -# Optimized, stripped build (equivalent to make build-release) +# Optimized, stripped Go build .\scripts\build.ps1 -Release ``` @@ -86,7 +90,9 @@ Defaults identify a development build. Image builds run checks against a temporary empty archive: database initialization, a DuckDB query, and the embedded Web UI and its JavaScript -asset. `scripts/smoke-container.sh` repeats these checks in the loaded image +asset. The runtime smoke also starts from environment variables without a +config file and verifies that its minted daemon key survives a restart. +`scripts/smoke-container.sh` repeats these checks in the loaded image with networking disabled. These commands build and check local artifacts; they do not publish them. @@ -112,6 +118,27 @@ automatically. Use `assert` and `require` from testify, with expected values first. See [AGENTS.md](https://github.com/kenn-io/msgvault/blob/main/AGENTS.md) for repository testing rules. +### Timing waits + +Use `testing/synctest` bubbles for work owned by the test process, including +goroutines, channels, timers, tickers, and fakes. Advance virtual time with +`synctest.Sleep` and wait for durable state with `synctest.Wait`. + +Keep real budgets for PostgreSQL and SQLite locks, database clocks, network +requests, subprocesses, DuckDB, and operating-system events. Name retained +sub-second testify budgets so their event and owner are clear. + +The helper check rejects bare totals below one second in `Eventually`, +`Eventuallyf`, `EventuallyWithT`, `EventuallyWithTf`, `Never`, and `Neverf`. +Named budgets and variables stay outside this rule. Virtual sleeps are valid +inside a bubble. CI runs this check on Ubuntu, so Windows-only test files still +need Windows validation. + +`make lint` and `make lint-ci` build a pinned golangci-lint with Kit's +`kennlint` plugin and run its `sleeptest` check. The check rejects `time.Sleep` +in tests outside a `synctest.Test` bubble. A kept real wait carries +`//nolint:kennlint // <what it waits for>` on the sleep line. + ### PostgreSQL tests `MSGVAULT_TEST_DB=postgres://...` runs PostgreSQL-backed @@ -182,13 +209,89 @@ CI's explicit `test-unsharded` and package-shard jobs keep their existing layout # Format code make fmt -# Run linter (requires golangci-lint) +# Run linter (builds the pinned golangci-lint with Kit's plugin; needs git) make lint # Check for issues go vet ./... ``` +## Profile Web UI search + +Inspect the `Server-Timing` headers on successful `/api/v1/explore` responses in +browser developer tools. Failed requests may contain only the phases completed +before the error. Durations are milliseconds: + +| Metric | Work measured | +| --- | --- | +| `candidates` | Resolve the search candidate pool, including the search phases below | +| `lexical` | Count full-text matches and fetch bounded, ranked message IDs | +| `embedding` | Embed the semantic or hybrid query | +| `retrieval` | Retrieve vector or fused results; `desc` names the accelerator path | +| `projection` | Build result rows from the analytical cache | +| `identities` | Hydrate identity matches for the returned rows | + +The `candidates` duration includes `lexical`, `embedding`, and `retrieval` work; +do not add those subphases to it. A reused semantic candidate snapshot skips +embedding and retrieval, so those metrics are absent. Group requests to +`/api/v1/explore/groups` report `grouping` for the analytical aggregation. + +Run the synthetic HTTP benchmark to measure candidate resolution through the +real SQLite full-text index and DuckDB result projection: + +```bash +go test -tags 'fts5 sqlite_vec' ./internal/api -run '^$' \ + -bench '^BenchmarkExploreFullText$' -benchtime=3x -count=3 -benchmem +``` + +It creates an isolated 20,000-message archive and measures rare, common, and +candidate-limited queries. Setup and a warm-up request are excluded. Compare +the same fixture and machine before and after a change; these timings do not +predict latency on a larger archive. + +For analytical grouping and domain drill-downs, build the larger synthetic +cache once in an empty scratch directory, then reuse it: + +```bash +explore_bench_root=$(mktemp -d) +go test -tags 'fts5 sqlite_vec' ./internal/query -run '^$' \ + -bench '^BenchmarkExploreScaleBuild$' -benchtime=1x \ + -args -relationship-bench-root="$explore_bench_root" +go test -tags 'fts5 sqlite_vec' ./internal/query -run '^$' \ + -bench '^BenchmarkExploreScaleQueries$' -benchtime=4x \ + -args -relationship-bench-root="$explore_bench_root" +``` + +This fixture has 2,562,000 messages and 71,486 people. The query benchmark +reports the first request and the median of later requests separately. It uses +the default interactive memory limit; DuckDB can spill work to temporary disk. +Add `-relationship-bench-memory=8GB` for diagnostic comparisons with older +queries that exceed that limit. Add +`-relationship-bench-profile` to write DuckDB operator profiles into the scratch +directory. Run timing comparisons without other heavy workloads, and retain +the same memory limit and thread settings. This fixture uses uniform senders +and one chat roster; it does not model real participant skew or attachments. + +To separate local vector retrieval from embedding-provider latency, run: + +```bash +go test -tags 'fts5 sqlite_vec' ./internal/vector/sqlitevec -run '^$' \ + -bench '^BenchmarkVectorRetrieval$' -benchtime=3x -count=3 -benchmem +``` + +This benchmark uses 100,000 synthetic messages with one 64-dimensional vector +each. It measures exact semantic retrieval and rare/common hybrid retrieval, +without an accelerator, embedding calls, or HTTP result projection. Use the +request timing headers to determine which phase needs attention on a real +archive before comparing it with this narrower benchmark. + +## Evaluate search quality + +Use [`msgvault eval`](cli-reference.md#eval) to compare keyword, semantic, and +hybrid results against queries and relevance ratings you supply. Keep the +archive, topics, and ratings the same when comparing runs. The command reports +ranking quality and query timings; it does not create the ratings for you. + ## vCard registry maintenance The lossless vCard 2.1/3.0/4.0 codec vendors the IANA vCard Elements registry diff --git a/docs/diagrams/README.md b/docs/diagrams/README.md index 238a9effe..7db73a1af 100644 --- a/docs/diagrams/README.md +++ b/docs/diagrams/README.md @@ -12,7 +12,7 @@ screenshot plus an ImageMagick trim/pad. | `account-collection-concept.html` | `docs/assets/generated/concepts/account-collection-concept.png` | Accounts, Identities, and Collections | | `deduplication-concept.html` | `docs/assets/generated/concepts/deduplication-concept.png` | Deduplication | | `safety-ladder-concept.html` | `docs/assets/generated/concepts/safety-ladder-concept.png` | Deduplication (the five-rung safety ladder) | -| `survivor-selection-concept.html` | `docs/assets/generated/concepts/survivor-selection-concept.png` | Deduplication (survivor selection) | +| `survivor-selection-concept.html` | `docs/assets/generated/concepts/survivor-selection-concept.png` | HTML reference; Deduplication uses Mermaid | | `oauth-multi-account-concept.html` | `docs/assets/generated/concepts/oauth-multi-account-concept.png` | Accounts page and the OAuth Setup guide | ## Building @@ -80,3 +80,7 @@ The deduplication and safety-ladder raster captures are currently omitted from the public reading path. Their HTML sources now distinguish IMAP Trash from explicit permanent deletion; regenerate and inspect both before restoring any links to the captures on the generated-assets branch. + +The survivor-selection guide uses Mermaid while its raster awaits regeneration. +Its HTML source includes the source metadata quality tier; inspect a regenerated +PNG before restoring the raster to the public reading path. diff --git a/docs/diagrams/survivor-selection-concept.html b/docs/diagrams/survivor-selection-concept.html index 2f8b97957..3b942896d 100644 --- a/docs/diagrams/survivor-selection-concept.html +++ b/docs/diagrams/survivor-selection-concept.html @@ -376,7 +376,7 @@ <h2>Stage 2 — Priority list</h2> <div class="tier-num">01</div> <div> <div class="tier-name">Source preference</div> - <div class="tier-hint">default order gmail, imap, mbox, emlx, hey; override with --prefer</div> + <div class="tier-hint">default order gmail, imap, msmail, mbox, emlx, hey; override with --prefer</div> </div> <div class="tier-side"><span class="tier-tag">policy</span></div> </div> @@ -419,6 +419,15 @@ <h2>Stage 2 — Priority list</h2> <div class="tier-arrow">↓</div> <div class="tier"> <div class="tier-num">06</div> + <div> + <div class="tier-name">Source metadata quality</div> + <div class="tier-hint">one point each: native message ID, threading evidence, RFC822 Message-ID</div> + </div> + <div class="tier-side"><span class="tier-tag">provenance</span></div> + </div> + <div class="tier-arrow">↓</div> + <div class="tier"> + <div class="tier-num">07</div> <div> <div class="tier-name">Richer label or folder metadata</div> <div class="tier-hint">Gmail labels, IMAP folders, Apple Mail mailboxes</div> @@ -427,7 +436,7 @@ <h2>Stage 2 — Priority list</h2> </div> <div class="tier-arrow">↓</div> <div class="tier"> - <div class="tier-num">07</div> + <div class="tier-num">08</div> <div> <div class="tier-name">Earlier archived timestamp (when meaningful)</div> <div class="tier-hint">older archive entry, all else equal</div> @@ -436,7 +445,7 @@ <h2>Stage 2 — Priority list</h2> </div> <div class="tier-arrow">↓</div> <div class="tier"> - <div class="tier-num">08</div> + <div class="tier-num">09</div> <div> <div class="tier-name">Stable row ID</div> <div class="tier-hint">final tie-breaker — guarantees deterministic output</div> diff --git a/docs/faq.md b/docs/faq.md index f776618f5..a3102751e 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -1,16 +1,20 @@ --- -last_edited: 2026-09-08 +last_edited: 2026-10-02 title: Frequently Asked Questions description: Common questions about msgvault, Gmail API safety, and what the tool can and cannot do. --- -<p class="faq-question">Can msgvault send email?</p> +<p class="faq-question">Can msgvault send email or prepare a reply?</p> -No. msgvault archives and analyzes messages; it does not compose, send, forward, -or reply to mail. Gmail authorization requests `gmail.modify` by default for -archive and deletion workflows. `add-account --readonly` requests read-only -access instead. See [read-only Gmail access](guides/oauth-setup.md#read-only-access) -for existing-account restrictions. +msgvault never sends email. For an IMAP source, an operator can grant access to +one exact Drafts folder; `draft-reply` can then create a plain-text reply draft +for review and sending in your usual mail application. The grant is disabled by +default. See [IMAP drafts](usage/imap.md#drafts). + +Gmail authorization requests `gmail.modify` by default for archive and deletion +workflows. `add-account --readonly` requests read-only access instead. See +[read-only Gmail access](guides/oauth-setup.md#read-only-access) for +existing-account restrictions. <p class="faq-question">What can an AI assistant do through MCP?</p> @@ -21,8 +25,10 @@ is not a promise that the assistant cannot access private archive content. MCP can stage a deletion manifest but cannot execute remote deletion, send mail, or sync new messages. Optional profile writes require `--allow-profile-writes`; -HTTP write tools also need `--http-allow-writes`. Execution of a staged mail -deletion remains a separate CLI step. See the [MCP tool and access reference](usage/chat.md). +calendar event mutations require `--allow-calendar-writes` and explicit +per-operation approval through an MCP client that supports elicitation. HTTP +writes also need `--http-allow-writes`. Execution of a staged mail deletion +remains a separate CLI step. See the [MCP tool and access reference](usage/chat.md). Treat imported messages, attachments, and generated briefs as untrusted input to an assistant. Choose an assistant and model provider you are willing to give @@ -40,10 +46,10 @@ features. See [recommended configuration](usage/recommended-configuration.md). <p class="faq-question">Why is a documented feature missing from my binary?</p> -The documentation follows current `main`, including work after 0.19.3 that is -not yet released. Check `msgvault version` and the installed command's `--help`, -then consult [the changelog](changelog.md#unreleased). A configured remote daemon also -needs a compatible version. +The documentation follows current `main`; your installed binary may be older. +Check `msgvault version` and the installed command's `--help`, then consult the +[0.20.0 changelog](changelog.md#0200). Upgrade the CLI and a configured remote +daemon together. <p class="faq-question">What is the web server for?</p> diff --git a/docs/guides/daemon-migration.md b/docs/guides/daemon-migration.md index a70830739..3d2febfde 100644 --- a/docs/guides/daemon-migration.md +++ b/docs/guides/daemon-migration.md @@ -68,6 +68,7 @@ older and restarts it automatically before issuing your command. This is the `[server].daemon_auto_restart = "newer"` default; set `"never"` if a supervisor (systemd, Docker) owns the daemon lifecycle, or `"always"` to restart on any version difference. +If a supervisor runs `msgvault serve`, also set `[server].daemon_auto_start = false` so a CLI command never starts a competing daemon before the supervisor does. ## One writer, visible waits diff --git a/docs/guides/oauth-setup.md b/docs/guides/oauth-setup.md index ae397c1e0..05e49f735 100644 --- a/docs/guides/oauth-setup.md +++ b/docs/guides/oauth-setup.md @@ -22,6 +22,7 @@ msgvault requires OAuth credentials to access the Gmail API. This section walks 2. In the search bar, search for "Gmail API" and click the **Gmail API** box 3. Click **Enable** 4. If you wish to sync Google Calendar too, click **Library**, search for "Google Calendar API", click the **Google Calendar API** box and click **Enable** +5. If you wish to sync Google Contacts over CardDAV, click **Library**, search for "CardDAV", click the **Google Contacts CardDAV API** box and click **Enable**. Without it, saving the CardDAV account fails during discovery because Google rejects every request with `SERVICE_DISABLED` ### Step 3: Configure OAuth Consent Screen @@ -337,8 +338,8 @@ You need to register an application in Microsoft Entra (Azure AD) before using ` - **Supported account types:** "Accounts in any organizational directory and personal Microsoft accounts" - **Redirect URI:** Platform = **Mobile and desktop applications**, URI = your `redirect_uri` from `config.toml` (default: `http://localhost:8089/callback/microsoft`) 3. Click **Register** -4. Under **API permissions**, click **Add a permission > APIs my organization uses**, search for **Office 365 Exchange Online**, select **Delegated permissions**, then add `IMAP.AccessAsUser.All` -5. Under **Authentication**, enable **Allow public client flows** (required for PKCE) +4. Under **API permissions**, click **Add a permission > Microsoft Graph > Delegated permissions**, then add `IMAP.AccessAsUser.All` +5. Under **Authentication**, enable **Allow public client flows** (required for PKCE and for `--headless`) 6. If you will use a custom `redirect_uri` in `config.toml`, make sure the Redirect URI in the app registration matches it exactly — including scheme, host, port, and path. For `https://localhost/` on a privileged port (e.g. 443), register that exact URI. 7. Copy the **Application (client) ID** from the app's Overview page @@ -373,6 +374,14 @@ This opens your browser for Microsoft OAuth consent. After you authorize, msgvau - Auto-detects the correct IMAP host based on account type - Configures XOAUTH2 authentication automatically +On a machine without a browser, such as a server or a container, add `--headless`: + +```bash +msgvault add-o365 you@outlook.com --headless +``` + +msgvault prints a Microsoft URL and a code. Open the URL on any device and enter the code. `add-teams` accepts the same flag. + Personal accounts (hotmail.com, outlook.com, live.com, msn.com) connect to `outlook.office.com`. Organizational accounts (company Microsoft 365) connect to `outlook.office365.com`. This detection is automatic. To restrict to a specific tenant at authorization time: @@ -411,6 +420,32 @@ Some organizations require administrator consent before delegated channel message permissions can be used. See [Microsoft Teams](/docs/usage/teams/) for the full Teams workflow. +### Microsoft Graph Mail Sync + +If IMAP is turned off for a mailbox, `add-o365 --graph` syncs it through the +Microsoft Graph mail API. It uses the same `[microsoft] client_id` and redirect +URI. Add the **Microsoft Graph** delegated permission `Mail.Read` to the app +registration, then authorize and sync: + +```bash +msgvault add-o365 you@example.com --graph +msgvault sync you@example.com +``` + +The token is saved under `tokens/msmail_<email>.json`, and the account has the +type `msmail`. Each mail folder becomes a label. The first sync downloads every +folder. Later syncs fetch only the changes, including moves between folders +and deletes. The daemon schedules the account like any other. + +To delete messages at the source with `delete-staged`, also add the delegated +permission `Mail.ReadWrite`. Sync does not use it. The first `delete-staged` +for the account asks to upgrade the token. See +[Deleting Email](/docs/usage/deletion/). + +A Graph account is a new account. If the same mailbox is also synced over +IMAP, the vault holds two copies. Run `msgvault dedup --collection` to hide the +extra copies, and `--undo` to reverse it. + ### Sync Your Email After adding the account, sync it the same way as any other account: @@ -421,7 +456,18 @@ msgvault sync-full you@outlook.com ### Headless Servers -On a headless server (SSH, VPS, Docker), authorize on a machine with a browser and copy the token file to the server: +Sign in from SSH, a server, or a container without opening a local browser: + +```bash +msgvault add-o365 you@outlook.com --headless +``` + +For Graph mail, add `--graph`. For Teams, run +`msgvault add-teams you@example.com --headless`. +Open the printed Microsoft URL on another device and enter the code. Complete +sign-in there; msgvault saves the token on the server. + +You can also authorize on another machine and copy its token. For IMAP mail: 1. On your local machine, run `msgvault add-o365 you@outlook.com` and complete the browser flow. 2. Copy the token to the server: diff --git a/docs/guides/remote-deployment.md b/docs/guides/remote-deployment.md index 6494991c1..6bec931ea 100644 --- a/docs/guides/remote-deployment.md +++ b/docs/guides/remote-deployment.md @@ -1,9 +1,12 @@ --- +last_edited: "2026-09-22" title: Remote Deployment description: Run msgvault in Docker on a remote host and provision it from a machine with a browser. --- -msgvault supports a remote-first workflow where you configure a remote instance using a local browser session, then deploy and sync on headless hardware. This works with any always-on host: a NAS (a good choice for RAID fault tolerance), a cloud VM, a Raspberry Pi, or any Linux server with Docker. +Run your archive on an always-on server and use it from your own computer. +Set up source credentials on a machine with a browser, then copy a deployment +bundle to a Linux host with Docker, such as a NAS, cloud VM, or Raspberry Pi. The flow is built on three capabilities: @@ -25,19 +28,15 @@ The flow is built on three capabilities: ## Docker Image -The image is published to GitHub Container Registry: +Choose a published image from the [container package](https://github.com/kenn-io/msgvault/pkgs/container/msgvault) +and set its full name in your Compose file. Version tags omit the leading `v`: +for example, release 0.19.3 uses `ghcr.io/kenn-io/msgvault:0.19.3`. -```bash -docker pull ghcr.io/kenn-io/msgvault:latest -``` - -| Tag | Description | -|-----|-------------| -| `latest` | Latest stable release from main branch | -| `v1.2.3` | Specific version | -| `1.2` | Latest patch of minor version | -| `1` | Latest minor/patch of major version | -| `sha-abc1234` | Specific commit (for debugging) | +`latest` is not a stable-release guarantee and can refer to an older development +snapshot. Repository-owned image publishing has been removed, so a new commit +or release tag does not automatically update GHCR. Check that the image you +choose contains the version you need. To deploy current source before a matching +image is published, follow [Container builds](../development.md#container-builds). **Architectures:** `linux/amd64` (Intel/AMD NAS, standard servers) and `linux/arm64` (Raspberry Pi 4/5, newer NAS). Docker selects the correct one automatically. @@ -116,6 +115,10 @@ services: start_period: 10s ``` +The wizard currently writes `:latest` as shown above. Before deployment, change +`image` to your chosen published version or digest. For an image built and +loaded locally, use its local tag and remove `pull_policy: always`. + ## 2) Deploy to Remote Host Copy the bundle and start services via SSH: @@ -327,10 +330,10 @@ docker exec -it msgvault msgvault tui # Interactive TUI # Restart using the currently installed image docker-compose restart -# Reconcile the service; generated bundles check for a newer latest image +# Reconcile the service with the image configured in Compose docker-compose up -d -# Explicitly update to the latest image +# Pull the configured image after choosing the version to deploy docker-compose pull docker-compose up -d @@ -339,9 +342,10 @@ docker-compose down ``` `restart` does not check the registry or replace the image. Generated bundles -set `pull_policy: always`, so `up -d` reconciles against GHCR. The explicit -`pull` followed by `up -d` sequence remains the clearest update procedure -across NAS Compose implementations. +set `pull_policy: always`, so `up -d` checks the configured tag in GHCR. A +version-pinned service stays on that version until you edit `image`. Back up +before upgrading and keep the client and daemon versions compatible. Then use +`pull` followed by `up -d` to deploy the selected image. Bundles generated before `pull_policy: always` was added are not rewritten automatically. Existing installations should either regenerate the bundle, diff --git a/docs/guides/sources.md b/docs/guides/sources.md index 8bad57667..a358c73c8 100644 --- a/docs/guides/sources.md +++ b/docs/guides/sources.md @@ -1,5 +1,5 @@ --- -last_edited: "2026-09-08" +last_edited: "2026-10-02" title: Choose a Source description: Find the right sync or import path for mail, chat, meetings, calendars, and contacts. --- @@ -19,7 +19,7 @@ non-Google provider. |---|---|---| | Gmail or Google Workspace | [Gmail setup](../setup.md#configure-oauth) | Google OAuth app and account authorization; read-only access is an option | | An IMAP mailbox | [IMAP sync](../usage/imap.md) | Server address and credentials, often an app password | -| Microsoft 365 mail | [Microsoft 365 setup](../cli-reference.md#add-o365) | A Microsoft OAuth app and IMAP access | +| Microsoft 365 or Outlook.com mail | [Microsoft mail setup](oauth-setup.md#microsoft-365-outlook-hotmail) | A Microsoft OAuth app; choose IMAP or Graph mail access | | Maildir or Maildir++ archive | [Maildir import](../usage/importing.md#import-maildir) | A stable snapshot with `cur`, `new`, and `tmp` directories | | MailMate-style `.mailbox` directories | [EML import](../usage/importing.md) | A `.mailbox` tree containing `.eml` files and an archive identifier | | MBOX, Apple Mail, or Outlook PST | [Local email import](../usage/importing.md) | An exported mailbox or readable local mail directory | @@ -38,7 +38,7 @@ also a separate opt-in because downloading an image can activate email tracking. | Microsoft Teams | [Teams sync](../usage/teams.md) | Chats, self-chat, channels, replies, and available media | | Discord | [Discord sync](../usage/discord.md) | Bot-accessible guild channels, threads, and forums; personal DMs are outside this integration | | Beeper Desktop | [Beeper sync](../usage/beeper.md) | History and media exposed by the running local Beeper API | -| WhatsApp, iMessage, Google Voice, Messenger | [Text message imports](../usage/text-messages.md) | Supported backups or exports, with your identity supplied where required | +| WhatsApp, iMessage, iMazing CSV, Google Voice, Messenger | [Text message imports](../usage/text-messages.md) | Supported backups or exports, with your identity supplied where required | | SMS Backup & Restore | [Android SMS and call logs](../usage/text-messages.md) | Local XML/ZIP or scheduled imports from a configured Drive folder | Chat media has size and room-participant limits. A message can be archived @@ -50,8 +50,8 @@ missing downloads. | Your source | Start here | What it adds | |---|---|---| -| Granola, Circleback, or Notion AI Meeting Notes | [Meeting notes and transcripts](../usage/meetings.md) | Searchable notes, transcripts where available, and participants | -| Another meeting capture tool | [Meeting import API](../api-server.md) | Provider-neutral ingestion keyed by source and external meeting ID | +| Granola, Plaud, Circleback, Notion AI Meeting Notes, or Muesli | [Meeting notes and transcripts](../usage/meetings.md) | Searchable notes, context exports, recorded actions, and meeting-time coverage | +| Another meeting capture tool | [Meeting import workflow](../usage/meetings.md#import-from-any-meeting-source) | Provider-neutral ingestion keyed by source and external meeting ID | | Google Calendar | [Calendar sync](../usage/calendar.md) | Events, organizers, attendees, recurrence, and cancellation state | | CardDAV address book | [CardDAV contacts](../usage/people-carddav.md) | Imported contacts and explicit publication of curated profiles | @@ -72,4 +72,6 @@ review its conflict and consent workflow before enabling that direction. For records already stored, keyword search and analytics use the archive. Optional [semantic search and profile automation](../usage/recommended-configuration.md) -have their own provider configuration and consent steps. +have their own provider configuration. Message embeddings are enabled through +configuration; people, visual, and document features also require their stated +consent steps. diff --git a/docs/guides/verification.md b/docs/guides/verification.md index 1d8160556..ede63fd0f 100644 --- a/docs/guides/verification.md +++ b/docs/guides/verification.md @@ -1,9 +1,18 @@ --- +last_edited: "2026-09-22" title: Verify Integrity -description: Verify your archive against Gmail. +description: Check a Gmail archive's database, raw-message coverage, and sampled MIME data. --- -## Usage +Check whether a Gmail archive is structurally readable and whether sampled raw +messages can be decompressed. The command also reports Gmail's current message +count beside the archive count. Those counts are a comparison, not proof that +the two systems contain the same messages. + +`verify` is Gmail-specific. It contacts Gmail with the selected account's +authorization, even when the archive itself is local. + +## Run a check ```bash # Default: sample 100 messages @@ -13,16 +22,22 @@ msgvault verify you@gmail.com msgvault verify you@gmail.com --sample 500 ``` -## What It Checks +## What it checks -The verify command compares your archive against Gmail through the configured -remote server or local daemon: +The command goes through the configured remote server or local daemon and +reports four checks: | Check | Description | |---|---| -| Message count | Compares local count vs Gmail message count | -| Raw MIME presence | Verifies sampled messages have raw MIME data stored | -| FTS index entries | Confirms sampled messages are indexed for full-text search | +| Database integrity | Runs SQLite `PRAGMA integrity_check` unless skipped. PostgreSQL archives must use `pg_amcheck` separately. | +| Message counts | Reports Gmail's profile total, the archive account total, and their signed difference. Gmail's total and msgvault's archive policy can cover different sets. | +| Raw MIME coverage | Counts archived messages that have stored raw MIME data and reports the percentage. | +| MIME sample | Selects up to `--sample` archived raw messages and checks that each stored MIME value can be decompressed. | + +This command does not compare Gmail and archive message IDs, inspect +attachments for completeness, or confirm that every archived message has an entry in the full-text +search index. See [rebuilding the search index](../cli-reference.md#rebuild-fts) +for index recovery. ## Flags @@ -32,9 +47,9 @@ remote server or local daemon: | `--skip-db-check` | `false` | Skip SQLite integrity check | | `--json` | `false` | Emit machine-readable JSON summary | -## When to Verify +## When to verify -- After initial full sync to confirm completeness +- After an initial full sync to check database and raw-MIME health - Before executing deletions from Gmail - Periodically to check for database corruption - After recovering from interrupted syncs diff --git a/docs/index.md b/docs/index.md index bbe3a38f8..3c3be25e0 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,5 +1,5 @@ --- -last_edited: "2026-09-08" +last_edited: "2026-10-02" title: Documentation description: Set up your archive, find messages and files, maintain people, and operate msgvault. --- @@ -12,14 +12,13 @@ the archive through the browser, terminal, CLI, or an agent. <p class="hero-actions"> <a class="md-button md-button--primary" href="/docs/setup/">Get started</a> - <a class="md-button" href="/docs/changelog/#unreleased">Changelog</a> + <a class="md-button" href="/docs/changelog/#0210">Changelog</a> </p> -!!! note "Returning after 0.19?" - These docs follow current `main`, including unreleased work after 0.19.3. - The [changelog](changelog.md#unreleased) lists new capabilities and - [upgrade notes](changelog.md#upgrade-and-compatibility), with released and - unreleased changes kept separate. +!!! note "Upgrading to 0.21?" + The [0.21.0 changelog](changelog.md#0210) lists the new capabilities. + Read the [upgrade notes](changelog.md#before-upgrading) before + opening an existing archive with the new version. ## Start an archive @@ -60,6 +59,7 @@ limit an archive view. | Verify stored mail | [Archive verification](guides/verification.md) | | Hide duplicate copies | [Deduplication](usage/deduplication.md) | | Remove mail from a provider | [Deletion staging and execution](usage/deletion.md) | +| Prepare a message draft for review | [Mail drafts](cli-reference.md#draft-reply), [Beeper drafts](cli-reference.md#beeper-chat-drafts), and [local chat drafts](cli-reference.md#local-chat-drafts) | | Keep a recoverable copy | [Backup and restore](usage/backup.md) | | Take data elsewhere | [Exporting](usage/exporting.md) | | Diagnose a problem | [Troubleshooting](troubleshooting.md) and [FAQ](faq.md) | diff --git a/docs/internal/README.md b/docs/internal/README.md index 6381d0b06..bf7199199 100644 --- a/docs/internal/README.md +++ b/docs/internal/README.md @@ -17,6 +17,7 @@ For maintenance rules, see the [documentation contributor guide](../README.md). | Slack ingestion and reply discovery | [Ingestion](slack-ingestion-design.md), [reply sweep](slack-reply-sweep-design.md) | [Slack](../usage/slack.md) | | Message exports | [Design](message-export-design.md) and [plan](message-export-plan.md) | [Exporting](../usage/exporting.md) | | People and relationships | [Relationship index](relationship-list-index-design.md), [merge reversal](person-merge-reversal.md), [conversation brief](last-time-we-talked-design.md) | [People and profiles](../usage/people.md) | +| Web UI navigation and visual language | [Design](web-ui-overhaul-design.md), [foundation plan](web-ui-overhaul-pr1-plan.md), [Everything and Files plan](web-ui-overhaul-pr2-plan.md), [People spec](web-ui-overhaul-pr3-spec.md), [People plan](web-ui-overhaul-pr3-plan.md), [Manage spec](web-ui-overhaul-pr4-spec.md), and [Manage plan](web-ui-overhaul-pr4-plan.md) | [Web UI](../web-ui.md) | | Daemon command routing | [CLI audit](daemon-cli-request-audit.md) | [Daemon guide](../guides/daemon-migration.md) | | PostgreSQL | [Original implementation tracker](PG_STATUS.md) | [PostgreSQL backend](../architecture/postgresql.md) | | Recovery | [Recovery notes](recovery.md) | [Backup](../usage/backup.md) and [troubleshooting](../troubleshooting.md) | diff --git a/docs/internal/accounts-identities-collections-dedup/spec.md b/docs/internal/accounts-identities-collections-dedup/spec.md index c759648ec..e8593befc 100644 --- a/docs/internal/accounts-identities-collections-dedup/spec.md +++ b/docs/internal/accounts-identities-collections-dedup/spec.md @@ -196,7 +196,11 @@ more of: sent-mail folder or had a Gmail `SENT` label. - `account-identifier` — the address matches the account's primary identifier (e.g. the Gmail address itself). -- `oauth` — OAuth or provider account metadata named the address. +- `oauth` — an authenticated Gmail profile matched the source account. + `identity discover --provider` previews this strong evidence; `--apply` + confirms it. Ordinary sync only refreshes already-confirmed identities, + preserving removals and `--no-default-identity`. +- `phone-e164` — a normalized phone number from an SMS or chat source. - `manual` — the user added the identifier interactively via `identity add`. - `config_migration` — the identifier was inserted by the one-time @@ -312,12 +316,17 @@ copy win on payload richness silently changes how the archive reads. preference runs in this order: 1. Source preference (when `--prefer` is configured, or the default - order: `gmail,imap,mbox,emlx,hey`). + order: `gmail,imap,msmail,mbox,emlx,hey`). 2. Complete original payload — has raw MIME, then, only when every eligible copy has the same normalized raw MIME hash, more attachments, an attachment-presence signal, and a larger original payload. -3. Source metadata quality — provider IDs, threading info, presence - of Message-ID. +3. Source metadata quality — one point each for a native Gmail, IMAP, or Microsoft Mail + message ID, threading evidence, and RFC822 Message-ID presence. + Threading evidence is a stored Gmail provider thread ID, preserved Google + Groups grouping derived from a valid exported X-GM-THRID, an archived + In-Reply-To header, or a resolved reply parent. Generic fallback conversation + keys do not count. Gmail uses stored conversation IDs for historical and + current rows; an ID equal to the message ID is ambiguous and earns no point. 4. Richer label or folder metadata. 5. Earlier `archived_at` timestamp (when meaningful). 6. Stable row ID, as the final tie-breaker. @@ -673,7 +682,7 @@ Find duplicate messages and (with `--undo`) reverse a previous run. | ----------------------------------- | --------- | ----------------------------- | ------------------------------------------------------------------ | | `--dry-run` | bool | `false` | Scan and report only. | | `--no-backup` | bool | `false` | Skip the pre-execute database backup. | -| `--prefer <list>` | string | (none) | Source-type preference order for survivor selection. When the flag is empty, implementations fall back to the documented default order: `gmail,imap,mbox,emlx,hey`. The fall-through gives a single source of truth for the default — implementations should not register the literal default string as the cobra-layer default. | +| `--prefer <list>` | string | (none) | Source-type preference order for survivor selection. When the flag is empty, implementations fall back to the documented default order: `gmail,imap,msmail,mbox,emlx,hey`. The fall-through gives a single source of truth for the default — implementations should not register the literal default string as the cobra-layer default. | | `--content-hash` | bool | `false` | Run the second-pass content-hash detection. | | `--undo <batch-id>` (repeatable) | string... | (none) | Reverse one or more named batches. | | `--account <name>` | string | (none) | Per-source scope. | diff --git a/docs/internal/web-ui-overhaul-design.md b/docs/internal/web-ui-overhaul-design.md new file mode 100644 index 000000000..e2fc74b49 --- /dev/null +++ b/docs/internal/web-ui-overhaul-design.md @@ -0,0 +1,612 @@ +# Web UI overhaul + +Status: approved 2026-09-28 and revised 2026-09-29 after review. PR 1 +(foundation and shell) is merged (#995), and PR 2 (Everything, Files, and +Saved views) is merged (#1009), and PR 3 (People) is merged (#1019). PR 4 +(Manage) is specified in [its spec](web-ui-overhaul-pr4-spec.md). Delivery is four stacked +pull requests, described under +[Delivery](#delivery). This record describes the intended end state; the +current source remains authoritative until each pull request lands. + +## Summary + +The Web UI exposes a large, capable archive, but its screens grew one feature +at a time. Ten peer tabs share one centered top bar, each workspace invents its +own header and toolbar, several controls do nothing, and many labels are raw +API codes. People who use msgvault every day have to learn each screen +separately. + +This overhaul gives every workspace the same shell, page structure, visual +language, and status vocabulary. It keeps every existing capability. Controls +move to predictable places; none are removed. The visual language follows the +docbank web restyle (kenn-io/docbank#722) so the kenn tools feel like one +family. + +## Problems observed + +Observed on `main` at 5d1ce392 with the Enron docs fixture at 1440×900 and +420×860, light and dark. + +- **Navigation has no hierarchy.** Relationships, Directory, Reviews, + Everything, Files, Saved Views, Sources, Operations, Deletions, and Settings + are equal-weight tabs. On a phone they collapse into a dropdown beside the + density selector. +- **Each page invents its header.** Sources, Deletions, and Saved Views show an + orange "ARCHIVE WORKSPACE" eyebrow; Operations shows "ARCHIVE OPERATIONS"; + Everything and Directory show none. Saved Views uses a centered column while + other pages fill the width. In Files the page title sits below its toolbar. +- **Everything stacks four control strips.** Search, the context bar, an + always-visible "No items selected" bar, and a Columns strip sit above the + results, with a hard-coded keyboard-hint footer below. The result count + appears twice ("20 items" and "20 results"). +- **Files is two different views.** Everything's "Show as: Files" renders + `explore/FilesPresentation.svelte` (grid "Files in current context"). The + Files workspace renders `FilesWorkspace` (grid "Files results") with other + columns and filters. In the Files workspace, "Show as: Table" silently moves + to Everything. +- **Directory shows seven inline filters.** The date filters are free-text + `YYYY-MM-DD` fields whose placeholder is truncated; an invalid date is + silently ignored. +- **Accent colors compete.** Search is blue, Save is purple, Save settings is + green, and eyebrows are orange. Timestamps and counts use monospace as + decoration. +- **Status is alarming or raw.** Operations shows a red dot for every feature + that is simply not configured. Sources shows `source_not_schedulable` in red + as its "Action". Deletions shows "No deletion manifests yet" in an orange + warning box. Raw codes also appear in Files (MIME families), Operations + (error codes), Deletions (reasons, ISO timestamps), Tasks, Meetings, and the + context-bar crumbs (`full_text`). +- **Some controls do nothing.** "Newest first" and the `s`/`r` shortcuts only + announce that sort is fixed; in Files the button has no handler. + "Open selection in source" never receives a handler. Directory → Media & + Files mounts `FilesWorkspace` without change handlers, so its filename + filter, type checkboxes, and sort headers are inert. Saved Views declares an + unused `selection` prop. `search/SearchBar.svelte` is imported only by its + test. +- **Settings contradicts itself.** Appearance says "Changes apply right away" + above a Save settings bar. The phrase means "no daemon restart after saving". + The shell reads `web.theme` and `web.density` only at sign-in, so saving them + may not update the open tab. +- **Useful actions are hidden.** Staging a deletion is reachable only with `d` + or `D` in Everything. Tasks are a collapsed disclosure in the reading pane. + "Save this view" lives on the Saved Views page, away from the view it saves. +- **Cross-links are missing.** Directory cannot open a person's Relationships + view. Nothing links to Reviews → Facts for a person, and Facts with no person + selected is a dead end. +- **The browser tab title is always "Everything · msgvault".** + +## Goals + +- One navigation model, one page structure, and one toolbar pattern across + every workspace. +- One accent color. Status colors mean status only. +- Human-readable labels for every code the UI displays, with the raw code kept + in a tooltip or detail view for diagnosis. +- Every existing capability reachable, with a visible entry point for actions + that are keyboard-only today. +- Existing URLs, URL state keys, API calls, and accessible names preserved + unless this document names the change. +- Light and dark themes, both densities, keyboard use, and phone widths work on + every screen. + +## Non-goals + +- New archive capabilities. The follow-ups listed below need backend work and + ship separately. +- Changes to the TUI, CLI, API, or MCP server. +- Replacing kit-ui components with local ones. Where kit-ui lacks a token, + `app.css` overrides the kit class, as docbank does. + +## Visual language + +These rules match docbank#722. + +- **Palette.** A new `web/palette.css` retunes kit-ui tokens: neutral surfaces, + one blue accent, and green, amber, and red for status only. It lives outside + `src/` because `kit-ui-check` guards component styles, not palette files. + `src/styles/tokens.css` keeps the semantic aliases it defines today. +- **Buttons.** One solid primary action per screen, in blue. Purple and green + are no longer button colors. Destructive confirmation buttons are red; the + button that opens a destructive review is not. +- **Type.** Platform system fonts. Sentence case for every label, heading, + table header, and chip ("Saved views", not "Saved Views"; no uppercase + letter-spaced section labels). `[data-section-label]` becomes sentence case. +- **Monospace** is for identifiers, hashes, code, keys, and cron text only. + `[data-mono]` becomes tabular-figure sans, so timestamps and counts stop + switching typeface. Identifiers that use `[data-mono]` today move to + `<code>` or `[data-metadata]`, which stay monospace. +- **Status vocabulary.** Every status dot and chip uses one mapping: + + | Meaning | Tone | Examples | + |---|---|---| + | Healthy or finished | green | Succeeded, Completed, Ready | + | In progress | blue | Running, Syncing, Queued | + | Needs attention | amber | Partial, Stale, Conflict | + | Failed | red | Failed | + | Off or not set up | gray | Not configured, Disabled, Cancelled | + +- **Empty states** use kit `EmptyState` in a neutral tone and name the next + step. A missing item is never a warning. +- **Code labels.** Each enum the UI displays gets one label map next to the + component that renders it. Unknown codes fall back to a sentence-cased form of + the code. The raw code stays available in a tooltip or detail panel. + +## Information architecture + +### Sidebar + +A labeled left sidebar replaces the top-bar tabs. It is a `nav` still named +"Primary", and the active item keeps `aria-current="page"`, which history focus +restoration depends on. + +| Group | Items (icon) | Workspace id | +|---|---|---| +| People | Relationships (users), Directory (contact), Reviews (check-check) | `relationships`, `directory`, `directory_review` | +| Archive | Everything (inbox), Files (paperclip), Saved views (bookmark) | `everything`, `files`, `saved_views` | +| Manage | Sources (plug), Operations (activity), Deletions (trash-2), Settings (settings) | `sources`, `operations`, `deletions`, `settings` | + +People comes first because Relationships is the default landing workspace. +Icons come from `@lucide/svelte`, which is already a dependency; final icon +choices may change during implementation. + +- **Layout.** PR 1 adds a local `AppSidebar` component and owns all of the + behavior below. kit `CollapsibleSidebar` is not used: its collapsed branch + renders only an expand toggle, and its narrow-screen overlay positions the + sidebar without a scrim or focus management. The sidebar has a fixed width + and no resize handle. +- **Expanded and rail modes.** Expanded, the sidebar is about 232px with group + headings. Collapsed, it is an icon rail about 56px wide. Each rail item shows + a kit `Tooltip` with its label and keeps its full accessible name. A toggle at + the bottom of the sidebar switches modes. The mode is stored per browser in + localStorage `msgvault.sidebar.collapsed`. Group headings are hidden in the + rail; a divider separates the groups. +- **Narrow screens.** Below 900px the sidebar is hidden and a menu button + labeled "Open navigation" appears at the start of the top bar. It opens the + expanded sidebar as a modal slide-out menu: + - A scrim covers the page. Selecting the scrim, pressing Escape, or choosing + a navigation item closes the menu. + - Focus moves to the current navigation item when the menu opens. kit + `trapFocus` keeps focus inside the menu while it is open, and the page + behind it is `inert`. + - Closing the menu returns focus to the menu button, except after choosing an + item, when focus follows the existing workspace-change focus rules. + - The narrow-screen menu ignores the rail setting and always shows labels. + - Escape closes the menu before any other Escape handler runs, so it never + also closes the reading pane or a grouping level. +- **Footer.** The archive status indicator (dot plus "Local archive", + "Searching", or "Attention") and a "Keyboard shortcuts" entry showing `?` + move to the sidebar footer. +- **Tab title.** The browser tab title follows the workspace, for example + "Directory · msgvault". + +### Top bar + +The top bar holds only global search and appearance controls. + +- **Global search.** The search form moves from Everything into the top bar and + reuses `search/SearchBar.svelte` (today imported only by its test). It keeps + the names form "Search Everything", searchbox "Search everything", radiogroup + "Search mode" with "Full text", "Semantic", and "Hybrid", and button + "Search". Query and mode state are unchanged (`query` in the `explore` URL + state, top-level `mode`, localStorage `msgvault-search-mode`). + - On Everything and Files, typing updates the current view as it does today. + - On any other workspace, typing is a local draft; Enter or Search commits the + query and opens Everything. + - `/` focuses the global search from any workspace. AppShell's `searchInput` + reference binds to this input, so "Refine search" and the saved-view focus + fallback keep working. + - Below 900px the mode control becomes a compact select inside the search + field; the radio semantics are kept at wider widths. +- **Theme.** kit `ThemeToggle` stays, matching the other kenn tools. +- **Display menu.** A new "Display" menu button holds the per-tab density + override (Auto, Compact, Comfortable) and "Use daemon theme". Both keep their + sessionStorage behavior (`msgvault.appearance.override`). The combobox + "Temporary density" becomes a radio group named "Temporary density" inside + this menu. + +### Page structure + +A shared local `PageHeader` component renders the title (h1), an optional +one-line description, right-aligned actions, and an optional row below for view +tabs. Every workspace uses it. Eyebrows are removed. Horizontal padding is the +same on every page. Data workspaces fill the width; form pages (Settings, +Saved views editing) use the same left edge with a maximum content width. + +- **View switches** (different views of the same data, such as Messages or + Files, and review type) use kit `SegmentedControl`. +- **Record sections** (sections of one record, such as person detail) use the + ARIA tablist pattern that `PersonDetail` already implements. + +### Toolbars + +Data workspaces use one toolbar row: search or filter input first, then +Filters, view options, and sort, with the result count at the right edge. When +filters, groupings, or a query are active, a second line shows them as +removable chips with readable labels. Notices (semantic coverage, "More results +may match") appear between the toolbar and the results only when they apply. + +### Keyboard + +The hard-coded footer in Everything and the key badges in the selection bar are +removed. The existing "Keyboard shortcuts" dialog, generated from +`lib/commands/registry.ts`, becomes the single reference. Before removing the +footer, the registry gains the entries only the footer documents today: +Shift+Space (extend selection) and plain `a` (select visible rows). + +## Workspaces + +### Everything + +``` +Everything [Save view…] +[Filters 2] [Show as: Table] [Group by] [Sort: Newest first] [Columns] [▭|▯] 20 items +Full text: "network" × Source: example@example.com × Grouped by Year × +┌ results ────────────────────────────────────────────────────────────────┐ +├ reading pane ─────────────────────────── [Open relationship] [Tasks] [×] ┤ +└─────────────────────────────────────────────────────────────────────────┘ + ┌ 3 selected · Select all 20 matching · Export · Review for deletion… · Clear ┐ +``` + +- **Toolbar.** Filters, Show as, Group by, Sort, and Columns sit in one row. + - Columns moves from a `<details>` strip into a menu with the same seven + checkboxes and the same `columns` URL state. + - Preview position moves into the toolbar as a segmented control that keeps + the text options "Below" and "Right" and the radiogroup name "Preview + position". It still appears only when the results are at least 960px + wide. + - The count appears once, at the right edge. +- **Sort.** "Newest first" becomes a menu listing the one supported order, + marked "Other orders aren't available yet". Its accessible name is + "Sort: Newest first" (previously "Sort: newest first"), and `s` opens it. `r` announces "Everything is always + shown newest first." While grouped, the Sort menu is hidden, `s` does + nothing, and `r` announces that sorting isn't available while grouped. +- **Context chips.** The query, filters, and groupings appear as removable chips + with readable labels ("Full text", "Source"). Grouping chips and saved-view + summaries use the same wording, such as "Grouped by Person". Removing a + grouping keeps the name "Remove {label} grouping". + - Known gap: chips and saved-view summaries show a source filter by its ID + (for example "Source: 7") until source names are resolved. +- **Selection bar.** The existing `SelectionBar` moves below the results as a + compact action strip. It appears only while at least one row is + selected. kit `BottomDock` is not used, because it is a resizable panel that + opens at half the viewport height. The bar keeps "Select all N matching + items", "Export selection", meeting-context export, and "Clear selection". + - It adds **Review for deletion…**, which starts the same flow as `d` (or `D` + in all-matching mode): `openDeletionReview` switches to Deletions and runs + the preflight. The button doesn't yet show the preflight's + `stage_deletion` reason when deletion is unavailable; that arrives with + the Deletions work in pull request 4. + - "Open selection in source" moves to an overflow menu. It is shown disabled + with its reason as a sentence, which matches today's behavior because no + handler exists. +- **Reading pane.** "Tasks for this message" becomes a header button labeled + "Tasks", without a count, that opens the same `TaskLinks` sheet. It keeps the + accessible name "Tasks for this message". + "Close" becomes an icon button that keeps the name "Close reading pane". The + meta strip uses readable labels. + +### Files + +Files becomes the single file view. + +- Everything's "Show as: Files" opens the Files workspace with the same query, + filters, and groupings. Files shows the same "Show as" control; choosing + Table or Timeline opens Everything. Switching is symmetric and visible. +- `explore/FilesPresentation.svelte` is deleted. Its per-row "Open containing + item" action moves into the Files grid as a row action. +- **Existing links and history.** Explore state can encode + `workspace: 'everything'` with `presentation: 'files'` in shared links, + browser history entries, and saved views. `normalize()` in + `src/lib/explore/state.svelte.ts` maps that combination to + `workspace: 'files'` and keeps the query, search mode, filters, grouping + chain, and columns. URL parsing, every commit, popstate restoration, and + saved-view navigation all pass through `normalize()`, so one rule covers + every entry path: + - Opening an old link lands in Files with the same context. The address bar + is rewritten with a history replace, so no extra history entry appears. + - Back and Forward into an old history entry restore Files with the same + context and the entry's keyboard focus and scroll state. + - An attachment selection (`selectedRow` of the form `attachment:<id>`) + reopens the same file viewer in Files. + - In the Files workspace, `presentation` is always `files`. Choosing Table or + Timeline sets `workspace: 'everything'` with that presentation. +- Saved views with `presentation: 'files'` open in the Files workspace through + the same rule. +- **Toolbar.** Files has two toolbar rows. The first is the shared toolbar: + Filters, Show as, Group by, and a **Sort** menu (Date, Filename, Size). The + second holds Filename search, a **Type** menu, and a **Visual search** + toggle. + - The Type menu replaces the eight raw MIME-family checkboxes with Images, + PDFs, Audio, Video, Text, Documents, Archives, and Other. URL state + `fileMIMEFamilies` is unchanged. With no type selected, the menu reads + "All types" and its items carry no "Not included" description. + - Column-header sorting still works and stays in sync with the Sort menu + (`fileSort`). + - Turning on Visual search adds the visual query, query image, and the + unchanged provider disclosure to the second row. +- **Columns.** Type shows a readable name ("PDF", "PNG image"); the raw MIME + type moves to the cell tooltip and the file viewer. Source shows the account + display name when one is available. +- **Fixes.** The Sort control works (it has no handler today). The count comes + from the Files query instead of "Count pending". Directory → Media & files + passes change handlers so its filters and sorting work. + +### Saved views + +- **Save view…** is a header action in Everything and Files. It opens a dialog + with Name and Description and saves the current view through the existing + saved-views API. It keeps the field names "Name" and "Description" and the + submit button "Save". In Files, the dialog states: "Filename, type, and file + sort aren't saved with the view." This limitation goes away only with the + backend follow-up below. +- The Saved views page becomes a library. Each view shows its name, + description, a readable summary of its query, filters, grouping, and layout, + and the actions "Open {name}", "Edit {name}", and "Delete {name}". The empty + state points to Save view… in Everything and Files. +- The unused `selection` prop is removed. + +### Relationships + +- The list search is relabeled "Filter people and domains" (placeholder) to + separate it from global search; its accessible name "Search people and + domains" is unchanged. +- The person header groups "Open in Directory" and "Same person…" as one action + set beside the Messages | Files view switch. +- On narrow screens the list-drawer button reads "People" instead of + "Contacts". + +### Directory + +- **Toolbar.** Search, a **Filters** popover, and a **Sort** menu. + - The popover holds contact state, category, organization, primary channel, + and last contacted. The URL keys (`directoryContactState`, + `directoryCategory`, `directoryOrganization`, `directoryPrimaryChannel`, + `directoryLastContactAfter`, `directoryLastContactBefore`) and API + parameters are unchanged. + - "Last contacted after" and "Last contacted before" stay two independent + fields, because either boundary can be set alone today. Each becomes a + native `<input type="date">` with a clear button, keeping its accessible + name and URL key. The native input produces only valid `YYYY-MM-DD` + values, so invalid dates can no longer be silently ignored. kit + `DateRangePicker` is not used: it commits only completed custom ranges. + - A one-sided filter shows as one chip, such as "Last contacted after + Jan 5, 2024". Removing it clears only that boundary. + - Sort keeps the three orders and the `directorySort` URL key. + - Active filters show as removable chips. +- **Person detail sections** (tablist "Person detail sections"): + + | Section | Contents | + |---|---| + | Overview | "Last time we talked" brief, agenda, contact state, activity, meeting activity | + | Profile | Structured profile (names, contact points, addresses, dates, categories, media), attributes, profile history | + | Organizations | Unchanged | + | Connections | Curated person-to-person relationships; renamed from "Relationships" to avoid confusion with the Relationships workspace | + | Network | Unchanged | + | Media & files | Person-scoped files, now with working filters and sort | + | Maintenance | Profile-maintenance tracking, CardDAV publication, merge history and split | + +- **Header actions.** + - **Open relationship** opens the person's Relationships view through the + shell's existing `openRelationship(participantID)`, using the person's + `participant_ids` from the person response. It appears only when that list + is not empty. PR 3 confirms which entry opens the person's relationship + cluster when a person has several participants. + - **Review facts** opens Reviews → Facts with `directoryPersonID` set. + - An overflow menu holds "Rename person", "View profile history", and "Delete + person". Rename and delete keep their existing confirmation steps and + accessible names. +- **Partial dates in editors** stay text fields because a date picker cannot + express `YYYY` or `YYYY-MM`. They get inline validation messages. + +### Reviews + +- One view switch under the header: Identity matches, Facts, Imported + relationships. It keeps the radiogroup "Review type" and the `reviewKind` URL + key. +- The second header ("Identity matches" with its own segmented control) is + removed. Each queue's status filter becomes a **Show** menu in its list + toolbar, keeping the names "Identity review state" and "Imported + relationship review state" and the URL keys `identityState` and + `relationshipReviewState`. +- Facts with no person selected shows a person picker that uses the existing + directory search, instead of only linking to Directory. + +### Sources + +- **Header.** Title "Sources", description "Accounts and imports in your + archive, and when they last synced." The action "View source operations" is + renamed **Sync history** and still opens Operations filtered to source sync. +- **Table.** + - Source: display name and a readable source type ("Mbox import", "Gmail"). + - Schedule: a sentence from CronField's summary logic, with the cron text in + a tooltip. + - Status: a compact chip. + - Last successful sync: unchanged. + - Action: "Sync now {name}", or a muted reason: + + | Code | Label | + |---|---| + | `source_not_schedulable` | Imported file — nothing to sync | + | `sync_already_running` | Sync in progress | + | `scheduler_unavailable` | Scheduler unavailable | + | `sync_not_configured` | Sync not set up | + | `sync_unavailable` | Sync unavailable | + +- Error messages, item errors, and scheduler errors move to an expandable row + detail. `sync_start_not_observed` and `stale_last_result` get sentences. + +### Operations + +- **Lane summary.** The five lane cards become one compact status list: one row + per operation kind, grouped by lane, showing the status tone, the latest + run's time, and the row's actions. + - "Not configured" shows as gray **Off** with a **Set up** link to the related + setting. + - "History available" is no longer printed; "History unavailable" still is. + - Related-status buttons ("Open Sources status" and the others) and actions + ("Start CardDAV sync", "Build visual index", "Resume visual index") keep + their names. +- **Toolbar.** Lane, Kind, State, and the date range stay in one row with their + URL keys. "Refresh operations" becomes kit `RefreshControl` in the header. +- **Runs.** Counters read "20 messages processed · 20 added". An unknown + trigger shows "—" instead of "Unspecified". Each `OperationPublicErrorCode` + maps to a sentence; the detail panel keeps the raw code. + +### Deletions + +- **Header.** Title "Deletions", description "Deletions you've staged. Nothing + is deleted until you run `msgvault delete-staged`." +- **Review.** The review panel appears when a selection arrives from + Everything. Without one, an empty state explains: "Select items in + Everything, then choose Review for deletion…". The summary shows expiry as + relative time and each unavailable reason as a sentence. Only "Confirm stage + deletion" is red. +- **Manifests.** A table of ID (monospace), description, items, status chip + (Pending, In progress, Completed, Failed, Cancelled), and the row actions + "Inspect {id}" and "Cancel {id}". The detail opens beside the table. + +### Settings + +- The category list stays as the page's second-level navigation under a normal + page header. The selected category is stored in a new `settingsCategory` + field of the `explore` URL state, so reload and Back keep the category. + `settingsAuthority` deep links keep working. +- **Save model.** + - Catalog categories show the save bar only when drafts exist, with + "Discard" and "Save changes" in blue. The status text "N unsaved changes" + stays. + - Each category states when saved changes apply: "After the daemon restarts", + or for Appearance "Right away — no restart needed". + - Controls that save through their own endpoints (provider credentials, + enrichment providers, CardDAV account, People sweep) say "Saves + immediately" beside their buttons. +- **Appearance.** After a successful save, SettingsWorkspace passes the saved + `web.*` values to App through a callback, and App updates its defaults. + - **Theme and density** apply to the open tab right away, unless a Display + menu override is active in this tab. The override still wins, as it does + today; the note under Appearance explains that "Use daemon theme" and + "Density: Auto" return to the saved default. + - **Default search mode** changes only future searches, not the current one. + Search mode is resolved in this order: explicit URL mode, then this + browser's remembered mode (localStorage `msgvault-search-mode`), then the + daemon default. Saving does not change the open view's mode or its URL, + so results on screen and shared links stay stable. Saving does replace + this browser's remembered mode with the saved value, so the next tab + opened here without a mode in its link uses the new default. Other + browsers keep their remembered mode until someone changes the mode there. + The setting's description says: "Used when a tab opens without a search + mode in its link. Your current search keeps its mode." +- The plain-HTTP warning stays as a compact kit `Notice`. + +### Sign-in and boot screens + +The login, connecting, connection-error, and OAuth-callback screens use the +same palette and type. + +## Control relocation + +Every control not listed here keeps its location, label, and accessible name. + +| Control | Today | After | +|---|---|---| +| Workspace tabs | Top bar center, nav "Primary" | Sidebar, nav "Primary"; icon rail when collapsed; modal menu below 900px | +| Archive status | Top bar right | Sidebar footer | +| Temporary density | Top bar combobox | Display menu radio group | +| Use daemon theme | Top bar button | Display menu item | +| Search form, mode, Search button | Everything search row | Top bar | +| Preview position | Everything header, text plus segmented control | Everything toolbar, text segmented control | +| Result count | Everything header and context bar | Toolbar right edge, once | +| Columns | `<details>` strip above the table | Toolbar menu | +| Newest first | Button that only announces | Sort menu with the one supported order | +| Keyboard hints | Everything footer, selection bar badges | Keyboard shortcuts dialog; sidebar footer entry | +| Selection bar | Always visible above results | Strip below results while a selection exists | +| Stage deletion entry | `d` / `D` only | Also "Review for deletion…" in the selection bar | +| Open selection in source | Selection bar, reason text | Selection bar overflow, disabled with sentence reason | +| Tasks for this message | Collapsed disclosure | Reading-pane header button "Tasks" | +| Close reading pane | Text button | Icon button, same name | +| Show as: Files | Everything-only files grid | Opens the Files workspace; old links and history entries normalize there | +| File type filter | Eight raw checkboxes | Type menu with readable names | +| File sort | Column headers only | Sort menu and column headers | +| Save this view | Saved Views page form | "Save view…" in Everything and Files headers | +| Directory filters | Seven inline controls | Filters popover and Sort menu | +| Directory date filters | Two `YYYY-MM-DD` text fields | Two independent native date fields in the Filters popover | +| Person detail Overview sections | One long Overview | Overview, Profile, Maintenance sections | +| Person "Relationships" tab | Record tab | Renamed "Connections" | +| Rename, profile history, delete person | Structured profile section | Person header overflow menu | +| Review state filters | Second header segmented control | Show menu in the queue toolbar | +| Sources header action | "View source operations" | "Sync history" | +| Refresh operations | Header button | Header `RefreshControl` | +| Settings category | Transient | `settingsCategory` URL state | + +New entry points: Directory → Relationships ("Open relationship"), Directory → +Reviews Facts ("Review facts"), and a person picker in Facts. + +## Tests and accessible names + +- Accessible names stay the same wherever the control survives in the same + form. The kit test helper `selectKitTopBarTab` in `web/tests/kit-ui.ts` + changes to click the sidebar item in nav "Primary". +- Expected test edits, listed in each pull request: + - The combobox "Temporary density" becomes a menu radio group + (`density-restoration`, `theme-keyboard`, `docs-fixture-screenshots`, + `e2e/accessibility`). + - The grid "Files in current context" and region "Files presentation" are + replaced by the Files grid (`presentations`). + - Visible labels change to sentence case, such as "Saved views". + - The person tab "Relationships" becomes "Connections", and Overview + sections move (`e2e/directory`, `directory-network`, `directory-review`, + `e2e/accessibility`). + - "View source operations" becomes "Sync history" (`operations`). +- Each pull request adds tests for the behavior it changes: the Save view + dialog and its Files note, the deletion entry in the selection bar, and the + Type and Sort menus. They also cover: + - Files round trips, including an old `everything` + `files` link, Back and + Forward into an old history entry, and an old saved view, each landing in + Files with the query, filters, grouping, and attachment selection intact. + - The narrow-screen menu: scrim, Escape, and item selection close it; focus + starts on the current item, stays inside while open, and returns to the + menu button; Escape does not also close the reading pane. + - The icon rail: each item keeps its accessible name, and the mode survives + a reload. + - Directory dates: setting and clearing each boundary alone. + - The Settings category URL. + - Appearance saves: theme and density update the open tab unless a Display + override is active; saving the default search mode leaves the open view's + mode and URL unchanged and changes the mode a new tab opens with. +- `e2e/accessibility` (axe) runs on every workspace in both themes. +- Each pull request includes before and after screenshots from the real daemon + with the Enron docs fixture, at desktop and phone widths, in light and dark. + The docs screenshots (`docs/screenshots/generate-web-fixture-screenshots.sh`) + are regenerated in the final pull request. + +## Delivery + +Four stacked pull requests. Each leaves the application shippable. + +1. **Foundation and shell.** Palette, type, and status vocabulary; the local + `AppSidebar` with its icon rail, narrow-screen modal menu, scrim and Escape + dismissal, focus trap, and focus restoration; top bar, global search, + Display menu; `PageHeader` applied to every workspace; empty-state + conventions; keyboard registry additions and footer removal; tab titles; + sign-in and boot screens. +2. **Everything, Files, and Saved views.** Toolbar, context chips, Sort and + Columns menus, selection bar, reading-pane header, Files unification with + the `normalize()` rule for old links, history, and saved views, Files + fixes, Save view dialog, Saved views library. +3. **People.** Relationships header, Directory filters and person sections, + cross-links, Reviews structure and fact person picker. +4. **Manage.** Sources, Operations, Deletions, Settings, and the code-label + maps they need; docs screenshots and the [Web UI guide](../web-ui.md). + +## Follow-ups + +These need backend changes and are not part of this overhaul. + +- **Reviews count.** No API returns a pending review count, so the sidebar shows + no badge. A count endpoint would enable one. +- **Files fields in saved views.** The saved-view schema has no fields for the + filename filter, type filter, or file sort, so saving a Files view keeps only + the shared context. +- **Sort orders.** Everything supports only newest first. +- **Open in source.** The daemon always reports + `open_in_source:trusted_source_link_unavailable`. diff --git a/docs/internal/web-ui-overhaul-pr1-plan.md b/docs/internal/web-ui-overhaul-pr1-plan.md new file mode 100644 index 000000000..0d1458ae3 --- /dev/null +++ b/docs/internal/web-ui-overhaul-pr1-plan.md @@ -0,0 +1,2111 @@ +# Web UI overhaul PR 1: foundation and shell — implementation plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use +> superpowers:subagent-driven-development (recommended) or +> superpowers:executing-plans to implement this plan task by task. Steps use +> checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace the ten-tab top bar with a grouped, collapsible sidebar and a +slim top bar that holds global search, and give every workspace the same page +header, palette, and type rules, without removing any capability. + +**Architecture:** New focused shell components (`AppSidebar`, +`NavigationDrawer`, `DisplayMenu`, `PageHeader`, and the reworked `SearchBar`) +live in `web/src/lib/components/shell/` and `web/src/lib/components/search/`. +`AppShell.svelte` composes them and keeps ownership of navigation state, +shortcuts, and focus. A palette file outside `src/` retunes kit-ui tokens. + +**Tech stack:** Svelte 5 (runes), `@kenn-io/kit-ui` at pin `e43c820f`, +`@lucide/svelte` 1.26.0, Vitest with Testing Library (jsdom), Playwright. + +**Spec:** [Web UI overhaul](web-ui-overhaul-design.md). Read its "Visual +language", "Information architecture", and "Control relocation" sections before +starting. This plan covers delivery item 1 only. + +## Global constraints + +- Keep every capability. A control that moves keeps its accessible name unless + the spec names the change. +- Keep URL state keys, the `explore` JSON format, and API calls unchanged. +- No new npm dependencies. Icons come from `@lucide/svelte` + (`@lucide/svelte/icons/<name>`). +- One solid primary button per screen: `tone="info" surface="solid"`. +- Sentence case for every visible label, heading, table header, and chip. +- Monospace only for identifiers, hashes, code, keys, and cron text. +- Raw colors appear only in `web/palette.css`. Components use tokens; + `bun run check:kit-ui` enforces this for `src/`. +- Tests use Vitest `expect` and Testing Library for unit tests, and Playwright + for browser tests. Prefer role and accessible-name queries. +- Run commands from `web/` unless a step says otherwise. +- Commit after each task with the `kenn-io-commit` skill. The repository is + public: run the private-data scan before each commit. + +## File structure + +| File | Responsibility | +|---|---| +| `web/palette.css` (new) | kit-ui token values: neutral surfaces, one blue accent, status colors, system fonts, type scale, light and dark | +| `web/src/main.ts` | Imports the palette after kit-ui's theme and before `app.css` | +| `web/src/styles/tokens.css` | Semantic aliases; `[data-mono]` and `[data-section-label]` rules | +| `web/src/app.css` | Base element rules and the few kit-class overrides kit has no token for | +| `web/src/lib/commands/registry.ts` | Adds the Shift+Space command so the dialog documents it | +| `web/src/lib/explore/state.svelte.ts` | Adds `commitSearchIn(workspace, query, mode)` | +| `web/src/lib/components/search/SearchBar.svelte` | Global search form: live or draft mode, narrow layout | +| `web/src/lib/components/shell/navigation.ts` (new) | Sidebar groups, labels, icons, and workspace ids | +| `web/src/lib/components/shell/AppSidebar.svelte` (new) | Grouped nav, icon rail, footer (status and shortcuts), collapse toggle | +| `web/src/lib/components/shell/NavigationDrawer.svelte` (new) | Narrow-screen modal: scrim, Escape, focus trap, focus restoration | +| `web/src/lib/components/shell/DisplayMenu.svelte` (new) | Per-tab density override and "Use daemon theme" | +| `web/src/lib/components/shell/PageHeader.svelte` (new) | Title, description, actions, and optional view row | +| `web/src/lib/components/shell/AppShell.svelte` | Composes the shell; wires search, sidebar, drawer, title | +| `web/src/lib/components/shell/EverythingWorkspace.svelte` | Loses its search form and keyboard footer; uses `PageHeader` | +| `web/src/lib/components/explore/SelectionBar.svelte` | Loses its key badges | +| Each workspace component | Uses `PageHeader`; loses eyebrows; sentence case | +| `web/src/App.svelte`, `web/src/lib/components/auth/Login.svelte` | Boot and sign-in screens use the palette and lose eyebrows | +| `web/tests/kit-ui.ts` | Navigation and display-menu helpers for Playwright | + +--- + +### Task 1: Palette and type rules + +**Files:** +- Create: `web/palette.css` +- Modify: `web/src/main.ts`, `web/src/styles/tokens.css`, `web/src/app.css` +- Modify (uppercase removal): `DeletionsWorkspace.svelte:462`, + `OperationRunDetail.svelte:152`, `OperationsWorkspace.svelte:377`, + `OperationRelatedStatus.svelte:167`, `CardDAVConflicts.svelte:308`, + `SourcesWorkspace.svelte:473`, `FilesPresentation.svelte:357`, + `GroupTable.svelte:437`, `IdentityCandidateCard.svelte:89`, + `EverythingTable.svelte:693`, `KeyboardHelp.svelte:78`, + `RelationshipReviewCard.svelte:61`, `FilesWorkspace.svelte:1107`, + `PersonTimeline.svelte:54`, `SavedViewsWorkspace.svelte:361` + +**Interfaces:** +- Produces: kit tokens with new values; `--nav-width: 232px`, + `--nav-rail-width: 56px`, `--nav-active-bg`, `--header-height: 48px`. + Later tasks read these tokens. + +CSS has no meaningful unit test. This task is verified by `check:kit-ui`, +the build, and screenshot review in Task 13. + +- [ ] **Step 1: Capture before screenshots** + +The fixture daemon from the design review runs at `http://127.0.0.1:18733`. If +it is not running, start it as described in +`docs/screenshots/generate-web-fixture-screenshots.sh`, with an isolated +`--home`. Save captures of all ten workspaces at 1440×900 light, 1440×900 dark, +and 420×860 light into the scratchpad `before/` directory, using the script +from the design review (`shoot.mjs`). + +- [ ] **Step 2: Create `web/palette.css`** + +```css +/* + * msgvault retunes kit-ui's tokens for a quiet archive workspace: platform + * system fonts, neutral surfaces, one blue accent, and green, amber, and red + * for status only. kit-ui components read only these tokens, so declaring + * them after kit-ui/theme.css restyles every component consistently. + * + * This is the palette definition file, so raw colors belong here. It lives + * outside src/ because kit-ui-check guards component styles, not palettes. + */ +:root { + --bg-primary: #f7f8fa; + --bg-surface: #ffffff; + --bg-surface-hover: #f2f4f7; + --bg-inset: #f4f5f7; + --border-default: #dde1e6; + --border-muted: #eceef1; + + --text-primary: #1b1f24; + --text-secondary: #464e58; + --text-muted: #666e79; + + --accent-blue: #0061d5; + --accent-amber: #b45309; + --accent-purple: #6d4ad6; + --accent-green: #047857; + --accent-red: #c62828; + --accent-teal: #0e7490; + + --nav-active-bg: #e6eefb; + + --shadow-sm: 0 1px 2px rgba(16, 24, 40, 0.05); + --shadow-md: 0 4px 12px rgba(16, 24, 40, 0.08); + --shadow-lg: 0 16px 40px rgba(16, 24, 40, 0.16); + --overlay-bg: rgba(16, 24, 40, 0.32); + + --radius-sm: 4px; + --radius-md: 6px; + --radius-lg: 8px; + + --font-sans: -apple-system, BlinkMacSystemFont, "Segoe UI", system-ui, Roboto, + "Helvetica Neue", Arial, sans-serif; + --font-mono: ui-monospace, "SF Mono", SFMono-Regular, Menlo, Consolas, + "Liberation Mono", monospace; + --letter-spacing-label: 0; + + --font-size-2xs: 0.6875rem; + --font-size-xs: 0.75rem; + --font-size-sm: 0.8125rem; + --font-size-md: 0.875rem; + --font-size-lg: 0.9375rem; + --font-size-xl: 1.125rem; + --font-size-2xl: 1.5rem; + + --header-height: 48px; + --nav-width: 232px; + --nav-rail-width: 56px; +} + +:root.dark { + --bg-primary: #151618; + --bg-surface: #1c1d20; + --bg-surface-hover: #26282b; + --bg-inset: #222427; + --border-default: #34373c; + --border-muted: #2a2c30; + + --text-primary: #ececee; + --text-secondary: #b8bcc3; + --text-muted: #8f949c; + + --accent-blue: #4d9cff; + --accent-amber: #f0a64a; + --accent-purple: #a995f5; + --accent-green: #4cc38a; + --accent-red: #f27474; + --accent-teal: #3fc1d6; + + --nav-active-bg: #243650; + + --shadow-sm: 0 1px 2px rgba(0, 0, 0, 0.3); + --shadow-md: 0 4px 12px rgba(0, 0, 0, 0.35); + --shadow-lg: 0 16px 40px rgba(0, 0, 0, 0.5); + --overlay-bg: rgba(0, 0, 0, 0.55); +} + +/* Repeats kit-ui's handheld type scale, which the base block above would + * otherwise override by source order. */ +@media (hover: none) and (pointer: coarse) { + :root { + --font-size-2xs: 0.75rem; + --font-size-xs: 0.875rem; + --font-size-sm: 0.9375rem; + --font-size-md: 1rem; + --font-size-lg: 1.0625rem; + --font-size-xl: 1.25rem; + --font-size-2xl: 1.75rem; + } +} +``` + +- [ ] **Step 3: Import the palette** + +In `web/src/main.ts`, change the import block to: + +```ts +import '@kenn-io/kit-ui/theme.css'; +import '../palette.css'; +import './app.css'; +``` + +In `web/src/app.css`, delete the line `@import '@kenn-io/kit-ui/theme.css';`, +because `main.ts` now imports it first. + +- [ ] **Step 4: Update the type rules in `web/src/styles/tokens.css`** + +1. Delete the `--text-muted: color-mix(...)` override and its comment. The + palette's `--text-muted` meets WCAG AA on both surfaces. +2. Replace the `[data-mono]` block and its comment with: + +```css +/* Data values (timestamps, counts, sizes) keep the UI typeface with tabular + * figures so columns align. Identifiers use <code> or [data-metadata]. */ +:where([data-mono]) { + font-variant-numeric: tabular-nums; +} +``` + +3. Replace the `[data-section-label]` block and its comment with: + +```css +/* Pane headers, month markers, and group headings: small sentence-case + * labels, never uppercase. */ +:where([data-section-label]) { + color: var(--text-muted); + font-family: var(--font-sans); + font-size: var(--font-size-xs); + font-weight: 600; +} +``` + +- [ ] **Step 5: Move identifiers off `[data-mono]`** + +Run `rg -n 'data-mono' src/lib --glob '*.svelte'`. For each hit, decide: +- The value is an ID, hash, raw code, email address in a code-like position, + or cron text: replace `data-mono` with `data-metadata`. +- The value is a timestamp, count, duration, or size: keep `data-mono`. + +List the files you changed in the commit body. + +- [ ] **Step 6: Remove uppercase styling** + +In each file listed under **Files**, delete `text-transform: uppercase;` and +any `letter-spacing` on the same rule. Where the rule also sets +`font-weight: 800`, change it to `600`. In `OperationsWorkspace.svelte:377` and +`OperationRelatedStatus.svelte:167` also change +`color: var(--status-warning-ink)` to `color: var(--text-muted)`, because +eyebrows are not warnings. Task 10 removes those eyebrows entirely. + +- [ ] **Step 7: Add kit overrides for treatments kit has no token for** + +Append to `web/src/app.css`: + +```css +/* kit-ui has no tokens for these treatments. If kit renames the classes, + * these overrides stop applying; check them when bumping the kit pin. */ +.kit-chip { + text-transform: none; + letter-spacing: 0; +} + +.kit-table th, +.kit-table-header-cell { + text-transform: none; + letter-spacing: 0; + color: var(--text-muted); + font-weight: 600; +} +``` + +Before committing, confirm each selector exists in kit with +`rg -n 'kit-chip\b|kit-table th|kit-table-header-cell' node_modules/@kenn-io/kit-ui/src/lib/components`. +Delete any rule whose selector kit does not use. + +- [ ] **Step 8: Verify** + +Run: `bun run check && bun run check:kit-ui && bun run build` +Expected: all pass with no warnings. + +Capture the ten workspaces again into `after-task1/` and compare with +`before/`. Nothing should be uppercase, and no text should fall below AA +contrast. + +- [ ] **Step 9: Commit** + +Subject: `feat(web): adopt the quiet kenn palette and sentence-case type`. + +--- + +### Task 2: Document Shift+Space in the shortcut registry + +The Everything footer is the only place that documents Shift+Space +(extending a selection). Task 9 removes the footer, so the registry must +document it first. + +**Files:** +- Modify: `web/src/lib/commands/registry.ts` +- Modify: `web/src/lib/components/shell/AppShell.svelte` (`relay`, + `relayGridKey`, `commandHandlers`) +- Test: `web/src/lib/components/shell/KeyboardHelp.test.ts`, + `web/src/lib/components/shell/AppShell.test.ts` + +**Interfaces:** +- Produces: command id `extend-selection`, label + `Extend selection to focused row`, keys `['Shift', 'Space']`, combo + `shift+space`, section `Selection`. + +- [ ] **Step 1: Write the failing tests** + +Add to `KeyboardHelp.test.ts`: + +```ts +it('documents extending a selection with Shift+Space', () => { + const handlers = Object.fromEntries( + COMMAND_DEFINITIONS.map(({ id }) => [id, () => undefined]) + ) as CommandHandlers; + render(KeyboardHelp, { commands: createCommandRegistry(handlers), onclose: () => undefined }); + + expect(screen.getByText('Extend selection to focused row')).toBeTruthy(); +}); +``` + +Import `COMMAND_DEFINITIONS`, `createCommandRegistry`, and `CommandHandlers` +from `../../commands/registry` if the file does not already. + +Add to `AppShell.test.ts`, next to the existing selection-shortcut tests: + +```ts +it('extends a selection with Shift+Space pressed outside the grid', async () => { + window.history.replaceState(null, '', `/?explore=${encodeURIComponent(JSON.stringify({ workspace: 'everything' }))}`); + const state = new ExploreState(window); + const rows = [0, 1, 2].map((index) => entry(index)); + render(AppShell, { + client: createAPIClient(vi.fn<typeof fetch>(async () => Response.json(exploreResponse({ rows, total_count: 3 })))), + state + }); + const grid = await screen.findByRole('grid', { name: 'Everything results' }); + grid.focus(); + await fireEvent.keyDown(grid, { key: ' ' }); + await fireEvent.keyDown(grid, { key: 'j' }); + await fireEvent.keyDown(grid, { key: 'j' }); + document.body.focus(); + + await fireEvent.keyDown(window, { key: ' ', shiftKey: true }); + + await waitFor(() => expect(screen.getByText('3 selected')).toBeTruthy()); + state.destroy(); +}); +``` + +`entry(index)` is the existing row fixture in `AppShell.test.ts`. + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `bunx vitest run src/lib/components/shell/KeyboardHelp.test.ts src/lib/components/shell/AppShell.test.ts -t "Shift"` +Expected: FAIL. The dialog lacks the label, and the selection stays at 1. + +- [ ] **Step 3: Add the command** + +In `registry.ts`, after the `toggle-selection` entry, add: + +```ts + command('extend-selection', 'Extend selection to focused row', ['Shift', 'Space'], ['shift+space'], 'Selection'), +``` + +In `AppShell.svelte`, let `relay` and `relayGridKey` carry modifier keys: + +```ts +function relayGridKey(event: KeyboardEvent, key: string, init: KeyboardEventInit = {}): void { + if (event.target instanceof Element && event.target.closest('button, a, summary, [role="button"]')) return; + const grid = currentGrid(); + if (!grid || event.target === grid) return; + grid.focus(); + grid.dispatchEvent(new KeyboardEvent('keydown', { key, bubbles: false, cancelable: true, ...init })); +} +``` + +```ts +function relay(event: KeyboardEvent | undefined, key: string | undefined = undefined, init: KeyboardEventInit = {}): void { + const resolvedKey = key ?? event?.key; + if (!resolvedKey) return; + if (event) { + relayGridKey(event, resolvedKey, init); + return; + } + queueMicrotask(() => { + const grid = currentGrid(); + if (!grid) return; + grid.focus(); + grid.dispatchEvent(new KeyboardEvent('keydown', { key: resolvedKey, bubbles: false, cancelable: true, ...init })); + }); +} +``` + +Add the handler to `commandHandlers`: + +```ts +'extend-selection': (event) => relay(event, ' ', { shiftKey: true }), +``` + +- [ ] **Step 4: Run the tests to verify they pass** + +Run the command from Step 2. +Expected: PASS. Then run `bunx vitest run src/lib/commands src/lib/components/shell` +and confirm nothing else regressed. + +- [ ] **Step 5: Commit** + +Subject: `feat(web): list Shift+Space in the keyboard shortcuts dialog`. + +--- + +### Task 3: One history entry for a search that changes workspace + +**Files:** +- Modify: `web/src/lib/explore/state.svelte.ts` +- Test: `web/src/lib/explore/state.test.ts` + +**Interfaces:** +- Produces: + `ExploreState.commitSearchIn(workspace: ExploreWorkspace, query: string, searchMode: ExploreSearchMode): void`. + It pushes one history entry that sets the workspace, query, and mode. It + clears the same transient fields as `commitWorkspace`, and it remembers the + mode like `commitSearch`. + +- [ ] **Step 1: Write the failing test** + +```ts +it('commits a search into another workspace as one history entry', () => { + window.history.replaceState(null, '', '/?workspace=sources&mode=full_text'); + const state = new ExploreState(window); + const before = window.history.length; + + state.commitSearchIn('everything', 'quarterly report', 'hybrid'); + + expect(window.history.length).toBe(before + 1); + expect(state.current.workspace).toBe('everything'); + expect(state.current.query).toBe('quarterly report'); + expect(state.current.searchMode).toBe('hybrid'); + expect(state.current.selectedRow).toBeNull(); + state.destroy(); +}); +``` + +The existing state tests drive the jsdom `window` directly, as this test does. + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `bunx vitest run src/lib/explore -t "one history entry"` +Expected: FAIL with `state.commitSearchIn is not a function`. + +- [ ] **Step 3: Implement** + +Add after `commitWorkspace` in `state.svelte.ts`: + +```ts + commitSearchIn(workspace: ExploreWorkspace, query: string, searchMode: ExploreSearchMode): void { + rememberSearchMode(searchMode, this.preferenceStorage); + this.navigate({ + workspace, + query, + searchMode, + analysisTarget: null, + selectedIdentifier: null, + activeRow: null, + selectedRow: null, + conversationAnchor: null, + scrollAnchor: null, + operationStatus: '', + settingsAuthority: '' + }, 'push'); + } +``` + +- [ ] **Step 4: Run the test to verify it passes** + +Run the command from Step 2. Expected: PASS. + +- [ ] **Step 5: Commit** + +Subject: `feat(web): commit a search into another workspace in one step`. + +--- + +### Task 4: Global search form + +`search/SearchBar.svelte` is imported only by its own test. It becomes the +top-bar search. It keeps today's Everything names: form "Search Everything", +searchbox "Search everything", radiogroup "Search mode", button "Search". + +**Files:** +- Modify: `web/src/lib/components/search/SearchBar.svelte` +- Test: `web/src/lib/components/search/SearchBar.test.ts` (replace its + contents) + +**Interfaces:** +- Consumes: `SearchModeControl` (unchanged), `ExploreSearchMode`. +- Produces: `SearchBar` props: + - `query: string` — committed query from explore state + - `mode: ExploreSearchMode` + - `live: boolean` — true on Everything and Files + - `compact: boolean` — true below 900px + - `onDraft: (query: string, mode: ExploreSearchMode) => void` — called while + typing or changing mode, only when `live` + - `onSubmit: (query: string, mode: ExploreSearchMode) => void` + - `inputEl?: HTMLInputElement` — `$bindable`, for `/` and "Refine search" + +- [ ] **Step 1: Write the failing tests** + +Replace `SearchBar.test.ts` with: + +```ts +import { fireEvent, render, screen } from '@testing-library/svelte'; +import { describe, expect, it, vi } from 'vitest'; + +import SearchBar from './SearchBar.svelte'; + +function setup(live: boolean) { + const onDraft = vi.fn(); + const onSubmit = vi.fn(); + render(SearchBar, { query: 'budget', mode: 'full_text', live, compact: false, onDraft, onSubmit }); + return { onDraft, onSubmit, input: screen.getByRole('searchbox', { name: 'Search everything' }) }; +} + +describe('SearchBar', () => { + it('reports each keystroke when live', async () => { + const { onDraft, input } = setup(true); + await fireEvent.input(input, { target: { value: 'budget q3' } }); + expect(onDraft).toHaveBeenLastCalledWith('budget q3', 'full_text'); + }); + + it('keeps typing local when not live and submits the draft', async () => { + const { onDraft, onSubmit, input } = setup(false); + await fireEvent.input(input, { target: { value: ' merger ' } }); + await fireEvent.click(screen.getByRole('radio', { name: 'Hybrid' })); + await fireEvent.submit(screen.getByRole('search', { name: 'Search Everything' })); + expect(onDraft).not.toHaveBeenCalled(); + expect(onSubmit).toHaveBeenCalledWith('merger', 'hybrid'); + }); + + it('shows the committed query again when it changes', async () => { + const { input } = setup(false); + expect((input as HTMLInputElement).value).toBe('budget'); + }); + + it('offers the search mode as a select when compact', () => { + render(SearchBar, { query: '', mode: 'semantic', live: true, compact: true, onDraft: vi.fn(), onSubmit: vi.fn() }); + expect(screen.getByRole('combobox', { name: /^Search mode:/ })).toBeTruthy(); + expect(screen.queryByRole('radiogroup', { name: 'Search mode' })).toBeNull(); + }); +}); +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `bunx vitest run src/lib/components/search/SearchBar.test.ts` +Expected: FAIL, because the current component uses different names and props. + +- [ ] **Step 3: Implement** + +Replace `SearchBar.svelte` with: + +```svelte +<script lang="ts"> + import { Button, SearchInput, SelectDropdown } from '@kenn-io/kit-ui'; + + import type { ExploreSearchMode } from '../../explore/models'; + import SearchModeControl from './SearchModeControl.svelte'; + + interface Props { + query: string; + mode: ExploreSearchMode; + live: boolean; + compact: boolean; + onDraft: (query: string, mode: ExploreSearchMode) => void; + onSubmit: (query: string, mode: ExploreSearchMode) => void; + inputEl?: HTMLInputElement; + } + + let { query, mode, live, compact, onDraft, onSubmit, inputEl = $bindable() }: Props = $props(); + + let draft = $state(''); + let draftMode = $state<ExploreSearchMode>('full_text'); + + $effect(() => { + draft = query; + draftMode = mode; + }); + + const modeOptions = [ + { value: 'full_text', label: 'Full text' }, + { value: 'semantic', label: 'Semantic' }, + { value: 'hybrid', label: 'Hybrid' } + ]; + + function changeQuery(value: string): void { + draft = value; + if (live) onDraft(value, draftMode); + } + + function changeMode(value: ExploreSearchMode): void { + draftMode = value; + if (live) onDraft(draft, value); + } + + function submit(event: SubmitEvent): void { + event.preventDefault(); + onSubmit(draft.trim(), draftMode); + } +</script> + +<form class="global-search" class:global-search--compact={compact} role="search" + aria-label="Search Everything" onsubmit={submit}> + <div class="global-search__query"> + <SearchInput + id="everything-search" + bind:inputEl + value={draft} + ariaLabel="Search everything" + placeholder="Search people, conversations, events, and files…" + block + oninput={changeQuery} + /> + </div> + {#if compact} + <SelectDropdown + title="Search mode" + value={draftMode} + options={modeOptions} + align="end" + onchange={(value) => changeMode(value as ExploreSearchMode)} + /> + {:else} + <SearchModeControl requestedMode={draftMode} onchange={changeMode} /> + <Button type="submit" label="Search" tone="info" surface="solid" /> + {/if} +</form> + +<style> + .global-search { + display: flex; + min-width: 0; + flex: 1; + align-items: center; + gap: var(--space-3); + } + + .global-search__query { + min-width: 0; + flex: 1; + max-width: 640px; + } + + .global-search--compact { + gap: var(--space-2); + } +</style> +``` + +If a kit component's prop name differs from the one used here (`bind:inputEl`, +`oninput`), match the usage in the current `EverythingWorkspace.svelte` +search form. That usage is known to work. + +- [ ] **Step 4: Run the tests to verify they pass** + +Run the command from Step 2. Expected: PASS. + +- [ ] **Step 5: Commit** + +Subject: `feat(web): turn SearchBar into the global search form`. + +--- + +### Task 5: Sidebar navigation + +**Files:** +- Create: `web/src/lib/components/shell/navigation.ts` +- Create: `web/src/lib/components/shell/AppSidebar.svelte` +- Test: `web/src/lib/components/shell/AppSidebar.test.ts` + +**Interfaces:** +- Produces from `navigation.ts`: + +```ts +export interface NavigationItem { id: ExploreWorkspace; label: string; icon: Component } +export interface NavigationGroup { label: string; items: NavigationItem[] } +export const NAVIGATION_GROUPS: NavigationGroup[]; +export function workspaceLabel(id: ExploreWorkspace): string; +export const SIDEBAR_COLLAPSED_KEY = 'msgvault.sidebar.collapsed'; +``` + +- Produces the `AppSidebar` props: + - `active: ExploreWorkspace` + - `collapsed: boolean` — rail mode; the drawer always passes `false` + - `showCollapseToggle: boolean` — false inside the drawer + - `status: { tone: 'working' | 'idle' | 'unclean'; label: string; text: string }` + - `onNavigate: (id: ExploreWorkspace) => void` + - `onToggleCollapsed: () => void` + - `onOpenShortcuts: () => void` + +- [ ] **Step 1: Write the failing tests** + +```ts +import { fireEvent, render, screen, within } from '@testing-library/svelte'; +import { describe, expect, it, vi } from 'vitest'; + +import AppSidebar from './AppSidebar.svelte'; + +const status = { tone: 'idle' as const, label: 'Local archive ready', text: 'Local archive' }; + +function renderSidebar(overrides: Record<string, unknown> = {}) { + const props = { + active: 'everything' as const, collapsed: false, showCollapseToggle: true, status, + onNavigate: vi.fn(), onToggleCollapsed: vi.fn(), onOpenShortcuts: vi.fn(), ...overrides + }; + render(AppSidebar, props); + return props; +} + +describe('AppSidebar', () => { + it('lists workspaces in People, Archive, Manage order with the active one current', () => { + renderSidebar(); + const nav = screen.getByRole('navigation', { name: 'Primary' }); + expect(within(nav).getAllByRole('button').map((b) => b.getAttribute('aria-label') ?? b.textContent?.trim())) + .toEqual(['Relationships', 'Directory', 'Reviews', 'Everything', 'Files', 'Saved views', + 'Sources', 'Operations', 'Deletions', 'Settings']); + expect(within(nav).getByRole('button', { name: 'Everything' }).getAttribute('aria-current')).toBe('page'); + expect(within(nav).getByRole('button', { name: 'Files' }).hasAttribute('aria-current')).toBe(false); + }); + + it('navigates when an item is chosen', async () => { + const props = renderSidebar(); + await fireEvent.click(screen.getByRole('button', { name: 'Deletions' })); + expect(props.onNavigate).toHaveBeenCalledWith('deletions'); + }); + + it('keeps full accessible names in the icon rail and hides group headings', () => { + renderSidebar({ collapsed: true }); + expect(screen.getByRole('button', { name: 'Saved views' })).toBeTruthy(); + expect(screen.queryByText('People')).toBeNull(); + }); + + it('toggles the rail and opens shortcuts from the footer', async () => { + const props = renderSidebar(); + await fireEvent.click(screen.getByRole('button', { name: 'Collapse sidebar' })); + await fireEvent.click(screen.getByRole('button', { name: /Keyboard shortcuts/ })); + expect(props.onToggleCollapsed).toHaveBeenCalled(); + expect(props.onOpenShortcuts).toHaveBeenCalled(); + expect(screen.getByText('Local archive')).toBeTruthy(); + }); +}); +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `bunx vitest run src/lib/components/shell/AppSidebar.test.ts` +Expected: FAIL, because the module does not exist. + +- [ ] **Step 3: Create `navigation.ts`** + +```ts +import type { Component } from 'svelte'; +import Activity from '@lucide/svelte/icons/activity'; +import Bookmark from '@lucide/svelte/icons/bookmark'; +import CheckCheck from '@lucide/svelte/icons/check-check'; +import Contact from '@lucide/svelte/icons/contact'; +import Inbox from '@lucide/svelte/icons/inbox'; +import Paperclip from '@lucide/svelte/icons/paperclip'; +import Plug from '@lucide/svelte/icons/plug'; +import Settings from '@lucide/svelte/icons/settings'; +import Trash2 from '@lucide/svelte/icons/trash-2'; +import Users from '@lucide/svelte/icons/users'; + +import type { ExploreWorkspace } from '../../explore/models'; + +export interface NavigationItem { + id: ExploreWorkspace; + label: string; + icon: Component; +} + +export interface NavigationGroup { + label: string; + items: NavigationItem[]; +} + +export const SIDEBAR_COLLAPSED_KEY = 'msgvault.sidebar.collapsed'; + +export const NAVIGATION_GROUPS: NavigationGroup[] = [ + { + label: 'People', + items: [ + { id: 'relationships', label: 'Relationships', icon: Users }, + { id: 'directory', label: 'Directory', icon: Contact }, + { id: 'directory_review', label: 'Reviews', icon: CheckCheck } + ] + }, + { + label: 'Archive', + items: [ + { id: 'everything', label: 'Everything', icon: Inbox }, + { id: 'files', label: 'Files', icon: Paperclip }, + { id: 'saved_views', label: 'Saved views', icon: Bookmark } + ] + }, + { + label: 'Manage', + items: [ + { id: 'sources', label: 'Sources', icon: Plug }, + { id: 'operations', label: 'Operations', icon: Activity }, + { id: 'deletions', label: 'Deletions', icon: Trash2 }, + { id: 'settings', label: 'Settings', icon: Settings } + ] + } +]; + +export function workspaceLabel(id: ExploreWorkspace): string { + for (const group of NAVIGATION_GROUPS) { + const item = group.items.find((candidate) => candidate.id === id); + if (item) return item.label; + } + return 'msgvault'; +} +``` + +- [ ] **Step 4: Create `AppSidebar.svelte`** + +```svelte +<script lang="ts"> + import { KbdBadge, StatusDot, Tooltip } from '@kenn-io/kit-ui'; + import Keyboard from '@lucide/svelte/icons/keyboard'; + import PanelLeftClose from '@lucide/svelte/icons/panel-left-close'; + import PanelLeftOpen from '@lucide/svelte/icons/panel-left-open'; + + import type { ExploreWorkspace } from '../../explore/models'; + import { NAVIGATION_GROUPS } from './navigation'; + + interface Props { + active: ExploreWorkspace; + collapsed: boolean; + showCollapseToggle: boolean; + status: { tone: 'working' | 'idle' | 'unclean'; label: string; text: string }; + onNavigate: (id: ExploreWorkspace) => void; + onToggleCollapsed: () => void; + onOpenShortcuts: () => void; + } + + let { active, collapsed, showCollapseToggle, status, onNavigate, onToggleCollapsed, onOpenShortcuts }: Props = $props(); +</script> + +<div class="sidebar" class:sidebar--rail={collapsed}> + <div class="sidebar__brand">msgvault</div> + <nav aria-label="Primary"> + {#each NAVIGATION_GROUPS as group, index (group.label)} + {#if collapsed} + {#if index > 0}<hr class="sidebar__divider" />{/if} + {:else} + <h2 class="sidebar__group">{group.label}</h2> + {/if} + <ul> + {#each group.items as item (item.id)} + {@const Icon = item.icon} + <li> + {#if collapsed} + <Tooltip text={item.label}> + <button type="button" class="sidebar__item" aria-label={item.label} + aria-current={item.id === active ? 'page' : undefined} onclick={() => onNavigate(item.id)}> + <Icon size={18} aria-hidden="true" /> + </button> + </Tooltip> + {:else} + <button type="button" class="sidebar__item" + aria-current={item.id === active ? 'page' : undefined} onclick={() => onNavigate(item.id)}> + <Icon size={18} aria-hidden="true" /><span>{item.label}</span> + </button> + {/if} + </li> + {/each} + </ul> + {/each} + </nav> + <div class="sidebar__footer"> + <span class="sidebar__status" title={status.label}> + <StatusDot status={status.tone} label={status.label} /> + {#if !collapsed}<span>{status.text}</span>{/if} + </span> + <button type="button" class="sidebar__item" aria-label="Keyboard shortcuts" onclick={onOpenShortcuts}> + <Keyboard size={18} aria-hidden="true" /> + {#if !collapsed}<span>Keyboard shortcuts</span><KbdBadge keys={['?']} />{/if} + </button> + {#if showCollapseToggle} + <button type="button" class="sidebar__item" aria-label={collapsed ? 'Expand sidebar' : 'Collapse sidebar'} + onclick={onToggleCollapsed}> + {#if collapsed}<PanelLeftOpen size={18} aria-hidden="true" />{:else}<PanelLeftClose size={18} aria-hidden="true" /><span>Collapse</span>{/if} + </button> + {/if} + </div> +</div> + +<style> + .sidebar { + display: flex; + width: var(--nav-width); + height: 100%; + flex-direction: column; + gap: var(--space-3); + padding: var(--space-3) var(--space-2); + overflow-y: auto; + background: var(--bg-surface); + border-right: 1px solid var(--border-default); + } + + .sidebar--rail { + width: var(--nav-rail-width); + align-items: center; + } + + .sidebar__brand { + padding: var(--space-1) var(--space-3); + color: var(--text-primary); + font-size: var(--font-size-md); + font-weight: 650; + } + + .sidebar--rail .sidebar__brand { + visibility: hidden; + } + + nav { + display: flex; + flex: 1; + flex-direction: column; + gap: var(--space-1); + } + + ul { + display: flex; + flex-direction: column; + gap: 2px; + margin: 0; + padding: 0; + list-style: none; + } + + .sidebar__group { + margin: var(--space-3) 0 var(--space-1); + padding: 0 var(--space-3); + color: var(--text-muted); + font-size: var(--font-size-xs); + font-weight: 600; + } + + .sidebar__divider { + width: 24px; + margin: var(--space-2) auto; + border: 0; + border-top: 1px solid var(--border-default); + } + + .sidebar__item { + display: flex; + width: 100%; + min-height: 32px; + align-items: center; + gap: var(--space-3); + padding: 0 var(--space-3); + border: 0; + border-radius: var(--radius-md); + background: transparent; + color: var(--text-secondary); + font-size: var(--font-size-md); + text-align: left; + cursor: pointer; + } + + .sidebar--rail .sidebar__item { + width: 36px; + justify-content: center; + padding: 0; + } + + .sidebar__item:hover { + background: var(--bg-surface-hover); + color: var(--text-primary); + } + + .sidebar__item[aria-current='page'] { + background: var(--nav-active-bg); + color: var(--text-primary); + font-weight: 600; + } + + .sidebar__item :global(.kit-kbd) { + margin-left: auto; + } + + .sidebar__footer { + display: flex; + flex-direction: column; + gap: 2px; + padding-top: var(--space-2); + border-top: 1px solid var(--border-muted); + } + + .sidebar__status { + display: flex; + min-height: 28px; + align-items: center; + gap: var(--space-2); + padding: 0 var(--space-3); + color: var(--text-muted); + font-size: var(--font-size-xs); + } +</style> +``` + +Check the `KbdBadge` class name with +`rg -n 'class="kit-kbd' node_modules/@kenn-io/kit-ui/src/lib/components/KbdBadge.svelte` +and adjust the `:global()` selector to match. + +- [ ] **Step 5: Run the tests to verify they pass** + +Run the command from Step 2. Expected: PASS. + +- [ ] **Step 6: Commit** + +Subject: `feat(web): add a grouped sidebar with an icon rail`. + +--- + +### Task 6: Narrow-screen navigation drawer + +**Files:** +- Create: `web/src/lib/components/shell/NavigationDrawer.svelte` +- Test: `web/src/lib/components/shell/NavigationDrawer.test.ts` + +**Interfaces:** +- Produces the `NavigationDrawer` props: `onclose: () => void` and + `children: Snippet`. The parent mounts the drawer only while it is open. +- Behavior contract: + - Mounting pushes the shortcut scope `navigation-drawer`. That suspends all + root shortcuts, including `close-layer`, so Escape cannot also close the + reading pane. + - Escape, registered in that scope, calls `onclose`. + - Selecting the scrim calls `onclose`. + - On mount, focus moves to the drawer's `[aria-current="page"]` button. kit + `trapFocus` keeps Tab inside and, on unmount, restores focus to the element + that had it before, which is the "Open navigation" button. + - Choosing an item also restores focus to "Open navigation". The spec's + "workspace-change focus rules" leave focus on the navigation control that + was used; in the drawer that control is the menu button. + +- [ ] **Step 1: Write the failing tests** + +```ts +import { fireEvent, render, screen } from '@testing-library/svelte'; +import { appShortcuts } from '@kenn-io/kit-ui'; +import { createRawSnippet } from 'svelte'; +import { describe, expect, it, vi } from 'vitest'; + +import NavigationDrawer from './NavigationDrawer.svelte'; + +const items = createRawSnippet(() => ({ + render: () => '<nav aria-label="Primary"><button>Files</button><button aria-current="page">Everything</button></nav>' +})); + +describe('NavigationDrawer', () => { + it('focuses the current item and closes on Escape without reaching root shortcuts', async () => { + const root = vi.fn(); + const unregister = appShortcuts.register('escape', root); + const onclose = vi.fn(); + render(NavigationDrawer, { onclose, children: items }); + + expect(document.activeElement).toBe(screen.getByRole('button', { name: 'Everything' })); + await fireEvent.keyDown(document.activeElement!, { key: 'Escape' }); + + expect(onclose).toHaveBeenCalledOnce(); + expect(root).not.toHaveBeenCalled(); + unregister(); + }); + + it('closes when the scrim is selected', async () => { + const onclose = vi.fn(); + render(NavigationDrawer, { onclose, children: items }); + await fireEvent.click(screen.getByRole('button', { name: 'Close navigation' })); + expect(onclose).toHaveBeenCalledOnce(); + }); + + it('returns focus to the opener when unmounted', () => { + const opener = document.createElement('button'); + document.body.append(opener); + opener.focus(); + const rendered = render(NavigationDrawer, { onclose: vi.fn(), children: items }); + rendered.unmount(); + expect(document.activeElement).toBe(opener); + opener.remove(); + }); +}); +``` + +The first test relies on `initShortcuts` having attached kit's keydown +listener. If it is not attached in unit tests, call `initShortcuts()` in a +`beforeAll` and its returned cleanup in `afterAll`, as `AppShell.svelte` does. + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `bunx vitest run src/lib/components/shell/NavigationDrawer.test.ts` +Expected: FAIL, because the module does not exist. + +- [ ] **Step 3: Implement** + +```svelte +<script lang="ts"> + import { appShortcuts, trapFocus } from '@kenn-io/kit-ui'; + import { onMount, tick, type Snippet } from 'svelte'; + + let { onclose, children }: { onclose: () => void; children: Snippet } = $props(); + let panel = $state<HTMLElement>(); + + onMount(() => { + const popScope = appShortcuts.pushScope('navigation-drawer'); + const unregister = appShortcuts.register('escape', () => onclose(), { scope: 'navigation-drawer' }); + void tick().then(() => panel?.querySelector<HTMLElement>('[aria-current="page"]')?.focus()); + return () => { + unregister(); + popScope(); + }; + }); +</script> + +<div class="drawer"> + <button type="button" class="drawer__scrim" aria-label="Close navigation" onclick={onclose}></button> + <div class="drawer__panel" role="dialog" aria-modal="true" aria-label="Navigation" tabindex="-1" + bind:this={panel} {@attach trapFocus}> + {@render children()} + </div> +</div> + +<style> + .drawer { + position: fixed; + inset: 0; + z-index: 50; + } + + .drawer__scrim { + position: absolute; + inset: 0; + border: 0; + background: var(--overlay-bg); + } + + .drawer__panel { + position: absolute; + inset: 0 auto 0 0; + height: 100%; + box-shadow: var(--shadow-lg); + } +</style> +``` + +`trapFocus` runs its initial focus before `tick()` resolves. The `tick()` +callback then moves focus to the current item, which is inside the trap. + +- [ ] **Step 4: Run the tests to verify they pass** + +Run the command from Step 2. Expected: PASS. + +- [ ] **Step 5: Commit** + +Subject: `feat(web): add the narrow-screen navigation drawer`. + +--- + +### Task 7: Display menu + +**Files:** +- Create: `web/src/lib/components/shell/DisplayMenu.svelte` +- Test: `web/src/lib/components/shell/DisplayMenu.test.ts` + +**Interfaces:** +- Produces the `DisplayMenu` props: + - `density: 'daemon' | 'compact' | 'comfortable'` — `'daemon'` means no + override + - `themeOverridden: boolean` + - `onDensityChange: (value: 'daemon' | 'compact' | 'comfortable') => void` + - `onUseDaemonTheme: () => void` +- Accessible names: trigger button "Display"; radio group "Temporary + density" with radios "Auto", "Compact", and "Comfortable"; menu item "Use + daemon theme", shown only when `themeOverridden`. + +- [ ] **Step 1: Write the failing tests** + +```ts +import { fireEvent, render, screen } from '@testing-library/svelte'; +import { describe, expect, it, vi } from 'vitest'; + +import DisplayMenu from './DisplayMenu.svelte'; + +describe('DisplayMenu', () => { + it('sets and clears the per-tab density override', async () => { + const onDensityChange = vi.fn(); + render(DisplayMenu, { density: 'daemon', themeOverridden: false, onDensityChange, onUseDaemonTheme: vi.fn() }); + await fireEvent.click(screen.getByRole('button', { name: 'Display' })); + + const group = screen.getByRole('group', { name: 'Temporary density' }); + expect(group).toBeTruthy(); + await fireEvent.click(screen.getByRole('menuitemradio', { name: 'Comfortable' })); + expect(onDensityChange).toHaveBeenCalledWith('comfortable'); + expect(screen.queryByRole('menuitem', { name: 'Use daemon theme' })).toBeNull(); + }); + + it('offers the daemon theme only while a theme override is active', async () => { + const onUseDaemonTheme = vi.fn(); + render(DisplayMenu, { density: 'compact', themeOverridden: true, onDensityChange: vi.fn(), onUseDaemonTheme }); + await fireEvent.click(screen.getByRole('button', { name: 'Display' })); + await fireEvent.click(screen.getByRole('menuitem', { name: 'Use daemon theme' })); + expect(onUseDaemonTheme).toHaveBeenCalledOnce(); + }); +}); +``` + +Check the roles kit renders with +`rg -n 'role=' node_modules/@kenn-io/kit-ui/src/lib/components/MenuRadio*.svelte node_modules/@kenn-io/kit-ui/src/lib/components/MenuItem.svelte`. +If the group role is `radiogroup`, use that in the test. + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `bunx vitest run src/lib/components/shell/DisplayMenu.test.ts` +Expected: FAIL, because the module does not exist. + +- [ ] **Step 3: Implement** + +```svelte +<script lang="ts"> + import { Menu, MenuContent, MenuItem, MenuRadioGroup, MenuRadioItem, MenuSeparator, MenuTrigger } from '@kenn-io/kit-ui'; + import SlidersHorizontal from '@lucide/svelte/icons/sliders-horizontal'; + + type Density = 'daemon' | 'compact' | 'comfortable'; + + interface Props { + density: Density; + themeOverridden: boolean; + onDensityChange: (value: Density) => void; + onUseDaemonTheme: () => void; + } + + let { density, themeOverridden, onDensityChange, onUseDaemonTheme }: Props = $props(); +</script> + +<Menu align="end"> + <MenuTrigger ariaLabel="Display" title="Display"> + <SlidersHorizontal size={18} aria-hidden="true" /> + </MenuTrigger> + <MenuContent ariaLabel="Display"> + <MenuRadioGroup ariaLabel="Temporary density" value={density} onchange={(value) => onDensityChange(value as Density)}> + <MenuRadioItem value="daemon">Auto</MenuRadioItem> + <MenuRadioItem value="compact">Compact</MenuRadioItem> + <MenuRadioItem value="comfortable">Comfortable</MenuRadioItem> + </MenuRadioGroup> + {#if themeOverridden} + <MenuSeparator /> + <MenuItem onselect={onUseDaemonTheme}>Use daemon theme</MenuItem> + {/if} + </MenuContent> +</Menu> +``` + +- [ ] **Step 4: Run the tests to verify they pass** + +Run the command from Step 2. Expected: PASS. + +- [ ] **Step 5: Commit** + +Subject: `feat(web): move per-tab display overrides into a menu`. + +--- + +### Task 8: Page header + +**Files:** +- Create: `web/src/lib/components/shell/PageHeader.svelte` +- Test: `web/src/lib/components/shell/PageHeader.test.ts` + +**Interfaces:** +- Produces the `PageHeader` props: + - `title: string` + - `description?: string` + - `actions?: Snippet` + - `view?: Snippet` — a row below the title for view switches + - `visuallyHiddenTitle?: boolean` — defaults to false. It exists only for + the narrow Operations detail view, which renders its own heading. + +- [ ] **Step 1: Write the failing test** + +```ts +import { render, screen } from '@testing-library/svelte'; +import { createRawSnippet } from 'svelte'; +import { describe, expect, it } from 'vitest'; + +import PageHeader from './PageHeader.svelte'; + +describe('PageHeader', () => { + it('renders one level-one heading, the description, and actions', () => { + const actions = createRawSnippet(() => ({ render: () => '<button>Refresh operations</button>' })); + render(PageHeader, { title: 'Operations', description: 'Background work and its history.', actions }); + + expect(screen.getByRole('heading', { level: 1, name: 'Operations' })).toBeTruthy(); + expect(screen.getByText('Background work and its history.')).toBeTruthy(); + expect(screen.getByRole('button', { name: 'Refresh operations' })).toBeTruthy(); + }); +}); +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `bunx vitest run src/lib/components/shell/PageHeader.test.ts` +Expected: FAIL, because the module does not exist. + +- [ ] **Step 3: Implement** + +```svelte +<script lang="ts"> + import type { Snippet } from 'svelte'; + + interface Props { + title: string; + description?: string; + actions?: Snippet; + view?: Snippet; + visuallyHiddenTitle?: boolean; + } + + let { title, description, actions, view, visuallyHiddenTitle = false }: Props = $props(); +</script> + +<header class="page-header"> + <div class="page-header__row"> + <div class="page-header__text" class:kit-sr-only={visuallyHiddenTitle}> + <h1>{title}</h1> + {#if description}<p>{description}</p>{/if} + </div> + {#if actions}<div class="page-header__actions">{@render actions()}</div>{/if} + </div> + {#if view}<div class="page-header__view">{@render view()}</div>{/if} +</header> + +<style> + .page-header { + display: flex; + flex-direction: column; + gap: var(--space-3); + } + + .page-header__row { + display: flex; + flex-wrap: wrap; + align-items: center; + justify-content: space-between; + gap: var(--space-3) var(--space-6); + } + + .page-header__text { + min-width: 0; + } + + h1 { + margin: 0; + color: var(--text-primary); + font-size: var(--font-size-xl); + font-weight: 650; + line-height: 1.25; + } + + p { + margin: var(--space-1) 0 0; + color: var(--text-muted); + font-size: var(--font-size-sm); + } + + .page-header__actions { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: var(--space-2); + } +</style> +``` + +- [ ] **Step 4: Run the test to verify it passes** + +Run the command from Step 2. Expected: PASS. + +- [ ] **Step 5: Commit** + +Subject: `feat(web): add a shared page header`. + +--- + +### Task 9: Compose the shell + +This task replaces kit `TopBar` in `AppShell.svelte` with the new layout. It +moves the search form out of Everything and removes the keyboard footer and +the selection-bar key badges. + +**Files:** +- Modify: `web/src/lib/components/shell/AppShell.svelte` +- Modify: `web/src/lib/components/shell/EverythingWorkspace.svelte` +- Modify: `web/src/lib/components/explore/SelectionBar.svelte` +- Modify: `web/src/App.svelte` (remove the static `<title>`) +- Test: `web/src/lib/components/shell/AppShell.test.ts`, + `web/src/lib/components/shell/EverythingWorkspace.test.ts` + +**Interfaces:** +- Consumes: `SearchBar` (Task 4), `AppSidebar`, `NAVIGATION_GROUPS`, + `workspaceLabel`, `SIDEBAR_COLLAPSED_KEY` (Task 5), `NavigationDrawer` + (Task 6), `DisplayMenu` (Task 7), `ExploreState.commitSearchIn` (Task 3). +- `EverythingWorkspace` changes from `bind:searchInput` to a plain prop + `searchInput: HTMLInputElement | undefined`. "Refine search" still calls + `searchInput?.focus()`. The `commitSearch` and `SearchCoverage` props are + unchanged. + +- [ ] **Step 1: Write the failing tests** + +Add to `AppShell.test.ts`. Replace the existing test "presents the primary +navigation tabs…" with the first test below; it checks the new order and +labels. + +```ts +it('groups workspaces in the sidebar with Relationships first', () => { + window.history.replaceState(null, '', `/?explore=${encodeURIComponent(JSON.stringify({ workspace: 'everything' }))}`); + const state = new ExploreState(window); + const rendered = render(AppShell, { + client: createAPIClient(vi.fn<typeof fetch>(async () => Response.json(exploreResponse()))), state, enabled: false + }); + const nav = screen.getByRole('navigation', { name: 'Primary' }); + expect(within(nav).getAllByRole('button').map((button) => button.textContent?.trim())).toEqual([ + 'Relationships', 'Directory', 'Reviews', 'Everything', 'Files', 'Saved views', 'Sources', 'Operations', 'Deletions', 'Settings' + ]); + expect(within(nav).getByRole('button', { name: 'Everything' }).getAttribute('aria-current')).toBe('page'); + rendered.unmount(); + state.destroy(); +}); + +it('names the browser tab after the workspace', async () => { + window.history.replaceState(null, '', `/?explore=${encodeURIComponent(JSON.stringify({ workspace: 'sources' }))}`); + const state = new ExploreState(window); + render(AppShell, { client: createAPIClient(vi.fn<typeof fetch>(async () => Response.json({ sources: [] }))), state, enabled: false }); + await waitFor(() => expect(document.title).toBe('Sources · msgvault')); + await fireEvent.click(screen.getByRole('button', { name: 'Directory' })); + await waitFor(() => expect(document.title).toBe('Directory · msgvault')); + state.destroy(); +}); + +it('opens Everything with the query when searching from another workspace', async () => { + window.history.replaceState(null, '', `/?explore=${encodeURIComponent(JSON.stringify({ workspace: 'sources' }))}`); + const state = new ExploreState(window); + const length = window.history.length; + render(AppShell, { client: createAPIClient(vi.fn<typeof fetch>(async () => Response.json(exploreResponse()))), state, enabled: false }); + + const search = screen.getByRole('searchbox', { name: 'Search everything' }); + await fireEvent.input(search, { target: { value: 'pipeline' } }); + expect(state.current.workspace).toBe('sources'); + await fireEvent.submit(screen.getByRole('search', { name: 'Search Everything' })); + + expect(state.current.workspace).toBe('everything'); + expect(state.current.query).toBe('pipeline'); + expect(window.history.length).toBe(length + 1); + state.destroy(); +}); + +it('updates Everything results as the global search is typed', async () => { + window.history.replaceState(null, '', `/?explore=${encodeURIComponent(JSON.stringify({ workspace: 'everything' }))}`); + const state = new ExploreState(window); + render(AppShell, { client: createAPIClient(vi.fn<typeof fetch>(async () => Response.json(exploreResponse()))), state, enabled: false }); + await fireEvent.input(screen.getByRole('searchbox', { name: 'Search everything' }), { target: { value: 'gas' } }); + expect(state.current.query).toBe('gas'); + state.destroy(); +}); + +it('remembers the collapsed sidebar across reloads', async () => { + window.history.replaceState(null, '', `/?explore=${encodeURIComponent(JSON.stringify({ workspace: 'everything' }))}`); + const state = new ExploreState(window); + const first = render(AppShell, { client: createAPIClient(vi.fn()), state, enabled: false }); + await fireEvent.click(screen.getByRole('button', { name: 'Collapse sidebar' })); + first.unmount(); + render(AppShell, { client: createAPIClient(vi.fn()), state, enabled: false }); + expect(screen.getByRole('button', { name: 'Expand sidebar' })).toBeTruthy(); + state.destroy(); +}); + +it('opens a modal navigation menu on narrow screens and closes it on Escape', async () => { + window.innerWidth = 480; + window.history.replaceState(null, '', `/?explore=${encodeURIComponent(JSON.stringify({ workspace: 'everything' }))}`); + const state = new ExploreState(window); + render(AppShell, { client: createAPIClient(vi.fn()), state, enabled: false }); + + expect(screen.queryByRole('navigation', { name: 'Primary' })).toBeNull(); + const opener = screen.getByRole('button', { name: 'Open navigation' }); + await fireEvent.click(opener); + const current = screen.getByRole('button', { name: 'Everything' }); + expect(document.activeElement).toBe(current); + + await fireEvent.keyDown(current, { key: 'Escape' }); + await waitFor(() => expect(screen.queryByRole('navigation', { name: 'Primary' })).toBeNull()); + expect(document.activeElement).toBe(opener); + window.innerWidth = 1024; + state.destroy(); +}); +``` + +Also update the test "focuses search with slash…". It must still pass with +the searchbox in the top bar and without the Everything search form. + +In `EverythingWorkspace.test.ts`, delete the assertions that query the +Everything search form or the keyboard footer. Add: + +```ts +it('leaves searching to the global search box', () => { + window.history.replaceState(null, '', `/?explore=${encodeURIComponent(JSON.stringify({ workspace: 'everything' }))}`); + const state = new ExploreState(window); + render(AppShell, { client: createAPIClient(vi.fn<typeof fetch>(async () => Response.json(exploreResponse()))), state, enabled: false }); + const main = screen.getByRole('main', { name: 'Everything' }); + expect(within(main).queryByRole('search')).toBeNull(); + expect(screen.getAllByRole('search')).toHaveLength(1); + expect(screen.queryByRole('contentinfo', { name: 'Keyboard shortcuts' })).toBeNull(); + state.destroy(); +}); +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `bunx vitest run src/lib/components/shell` +Expected: the new tests FAIL. + +- [ ] **Step 3: Restructure the AppShell markup** + +Replace the `<TopBar …>…</TopBar>` block and the `app-shell` wrapper with this +structure. Keep every `{#if exploreState.current.workspace === …}` branch +inside `<div class="app-main">` unchanged, and keep the elements that follow +the wrapper (palette, keyboard help, viewers) as they are. + +```svelte +<svelte:window bind:innerWidth={viewportWidth} /> + +<div class="app-shell" class:app-shell--narrow={narrow}> + <span class="kit-sr-only" role="status" aria-label="Operation status" aria-live="polite"> + {#key operationAnnouncement.key}<span>{operationAnnouncement.message}</span>{/key} + </span> + {#if !narrow} + <AppSidebar active={exploreState.current.workspace} collapsed={sidebarCollapsed} showCollapseToggle + status={archiveStatus} onNavigate={openWorkspaceTab} onToggleCollapsed={toggleSidebar} + onOpenShortcuts={() => { keyboardHelpOpen = true; }} /> + {/if} + <div class="app-column" inert={drawerOpen}> + <header class="app-top-bar"> + {#if narrow} + <IconButton label="Open navigation" onclick={() => { drawerOpen = true; }} bind:element={drawerOpener}> + <Menu size={18} aria-hidden="true" /> + </IconButton> + {/if} + <SearchBar query={exploreState.current.query} mode={exploreState.current.searchMode} + live={exploreState.current.workspace === 'everything' || exploreState.current.workspace === 'files'} + compact={narrow} bind:inputEl={searchInput} onDraft={(query, mode) => exploreState.replaceSearchDraft(query, mode)} + onSubmit={submitGlobalSearch} /> + <div class="app-top-bar__end" aria-label="Appearance controls"> + <ThemeToggle /> + <DisplayMenu density={appearance.temporary.density ?? 'daemon'} + themeOverridden={appearance.temporary.theme !== undefined} + onDensityChange={applyTemporaryDensity} onUseDaemonTheme={() => appearance.clearTemporary('theme')} /> + </div> + </header> + <div class="app-main"> + <!-- existing workspace branches, unchanged --> + </div> + </div> + {#if narrow && drawerOpen} + <NavigationDrawer onclose={() => { drawerOpen = false; }}> + <AppSidebar active={exploreState.current.workspace} collapsed={false} showCollapseToggle={false} + status={archiveStatus} onNavigate={(id) => { drawerOpen = false; openWorkspaceTab(id); }} + onToggleCollapsed={() => undefined} onOpenShortcuts={() => { drawerOpen = false; keyboardHelpOpen = true; }} /> + </NavigationDrawer> + {/if} +</div> +``` + +Check the `IconButton` prop names with +`sed -n '/interface Props/,/}: Props/p' node_modules/@kenn-io/kit-ui/src/lib/components/IconButton.svelte`. +If it has no element binding, wrap it in a `<span bind:this>` and focus the +`button` inside. `trapFocus` restores focus to the opener either way, so the +reference is needed only if restoration fails in the browser test. + +- [ ] **Step 4: Add the script state and functions** + +In `AppShell.svelte`'s script: + +```ts +import AppSidebar from './AppSidebar.svelte'; +import DisplayMenu from './DisplayMenu.svelte'; +import NavigationDrawer from './NavigationDrawer.svelte'; +import SearchBar from '../search/SearchBar.svelte'; +import { SIDEBAR_COLLAPSED_KEY, workspaceLabel } from './navigation'; +import Menu from '@lucide/svelte/icons/menu'; +``` + +Add `IconButton` to the kit-ui import. Remove `TopBar`, `SelectDropdown`, +`StatusDot`, `Button` (if no longer used), and the `tabs` and +`densityOptions` constants. + +```ts +const NARROW_WIDTH = 900; +let viewportWidth = $state(typeof window === 'undefined' ? 1280 : window.innerWidth); +const narrow = $derived(viewportWidth < NARROW_WIDTH); +let drawerOpen = $state(false); +let drawerOpener = $state<HTMLElement>(); +let sidebarCollapsed = $state(readSidebarCollapsed()); + +function readSidebarCollapsed(): boolean { + try { + return localStorage.getItem(SIDEBAR_COLLAPSED_KEY) === 'true'; + } catch { + return false; + } +} + +function toggleSidebar(): void { + sidebarCollapsed = !sidebarCollapsed; + try { + localStorage.setItem(SIDEBAR_COLLAPSED_KEY, String(sidebarCollapsed)); + } catch { + // Storage may be disabled; the rail still toggles for this page view. + } +} + +$effect(() => { + if (!narrow) drawerOpen = false; +}); + +$effect(() => { + document.title = `${workspaceLabel(exploreState.current.workspace)} · msgvault`; +}); + +const archiveStatus = $derived( + loader.loading + ? { tone: 'working' as const, label: 'Searching', text: 'Searching' } + : loader.error || loader.unavailable + ? { tone: 'unclean' as const, label: 'Archive needs attention', text: 'Attention' } + : { tone: 'idle' as const, label: 'Local archive ready', text: 'Local archive' } +); + +function submitGlobalSearch(query: string, mode: ExploreSearchMode): void { + const workspace = exploreState.current.workspace; + if (workspace === 'everything' || workspace === 'files') { + commitSearch(query, mode); + focusGrid(); + return; + } + beforeCommit(); + exploreState.commitSearchIn('everything', query, mode); +} +``` + +`applyTemporaryDensity` keeps its body. It now receives +`'daemon' | 'compact' | 'comfortable'` from `DisplayMenu`. + +- [ ] **Step 5: Update EverythingWorkspace** + +In `EverythingWorkspace.svelte`: +1. Delete the `<form class="search-bar" …>…</form>` block, the + `submitSearch` function, and the `.search-bar` and `.query-control` styles + and their media-query rules. +2. Change the `searchInput` prop from `$bindable()` to a plain prop, and + update the `Props` interface. +3. Delete the `<footer class="keyboard-help" …>` block, its styles, and the + `KbdBadge` import if nothing else uses it. +4. Keep "Refine search", `SearchCoverage`, and every other block. + +In `AppShell.svelte`, change `bind:searchInput` on `<EverythingWorkspace>` to +`{searchInput}`. + +In `SelectionBar.svelte`, delete the "Space toggle" and "A visible" key-badge +markup at lines 65–66 and any styles only they use. + +In `App.svelte`, delete `<svelte:head><title>Everything · msgvault`. +AppShell now owns the title. The boot and login screens set theirs in Task 11. + +- [ ] **Step 6: Add the layout styles** + +Replace the `.app-shell`, `.brand`, `.app-shell :global(.kit-top-bar…)`, +`.archive-state…`, and `.appearance-controls` rules with: + +```css +.app-shell { + display: flex; + height: 100vh; + min-height: 100vh; + overflow: hidden; + background: var(--bg-primary); + color: var(--text-primary); +} + +.app-column { + display: flex; + min-width: 0; + flex: 1; + flex-direction: column; +} + +.app-top-bar { + display: flex; + min-height: var(--header-height); + align-items: center; + gap: var(--space-3); + padding: 0 var(--space-5); + background: var(--bg-surface); + border-bottom: 1px solid var(--border-default); +} + +.app-shell--narrow .app-top-bar { + padding: 0 var(--space-3); +} + +.app-top-bar__end { + display: flex; + align-items: center; + gap: var(--space-2); + margin-left: auto; +} + +.app-main { + display: flex; + min-height: 0; + flex: 1; + flex-direction: column; + overflow: hidden; +} +``` + +Keep `.files-shell` and `.archive-navigation-status`. + +- [ ] **Step 7: Run the tests to verify they pass** + +Run: `bunx vitest run src/lib/components/shell src/lib/components/search src/lib/components/explore` +Expected: PASS. Fix any older test that looked for the Everything search form +inside `main[aria-label="Everything"]`; the searchbox is now in the top bar. + +- [ ] **Step 8: Check types and build** + +Run: `bun run check && bun run check:kit-ui && bun run build` +Expected: no errors or warnings. + +- [ ] **Step 9: Commit** + +Subject: `feat(web): move navigation to a sidebar and search to the top bar`. + +--- + +### Task 10: Page headers and sentence case in every workspace + +**Files:** +- Modify: `EverythingWorkspace.svelte`, `FilesWorkspace.svelte`, + `SavedViewsWorkspace.svelte`, `SourcesWorkspace.svelte`, + `OperationsWorkspace.svelte`, `OperationRelatedStatus.svelte`, + `DeletionsWorkspace.svelte`, `DirectoryWorkspace.svelte`, + `DirectoryReviewCentre.svelte`, `RelationshipsWorkspace.svelte`, + `SettingsWorkspace.svelte`, `PersonTimeline.svelte` (eyebrow only), + and `AppShell.svelte` (`.files-shell`) +- Test: `web/src/lib/components/shell/AppShell.test.ts` + +**Interfaces:** +- Consumes: `PageHeader` (Task 8). + +Use these titles and descriptions. Existing descriptions are kept where they +already say what the page is for. + +| Workspace | Title | Description | Actions and view row | +|---|---|---|---| +| Relationships | Relationships | People and domains you've exchanged messages with. | none | +| Directory | Directory | People you've saved, with profiles and contact details. | Promote to person (when present) | +| Reviews | Reviews | Decide which identities and facts belong together. | view row: the existing "Review type" control | +| Everything | Everything | none | `actions`: the existing Preview position control and result count | +| Files | Files (Attachments when person-scoped) | none | file count in `actions` | +| Saved views | Saved views | Searches and layouts you've saved to reuse. | none | +| Sources | Sources | Accounts and imports in your archive, and when they last synced. | View source operations | +| Operations | Operations | Background work and its history. | Refresh operations | +| Deletions | Deletions | Deletions you've staged. Nothing is deleted until you run `msgvault delete-staged`. | none | +| Settings | Settings | How this daemon and the web app behave. | none | + +The "Live authority" related-status view in `OperationRelatedStatus.svelte` +keeps its own `h1`, because it replaces the Operations page body. Remove only +its eyebrow paragraph. + +- [ ] **Step 1: Write the failing test** + +```ts +it.each([ + ['relationships', 'Relationships'], ['directory', 'Directory'], ['directory_review', 'Reviews'], + ['everything', 'Everything'], ['files', 'Files'], ['saved_views', 'Saved views'], + ['sources', 'Sources'], ['operations', 'Operations'], ['deletions', 'Deletions'] +])('shows one visible page title in %s', async (workspace, title) => { + window.history.replaceState(null, '', `/?explore=${encodeURIComponent(JSON.stringify({ workspace }))}`); + const state = new ExploreState(window); + render(AppShell, { client: createAPIClient(vi.fn(async () => Response.json(exploreResponse()))), state, enabled: false }); + const headings = await screen.findAllByRole('heading', { level: 1 }); + expect(headings.map((heading) => heading.textContent?.trim())).toEqual([title]); + expect(headings[0]!.closest('.kit-sr-only')).toBeNull(); + expect(screen.queryByText(/archive workspace|archive operations/i)).toBeNull(); + state.destroy(); +}); +``` + +Settings is covered separately, because it renders through the `settings` +snippet. Add a test in `SettingsWorkspace.test.ts` that finds the visible +level-one heading "Settings". + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `bunx vitest run src/lib/components/shell/AppShell.test.ts -t "page title"` +Expected: FAIL for Relationships (screen-reader-only title), Saved views +(title case), and the workspaces with eyebrows. + +- [ ] **Step 3: Apply `PageHeader`** + +For each workspace, replace its header markup with `` using the +table above. Move existing action buttons into the `actions` snippet without +changing their labels or handlers. Delete eyebrow paragraphs and their CSS. +Delete each workspace's local `h1` styles. + +Use the same outer padding everywhere: +`padding: var(--space-5) var(--space-6) var(--space-4);`, with +`padding-inline: var(--space-4)` below 760px. Data workspaces fill the width: +remove `max-width` and `margin-inline: auto` from `.everything-workspace`, +`.files-shell`, and `.saved-views`. Saved views keeps a readable width on its +form content only: `max-width: 960px` on the form and list, not on the page. + +For Relationships, render `PageHeader` above the list and detail split, and +remove the screen-reader-only `h1`. For Settings, replace the +screen-reader-only `h1` with `PageHeader` above `SettingsLayout`. + +For Files, move the Everything `ContextBar` in `AppShell.svelte`'s `files-shell` +below the Files `PageHeader`. Do this by moving `FilesWorkspace`'s header into +`AppShell`'s `files-shell` above `ContextBar`. `FilesWorkspace` keeps rendering +its own header when `personScoped`, because Relationships and Directory embed +it without the shell. + +- [ ] **Step 4: Apply sentence case** + +Change these visible strings, keeping the accessible names the spec names: +- `main aria-label="Saved Views"` → `"Saved views"`; heading and empty state + "No Saved Views yet" → "No saved views yet"; "Loading Saved Views…" → + "Loading saved views…"; modal "Delete Saved View?" → "Delete saved view?" +- The command palette entry "Open Reviews" keeps its label; it names a + workspace. + +Then run `rg -n "Saved Views" src --glob '*.svelte'` and change each remaining +visible occurrence the same way. Playwright's default name match is +case-insensitive, so specs that use `'Saved Views'` without `exact: true` keep +working. Task 12 fixes any spec that uses `exact: true`. + +- [ ] **Step 5: Run the tests to verify they pass** + +Run: `bunx vitest run src/lib/components` +Expected: PASS. + +- [ ] **Step 6: Check and commit** + +Run: `bun run check && bun run check:kit-ui` +Subject: `feat(web): give every workspace the same page header`. + +--- + +### Task 11: Sign-in and boot screens + +**Files:** +- Modify: `web/src/App.svelte` (boot markup and styles) +- Modify: `web/src/lib/components/auth/Login.svelte` +- Test: `web/src/App.test.ts` + +- [ ] **Step 1: Write the failing test** + +Add to `App.test.ts`, following its existing session fixtures: + +```ts +it('titles the connecting screen msgvault', () => { + const session = createSessionController(() => new Promise(() => undefined)); + render(App, { session }); + expect(screen.getByRole('main', { name: 'Connecting' })).toBeDefined(); + expect(document.title).toBe('msgvault'); +}); +``` + +The eyebrow removal is visual and is checked in the Task 13 screenshots. + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `bunx vitest run src/App.test.ts -t "connecting screen msgvault"` +Expected: FAIL. + +- [ ] **Step 3: Implement** + +In `App.svelte`, add +`{#if !shellMounted || messageID !== undefined}msgvault{/if}`. +Replace each `

msgvault

` with a brand line matching the +sidebar brand: `

msgvault

`. Use sentence case and +palette tokens in the `.boot` styles: 14px base, `--text-muted` for secondary +text, one solid "Retry" button. + +In `Login.svelte`, make the same eyebrow change. Keep the heading "Log in" +and every field name. + +- [ ] **Step 4: Run the tests to verify they pass** + +Run: `bunx vitest run src/App.test.ts src/lib/components/auth` +Expected: PASS. + +- [ ] **Step 5: Commit** + +Subject: `feat(web): match the sign-in and boot screens to the new palette`. + +--- + +### Task 12: Browser tests + +**Files:** +- Modify: `web/tests/kit-ui.ts` +- Modify: `web/tests/density-restoration.spec.ts`, + `web/tests/theme-keyboard.spec.ts`, + `web/tests/docs-fixture-screenshots.spec.ts`, + `web/tests/e2e/accessibility.spec.ts`, and any spec found by the searches in + Step 1 +- Create: `web/tests/shell-navigation.spec.ts` + +**Interfaces:** +- Produces in `tests/kit-ui.ts`: + - `selectWorkspace(page, label)` — replaces `selectKitTopBarTab` + - `setTemporaryDensity(page, 'Auto' | 'Compact' | 'Comfortable')` + +- [ ] **Step 1: Find the affected specs** + +Run: +```bash +rg -ln "selectKitTopBarTab|Temporary density|Density: |name: 'Primary'|keyboard-help|Preview position" tests +rg -n "exact: true" tests | rg -i "saved views" +``` + +- [ ] **Step 2: Update the helpers** + +Replace `selectKitTopBarTab` in `tests/kit-ui.ts` with: + +```ts +export async function selectWorkspace(page: Page, label: string): Promise { + const opener = page.getByRole('button', { name: 'Open navigation' }); + if (await opener.isVisible()) await opener.click(); + await page.getByRole('navigation', { name: 'Primary' }).getByRole('button', { name: label, exact: true }).click(); +} + +export async function setTemporaryDensity( + page: Page, + density: 'Auto' | 'Compact' | 'Comfortable' +): Promise { + await page.getByRole('button', { name: 'Display' }).click(); + await page.getByRole('menuitemradio', { name: density }).click(); + await page.keyboard.press('Escape'); +} +``` + +Rename every `selectKitTopBarTab` call to `selectWorkspace`, and use sentence +case labels ("Saved views"). Replace every +`selectKitOption(page, 'Temporary density', 'Density: X')` with +`setTemporaryDensity(page, 'X')`. Replace assertions on the combobox +"Temporary density: …" with assertions on the checked radio: +`page.getByRole('menuitemradio', { name: 'Compact', checked: true })` after +opening the Display menu. + +- [ ] **Step 3: Write the new spec** + +`tests/shell-navigation.spec.ts` uses the same archive fixture routing as +`session-navigation.spec.ts`; copy its `beforeEach` setup. + +```ts +import { expect, test } from '@playwright/test'; + +import { selectWorkspace } from './kit-ui'; + +test('sidebar rail keeps names and survives reload', async ({ page }) => { + await page.goto('/?workspace=everything'); + await page.getByRole('button', { name: 'Collapse sidebar' }).click(); + await expect(page.getByRole('navigation', { name: 'Primary' }).getByRole('button', { name: 'Saved views' })).toBeVisible(); + await page.reload(); + await expect(page.getByRole('button', { name: 'Expand sidebar' })).toBeVisible(); +}); + +test('narrow navigation menu traps focus and closes three ways', async ({ page }) => { + await page.setViewportSize({ width: 420, height: 860 }); + await page.goto('/?workspace=everything'); + const opener = page.getByRole('button', { name: 'Open navigation' }); + + await opener.click(); + await expect(page.getByRole('button', { name: 'Everything' })).toBeFocused(); + for (let index = 0; index < 20; index += 1) await page.keyboard.press('Tab'); + await expect(page.getByRole('dialog', { name: 'Navigation' }).locator(':focus')).toHaveCount(1); + await page.keyboard.press('Escape'); + await expect(opener).toBeFocused(); + + await opener.click(); + await page.getByRole('button', { name: 'Close navigation' }).click({ position: { x: 400, y: 400 } }); + await expect(opener).toBeFocused(); + + await selectWorkspace(page, 'Sources'); + await expect(page.getByRole('heading', { level: 1, name: 'Sources' })).toBeVisible(); + await expect(page.getByRole('dialog', { name: 'Navigation' })).toHaveCount(0); +}); + +test('Escape in the narrow menu leaves an open reading pane open', async ({ page }) => { + await page.setViewportSize({ width: 420, height: 860 }); + await page.goto('/?workspace=everything'); + await page.getByRole('grid', { name: 'Everything results' }).getByRole('row').nth(1).click(); + await expect(page.getByRole('complementary', { name: /^Reading pane/ })).toBeVisible(); + await page.getByRole('button', { name: 'Open navigation' }).click(); + await page.keyboard.press('Escape'); + await expect(page.getByRole('complementary', { name: /^Reading pane/ })).toBeVisible(); +}); + +test('global search from another workspace opens Everything', async ({ page }) => { + await page.goto('/?workspace=sources'); + await page.getByRole('searchbox', { name: 'Search everything' }).fill('fixture'); + await page.keyboard.press('Enter'); + await expect(page.getByRole('main', { name: 'Everything' })).toBeVisible(); + await page.goBack(); + await expect(page.getByRole('main', { name: 'Sources' })).toBeVisible(); +}); +``` + +- [ ] **Step 4: Run the browser tests** + +From the repository root, run: `make web-test-browser` +Expected: all specs pass, including `e2e/accessibility` (axe) in both themes. + +- [ ] **Step 5: Commit** + +Subject: `test(web): cover the sidebar, navigation menu, and global search`. +In the body, list every existing spec edit and why it was needed, as the spec's +"Tests and accessible names" section requires. + +--- + +### Task 13: Verify and prepare the pull request + +- [ ] **Step 1: Run the full web checks** + +From the repository root: +`make web-check && make web-test && make web-test-browser && make lint-ci` +Expected: all pass with no warnings. + +- [ ] **Step 2: Capture after screenshots** + +Rebuild the branch binary with `make build` and restart the isolated fixture +daemon with the new binary. Capture the same set as Task 1 Step 1, plus the +collapsed rail at 1440×900 and the open narrow menu at 420×860, into `after/`. +View every image. Check that: +- no page shows an eyebrow, uppercase label, or duplicate title; +- one blue primary button at most per screen; +- the top bar never wraps at 420px; +- dark mode has no unreadable text. + +- [ ] **Step 3: Review the diff** + +Run `git diff main...HEAD --stat` and read the full diff. Remove unused +imports, styles, and props. Confirm nothing in `web/src/lib/api/generated` +changed. + +- [ ] **Step 4: Open the pull request** + +Use the `kenn-io-commit-push-pr` skill, and the `kenn-io-pr-desc` skill for +the body. Run the private-data scrub on the body and on every screenshot. The +Enron fixture captures are the documented exception in `AGENTS.md`. + +The body describes the result for people using the Web UI, what reviewers +should check (accessible-name changes, kit overrides in `app.css`, the palette +outside `src/`), and before and after screenshots. It has no validation or +test-plan section. + +## Self-review notes + +- **Spec coverage for delivery item 1:** + - Palette, type, and status tones: Task 1. + - Sidebar with rail, footer, and collapse storage: Task 5 and Task 9. + - Narrow menu with scrim, Escape, focus trap, focus restoration, and Escape + precedence: Task 6, Task 9, and Task 12. + - Top bar, global search, and `/`: Task 3, Task 4, and Task 9. + - Display menu: Task 7 and Task 9. + - `PageHeader` on every workspace: Task 8 and Task 10. + - Keyboard registry and footer removal: Task 2 and Task 9. + - Tab titles: Task 9 and Task 11. + - Sign-in and boot screens: Task 11. +- **Deferred to later pull requests, per the spec:** empty-state copy changes + in Deletions and Saved views, the code-label maps, the selection-bar + redesign, and everything else in delivery items 2 through 4. +- **Focus after choosing a drawer item** returns to "Open navigation". This is + the drawer's equivalent of today's rule, where focus stays on the tab you + clicked. diff --git a/docs/internal/web-ui-overhaul-pr2-plan.md b/docs/internal/web-ui-overhaul-pr2-plan.md new file mode 100644 index 000000000..90eb94007 --- /dev/null +++ b/docs/internal/web-ui-overhaul-pr2-plan.md @@ -0,0 +1,941 @@ +# Web UI overhaul PR 2: Everything, Files, and Saved views — implementation plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use +> superpowers:subagent-driven-development (recommended) or +> superpowers:executing-plans to implement this plan task by task. Steps use +> checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Give Everything and Files one toolbar row, readable context chips, a +selection bar that appears only when needed, a single Files view, and a Save +view dialog next to the view it saves, without removing any capability. + +**Architecture:** `ContextBar` becomes the one toolbar for Everything and +Files. It takes a sort configuration, a count label, and optional slots instead +of fixed controls. Label maps live in one new module. Saved-view conversion +moves out of `SavedViewsWorkspace` into a module shared by a new +`SaveViewDialog`. A rule in `normalize()` in `explore/state.svelte.ts` sends +every Everything-as-Files state to the Files workspace. + +**Tech stack:** Svelte 5 (runes), `@kenn-io/kit-ui` (`FilterDropdown`, +`SelectDropdown`, `Menu`, `Modal`, `IconButton`), `@lucide/svelte`, Vitest +with Testing Library, Playwright. + +**Spec:** [Web UI overhaul](web-ui-overhaul-design.md), sections +"Everything", "Files", "Saved views", and "Control relocation". This plan +covers delivery item 2. PR 1 is merged (#995); its shell, `PageHeader`, +palette, and focus rules are the baseline. + +## Global constraints + +- Keep every capability. A control that moves keeps its accessible name unless + this plan or the spec names the change. +- Keep URL state keys, the `explore` JSON format, and API calls unchanged. The + only state rule added is the Files normalization in Task 2. +- No new npm dependencies. Icons come from `@lucide/svelte`. +- One solid primary button per screen: `tone="info" surface="solid"`. Purple + (`workflow`) and green (`success`) are not button colors. +- Sentence case for every visible label. Monospace only for identifiers, + hashes, code, keys, and cron text. Raw colors only in `web/palette.css`. +- Raw API codes are never the only text a person sees. Each displayed code gets + a readable label; the raw value may stay in a `title` tooltip. +- Tests use Vitest and Testing Library or Playwright, query by role and + accessible name, and use failure-safe cleanup (`afterEach` or + `try`/`finally`). +- Run commands from `web/` unless a step says otherwise. After each task run + `bun run check`, `bun run check:kit-ui`, and the focused tests. +- Commit after each task with the `kenn-io-commit` rules: new commits only, + conventional imperative subject, a short why body, the attribution trailer + the controller gives you, and a private-data check (public repository). + +## File structure + +| File | Responsibility | +|---|---| +| `web/src/styles/tokens.css` | New `--page-gutter` token | +| `web/src/lib/explore/labels.ts` (new) | Readable labels for search modes, filter dimensions, presentations, file types, and preflight reasons | +| `web/src/lib/explore/state.svelte.ts` | Files normalization in `normalize()` | +| `web/src/lib/components/explore/ContextBar.svelte` | The one toolbar: Filters, Show as, Group by, Sort, extra controls slot, count; removable chips; filter panel | +| `web/src/lib/components/explore/EverythingTable.svelte` | Columns become fully controlled; the `
` picker leaves | +| `web/src/lib/components/explore/ColumnsMenu.svelte` (new) | Columns picker as a kit `FilterDropdown` | +| `web/src/lib/components/explore/SelectionBar.svelte` | Compact strip below results, only while something is selected; Review for deletion; overflow menu | +| `web/src/lib/components/shell/EverythingWorkspace.svelte` | Toolbar composition, notices below the toolbar, selection bar placement, Save view action | +| `web/src/lib/components/reader/ReadingPane.svelte` | Tasks button, icon Close, readable meta strip | +| `web/src/lib/components/tasks/TaskLinks.svelte` | Reports its linked-task count | +| `web/src/lib/components/files/FilesWorkspace.svelte` | Type menu, readable Type column, containing-item row action, Visual search label, no duplicate region | +| `web/src/lib/components/explore/FilesPresentation.svelte` (+ test) | Deleted | +| `web/src/lib/components/shell/AppShell.svelte` | Files toolbar wiring, Show as routing, Save view action, deletion review from the bar, Escape containment, stale count reset | +| `web/src/lib/components/directory/PersonDetail.svelte` | Media & files filters and sort work | +| `web/src/lib/saved-views/canonical.ts` (new) | Explore state ⇄ saved-view canonical state, readable summary | +| `web/src/lib/components/saved-views/SaveViewDialog.svelte` (new) | Name, Description, Save; Files note | +| `web/src/lib/components/saved-views/SavedViewsWorkspace.svelte` | Library only | + +--- + +### Task 1: One page gutter + +The page padding is repeated in about ten workspaces, and the top bar uses a +different inset (`--space-5`) from the pages (`--space-6`), so their left +edges do not line up. + +**Files:** +- Modify: `web/src/styles/tokens.css` +- Modify: every workspace root that sets + `padding: var(--space-5) var(--space-6) var(--space-4)` or + `padding-inline: var(--space-4)` at 760px (find them with + `rg -n "space-6\) var\(--space-4\)|padding-inline: var\(--space-4\)" src --glob '*.svelte'`), + and `.app-top-bar` in `AppShell.svelte` + +- [ ] **Step 1: Add the token** + +Append to the `:root` block in `tokens.css`: + +```css + --page-gutter: var(--space-6); +``` + +and after that block: + +```css +@media (max-width: 760px) { + :root { + --page-gutter: var(--space-4); + } +} +``` + +- [ ] **Step 2: Use it** + +Replace each workspace root's horizontal padding with +`padding: var(--space-5) var(--page-gutter) var(--space-4);` and delete the +now-redundant 760px `padding-inline` overrides. Set `.app-top-bar` horizontal +padding to `var(--page-gutter)` and delete its narrow override. + +- [ ] **Step 3: Verify** + +Run `bun run check && bun run check:kit-ui && bun run test`. Capture +`everything`, `sources`, and `settings` at 1440 and 420 with the controller's +screenshot helper, and confirm the top-bar search box and page titles share a +left edge. + +- [ ] **Step 4: Commit** + +Subject: `refactor(web): share one page gutter across the shell`. + +--- + +### Task 2: Everything-as-Files links open the Files workspace + +**Files:** +- Modify: `web/src/lib/explore/state.svelte.ts` (`normalize()`, around the + `presentation` and `workspace` derivations) +- Test: `web/src/lib/explore/state.test.ts`, `web/src/lib/components/shell/AppShell.test.ts` + +**Interfaces:** +- Produces: after `normalize()`, `workspace === 'files'` implies + `presentation === 'files'`, and `presentation === 'files'` implies + `workspace === 'files'`. Everything never holds `presentation: 'files'`. + +- [ ] **Step 1: Write the failing tests** + +In `state.test.ts`: + +```ts +it('sends an Everything-as-Files link to the Files workspace with its context', () => { + const restored = parseExploreURLState( + `?workspace=everything&mode=hybrid&explore=${encodeURIComponent(JSON.stringify({ + presentation: 'files', query: 'invoice', filters: [{ dimension: 'source', values: ['7'] }], + groupingChain: ['year'], columns: ['kind', 'title'] + }))}` + ); + expect(restored).toMatchObject({ + workspace: 'files', presentation: 'files', searchMode: 'hybrid', query: 'invoice', + filters: [{ dimension: 'source', values: ['7'] }], groupingChain: ['year'], columns: ['kind', 'title'] + }); +}); + +it('keeps Files presentation in the Files workspace', () => { + const restored = parseExploreURLState( + `?workspace=files&explore=${encodeURIComponent(JSON.stringify({ presentation: 'table' }))}` + ); + expect(restored.presentation).toBe('files'); +}); + +it('restores Files for an old Everything-as-Files history entry', async () => { + window.history.replaceState(null, '', '/?workspace=sources'); + const state = new ExploreState(window); + try { + window.history.pushState( + { + exploreSearch: '?workspace=everything', + exploreState: { workspace: 'everything', presentation: 'files', query: 'budget' } + }, + '', + '/?workspace=everything' + ); + window.history.pushState(null, '', '/?workspace=sources'); + const restored = new Promise((resolve) => window.addEventListener('popstate', resolve, { once: true })); + window.history.back(); + await restored; + expect(state.current).toMatchObject({ workspace: 'files', presentation: 'files', query: 'budget' }); + } finally { + state.destroy(); + } +}); +``` + +`readURLState()` uses `history.state.exploreState` only when +`history.state.exploreSearch` equals `location.search`, which the pushed entry +above satisfies, so the test exercises the real popstate path. + +In `AppShell.test.ts`, add a test that loads +`?workspace=everything&explore={"presentation":"files","selectedRow":"attachment:5"}` +with a fetch mock that answers the attachment metadata endpoint the +contextual viewer uses (see `openContextualFile` and the effect that reopens +the viewer from `parseAttachmentSelection`), and asserts the page shows +`main` "Files" and the dialog "View ". + +- [ ] **Step 2: Run to verify they fail** + +Run: `bunx vitest run src/lib/explore/state.test.ts src/lib/components/shell/AppShell.test.ts -t "Files"` +Expected: FAIL; the workspace stays `everything`. + +- [ ] **Step 3: Implement** + +In `normalize()`, after `workspace` and `presentation` are derived: + +```ts + const filesView = workspace === 'files' || (workspace === 'everything' && presentation === 'files'); + const normalizedWorkspace = filesView ? 'files' : workspace; + const normalizedPresentation = filesView + ? 'files' + : presentation === 'files' ? defaultExploreURLState.presentation : presentation; +``` + +Use `normalizedWorkspace` and `normalizedPresentation` in the returned object. +When the address bar holds a link that normalizes differently, the existing +constructor path already rewrites it; confirm the URL after load reads +`workspace=files` and add that assertion to the AppShell test. + +If the contextual viewer effect only runs for `workspace === 'everything'`, +extend it to `files` so `attachment:` reopens the viewer there. + +- [ ] **Step 4: Run to verify they pass, then run `bunx vitest run src/lib/explore src/lib/components/shell`** + +- [ ] **Step 5: Commit** + +Subject: `feat(web): open Everything-as-Files links in the Files workspace`. + +--- + +### Task 3: Readable labels + +**Files:** +- Create: `web/src/lib/explore/labels.ts` +- Test: `web/src/lib/explore/labels.test.ts` + +**Interfaces:** +- Produces: + +```ts +export function searchModeLabel(mode: ExploreSearchMode): string; +export function filterDimensionLabel(dimension: ExploreFilterDimension): string; +export function presentationLabel(presentation: ExploreURLState['presentation']): string; +export function fileTypeLabel(mimeType: string | undefined, family: FileMIMEFamily | undefined): string; +export const FILE_FAMILY_LABELS: Record; +export function preflightReasonLabel(action: string, reason: string): string; +``` + +- [ ] **Step 1: Write the failing tests** + +```ts +import { describe, expect, it } from 'vitest'; + +import { + fileTypeLabel, filterDimensionLabel, preflightReasonLabel, presentationLabel, searchModeLabel +} from './labels'; + +describe('explore labels', () => { + it('names search modes and presentations', () => { + expect(searchModeLabel('full_text')).toBe('Full text'); + expect(searchModeLabel('semantic')).toBe('Semantic'); + expect(searchModeLabel('hybrid')).toBe('Hybrid'); + expect(presentationLabel('timeline')).toBe('Timeline'); + }); + + it('names filter dimensions and falls back to sentence case', () => { + expect(filterDimensionLabel('source')).toBe('Source'); + expect(filterDimensionLabel('participant')).toBe('Person'); + expect(filterDimensionLabel('message_type' as never)).toBe('Message type'); + }); + + it('names file types from the MIME type, then the family', () => { + expect(fileTypeLabel('application/pdf', 'pdf')).toBe('PDF'); + expect(fileTypeLabel('image/png', 'image')).toBe('PNG image'); + expect(fileTypeLabel('image/webp', 'image')).toBe('WEBP image'); + expect(fileTypeLabel('application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'document')) + .toBe('Word document'); + expect(fileTypeLabel('', 'archive')).toBe('Archive'); + expect(fileTypeLabel(undefined, undefined)).toBe('Unknown type'); + }); + + it('turns preflight reasons into sentences', () => { + expect(preflightReasonLabel('open_in_source', 'trusted_source_link_unavailable')) + .toBe('Your sources don’t provide links to open these items.'); + expect(preflightReasonLabel('export', 'browser_export_requires_single_message')) + .toBe('Export works for one message at a time.'); + expect(preflightReasonLabel('export', 'some_new_reason')).toBe('Some new reason.'); + }); +}); +``` + +Use the real `ExploreFilterDimension` union from `explore/models.ts`: add a +label for every member and keep the fallback for unknown values. Use +`groupingDimensionLabel` from `grouping/catalog.ts` where a dimension is also a +grouping, so the two never disagree. + +- [ ] **Step 2: Run to verify it fails** (module missing). + +- [ ] **Step 3: Implement** + +```ts +import { groupingDimensionLabel, isGroupingDimension } from '../grouping/catalog'; +import type { ExploreFilterDimension, ExploreSearchMode, ExploreURLState, FileMIMEFamily } from './models'; + +const SEARCH_MODES: Record = { + full_text: 'Full text', + semantic: 'Semantic', + hybrid: 'Hybrid' +}; + +const PRESENTATIONS: Record = { + table: 'Table', + timeline: 'Timeline', + files: 'Files' +}; + +const FILTER_DIMENSIONS: Partial> = { + participant: 'Person', + identity: 'Identity' +}; + +export const FILE_FAMILY_LABELS: Record = { + image: 'Images', + pdf: 'PDFs', + audio: 'Audio', + video: 'Video', + text: 'Text', + document: 'Documents', + archive: 'Archives', + other: 'Other' +}; + +const FAMILY_SINGULAR: Record = { + image: 'Image', + pdf: 'PDF', + audio: 'Audio', + video: 'Video', + text: 'Text', + document: 'Document', + archive: 'Archive', + other: 'File' +}; + +const KNOWN_MIME: Record = { + 'application/pdf': 'PDF', + 'application/zip': 'ZIP archive', + 'application/msword': 'Word document', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': 'Word document', + 'application/vnd.ms-excel': 'Excel spreadsheet', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': 'Excel spreadsheet', + 'application/vnd.ms-powerpoint': 'PowerPoint presentation', + 'application/vnd.openxmlformats-officedocument.presentationml.presentation': 'PowerPoint presentation', + 'text/plain': 'Text', + 'text/html': 'HTML', + 'text/csv': 'CSV', + 'text/calendar': 'Calendar invite', + 'message/rfc822': 'Email message' +}; + +const REASONS: Record = { + 'open_in_source:trusted_source_link_unavailable': 'Your sources don’t provide links to open these items.', + 'export:browser_export_requires_single_message': 'Export works for one message at a time.', + 'export:selection_has_no_exportable_raw_message': 'The selection has no original message to export.', + 'export:raw_message_unavailable': 'The original message isn’t available.', + 'export_files:selection_contains_no_files': 'The selection has no files.', + 'stage_deletion:selection_contains_items_that_cannot_be_deleted_from_source': + 'Some selected items can’t be deleted from their source.' +}; + +function sentenceCase(code: string): string { + const words = code.replace(/[_-]+/g, ' ').trim(); + return words ? words[0]!.toUpperCase() + words.slice(1) : ''; +} + +export function searchModeLabel(mode: ExploreSearchMode): string { + return SEARCH_MODES[mode]; +} + +export function presentationLabel(presentation: ExploreURLState['presentation']): string { + return PRESENTATIONS[presentation]; +} + +export function filterDimensionLabel(dimension: ExploreFilterDimension): string { + const explicit = FILTER_DIMENSIONS[dimension]; + if (explicit) return explicit; + if (isGroupingDimension(dimension)) return groupingDimensionLabel(dimension); + return sentenceCase(dimension); +} + +export function fileTypeLabel(mimeType: string | undefined, family: FileMIMEFamily | undefined): string { + const mime = mimeType?.toLowerCase().split(';')[0]?.trim() ?? ''; + const known = KNOWN_MIME[mime]; + if (known) return known; + const [kind, subtype] = mime.split('/'); + if ((kind === 'image' || kind === 'audio' || kind === 'video') && subtype) { + return `${subtype.replace(/^x-/, '').toUpperCase()} ${kind}`; + } + return family ? FAMILY_SINGULAR[family] : 'Unknown type'; +} + +export function preflightReasonLabel(action: string, reason: string): string { + return REASONS[`${action}:${reason}`] ?? `${sentenceCase(reason)}.`; +} +``` + +If `groupingDimensionLabel` for a dimension returns something that reads +wrongly as a filter chip, add an explicit `FILTER_DIMENSIONS` entry instead of +changing the grouping catalog. + +- [ ] **Step 4: Run to verify it passes.** + +- [ ] **Step 5: Commit** + +Subject: `feat(web): add readable labels for explore codes`. + +--- + +### Task 4: One toolbar row with removable chips + +**Files:** +- Modify: `web/src/lib/components/explore/ContextBar.svelte` +- Create: `web/src/lib/components/explore/ColumnsMenu.svelte` +- Modify: `web/src/lib/components/explore/EverythingTable.svelte` + (delete the `
` Columns picker at ~lines 426-437; keep + `columns` controlled by the prop) +- Modify: `web/src/lib/components/shell/EverythingWorkspace.svelte` +- Modify: `web/src/lib/components/shell/AppShell.svelte` + (`openContextControl`, `fixedSortNotice`) +- Test: `ContextBar.test.ts`, `ColumnsMenu.test.ts` (new), + `EverythingTable.test.ts`, `EverythingWorkspace.test.ts`, `AppShell.test.ts` + +**Interfaces:** +- Consumes: `labels.ts` (Task 3). +- Produces the new `ContextBar` props (existing ones unchanged unless listed): + - `sort: { options: { value: string; label: string }[]; value: string; note?: string; onchange?: (value: string) => void }` + rendered as a kit `SelectDropdown` titled "Sort" inside + `
`. Accessible name: "Sort: ". + - `countLabel: string` — the one visible count, at the right edge. + - `extra?: Snippet` — controls placed before the count (Columns, Preview + position). + - `onRemoveQuery?: () => void`, `onRemoveFilter?: (index: number) => void`. + - `totalCount` is removed; callers pass `countLabel`. +- `ColumnsMenu` props: `columns: ExploreColumn[]`, + `onchange: (columns: ExploreColumn[]) => void`. Kit `FilterDropdown` with + `label="Columns"`, one section of seven items (Kind, People / source, + Subject / title, Excerpt, Time, Attachments, Size). Toggling the last visible + column is ignored so at least one stays, matching the table's current rule. + +- [ ] **Step 1: Write the failing tests** + +`ContextBar.test.ts` (replace assertions that depended on the old strings): + +```ts +it('shows readable, removable chips for the query, filters, and groupings', async () => { + const onRemoveQuery = vi.fn(); + const onRemoveFilter = vi.fn(); + const onRemoveGroup = vi.fn(); + render(ContextBar, { + client: createAPIClient(vi.fn()), + query: 'network', searchMode: 'full_text', + filters: [{ dimension: 'source', values: ['7'] }], + groupingChain: ['year'], + countLabel: '20 items', + sort: { options: [{ value: 'newest', label: 'Newest first' }], value: 'newest' }, + onAddGroup: vi.fn(), onRemoveGroup, onClearFilters: vi.fn(), onFiltersChange: vi.fn(), + onRemoveQuery, onRemoveFilter + }); + expect(screen.getByText('Full text: “network”')).toBeTruthy(); + expect(screen.getByText('Source: 7')).toBeTruthy(); + expect(screen.queryByText(/full_text/)).toBeNull(); + await fireEvent.click(screen.getByRole('button', { name: 'Remove search' })); + await fireEvent.click(screen.getByRole('button', { name: 'Remove Source filter' })); + await fireEvent.click(screen.getByRole('button', { name: 'Remove Year grouping' })); + expect(onRemoveQuery).toHaveBeenCalledOnce(); + expect(onRemoveFilter).toHaveBeenCalledWith(0); + expect(onRemoveGroup).toHaveBeenCalledWith(0); + expect(screen.getAllByText('20 items')).toHaveLength(1); +}); + +it('hides the chip line when nothing is active', () => { + render(ContextBar, { + client: createAPIClient(vi.fn()), + query: '', searchMode: 'full_text', filters: [], groupingChain: [], + countLabel: '20 items', + sort: { options: [{ value: 'newest', label: 'Newest first' }], value: 'newest' }, + onAddGroup: vi.fn(), onRemoveGroup: vi.fn(), onClearFilters: vi.fn(), onFiltersChange: vi.fn() + }); + expect(screen.queryByText('All archive entries')).toBeNull(); + expect(screen.queryByRole('button', { name: /^Remove / })).toBeNull(); +}); +``` + +`ColumnsMenu.test.ts`: open "Columns", toggle "Excerpt" off, expect +`onchange` without `excerpt`; with only `['title']`, toggling "Subject / title" +does not call `onchange`. + +`AppShell.test.ts`: on Everything, press `s` and assert the combobox named +`/^Sort: Newest first/` has focus and its listbox shows the note "Other +orders aren't available yet"; press `r` and assert the "Sort status" live +region announces the fixed order. Assert the result count text appears once +in `main` "Everything". + +- [ ] **Step 2: Run to verify they fail.** + +- [ ] **Step 3: Implement** + 1. `ContextBar`: + - One `.context-controls` row: Filters button, "Show as" + `SelectDropdown`, "Group by" picker (unchanged), the Sort + `SelectDropdown` inside `data-sort-menu`, `{@render extra?.()}`, then + `{countLabel}` + pushed right. The count uses tabular sans (`data-mono`), not a box. + - A `.context-chips` line rendered only when a query, filter, or grouping + is active. Chips: + - Query: `${searchModeLabel(searchMode)}: “${query}”`, with an + `IconButton` named "Remove search" calling `onRemoveQuery`. + - Filters: `${filterDimensionLabel(dimension)}: ${values.join(', ')}`, + with an `IconButton` named `Remove ${label} filter`. + - Groupings: `Grouped by ${label}`, keeping the name + `Remove ${label} grouping`. + - Show the sort `note` as a disabled `SelectDropdown` option whose label + is the note text, if the kit renders disabled options; otherwise render + it as `title` on the trigger and mention the fallback in your report. + - Delete the "All archive entries" empty text. + 2. `ColumnsMenu.svelte`: kit `FilterDropdown` as specified; each item's + `closeOnSelect` is `false`. + 3. `EverythingTable`: delete the `
` block and its styles. Keep the + `columns` prop and derive `visibleColumns` from it (it no longer owns a + local copy). Leave `onColumnsChange` removal to the caller: delete the + prop if nothing else uses it. + 4. `EverythingWorkspace`: + - Remove the count and Preview position from `PageHeader` actions. + - Pass `countLabel`: the existing text ("N results shown" when the + candidate pool is saturated, "N items" when counted, "Counting…" while + loading). + - Pass `sort={{ options: [{ value: 'newest', label: 'Newest first' }], value: 'newest', note: 'Other orders aren’t available yet', onchange: fixedSortNotice }}`. + - `extra`: `ColumnsMenu` (table presentation only) wired to + `exploreState.replaceTransient({ columns })`, and the Preview position + `SegmentedControl` (unchanged names; still only when + `canPreviewRight`), with the visible "Preview position" text removed + (the control keeps its accessible name). + - `onRemoveQuery`: `commitSearch('', searchMode)`. + - `onRemoveFilter`: commit the filters without that index, resetting + `activeRow`, `selectedRow`, and `scrollAnchor` like `onFiltersChange`. + - Move the "More results may match" notice, `SearchCoverage`, and the + semantic scope note below `ContextBar`, above the results. + 5. `AppShell`: `openContextControl('sort')` and `fixedSortNotice` target + `[data-sort-menu] button` instead of `button[aria-label="Sort: newest first"]`. + +- [ ] **Step 4: Run** `bunx vitest run src/lib/components/explore src/lib/components/shell`. + +- [ ] **Step 5: Commit** + +Subject: `feat(web): put Everything's controls in one toolbar row`. + +--- + +### Task 5: Selection bar only when something is selected + +**Files:** +- Modify: `web/src/lib/components/explore/SelectionBar.svelte` +- Modify: `web/src/lib/components/shell/EverythingWorkspace.svelte` +- Modify: `web/src/lib/components/shell/AppShell.svelte` (pass + `onReviewDeletion`) +- Test: `SelectionBar.test.ts`, `AppShell.test.ts` + +**Interfaces:** +- Consumes: `preflightReasonLabel` (Task 3); AppShell's existing + `openDeletionReview(mode)`. +- Produces: `SelectionBar` prop `onReviewDeletion?: (mode: 'explicit' | 'all_matching') => void`. + Rendered only when `selection.count > 0` or `selection.mode === 'all_matching'`. + +- [ ] **Step 1: Write the failing tests** + +`SelectionBar.test.ts`: +- With an empty explicit selection, nothing renders (no "No items selected"). +- With 3 selected, "3 selected" and a button "Review for deletion…" render; + clicking it calls `onReviewDeletion('explicit')`; after + `selectAllMatching`, it calls `onReviewDeletion('all_matching')`. +- With a preflight whose `unavailable_actions` contains + `{ action: 'open_in_source', reason: 'trusted_source_link_unavailable' }`, + opening the menu "More selection actions" shows a disabled item "Open + selection in source" and the text "Your sources don’t provide links to open + these items."; the raw code is not visible text. +- With an export reason, the text reads the sentence from + `preflightReasonLabel`, not `Export: browser_export_requires_single_message`. + +`AppShell.test.ts`: select a row, click "Review for deletion…", assert the +workspace becomes `deletions` and the deletion preflight request is sent, the +same as pressing `d`. + +- [ ] **Step 2: Run to verify they fail.** + +- [ ] **Step 3: Implement** + - `SelectionBar`: wrap the markup in `{#if selection.mode === 'all_matching' || selection.count > 0}`. + Keep the status text, "Select all N matching items", "Export selection", + meeting-context export, and "Clear selection". Add + `Button label="Review for deletion…"` (neutral, not red) calling + `onReviewDeletion(selection.mode === 'all_matching' ? 'all_matching' : 'explicit')`. + Replace the "Open in source" text or button with a kit `Menu` whose trigger + is an `IconButton` named "More selection actions" (lucide `ellipsis`); it + holds `MenuItem` "Open selection in source" (disabled when a reason exists, + enabled and calling `onOpenInSource` otherwise) and, when disabled, the + reason sentence as muted text. Export's unavailable text becomes + `Export unavailable: ` with the raw code in `title`. + - `EverythingWorkspace`: move `` from above the table and + timeline to below them inside `.results-primary`, as a + `position: sticky; bottom: 0` strip with the surface background and a top + border. Pass `onReviewDeletion`. + - `AppShell`: thread `onReviewDeletion={openDeletionReview}` to + `EverythingWorkspace`. + +- [ ] **Step 4: Run** `bunx vitest run src/lib/components/explore src/lib/components/shell`. + +- [ ] **Step 5: Commit** + +Subject: `feat(web): show the selection bar only while items are selected`. + +--- + +### Task 6: Reading-pane header + +**Files:** +- Modify: `web/src/lib/components/reader/ReadingPane.svelte` +- Modify: `web/src/lib/components/tasks/TaskLinks.svelte` +- Test: `ReadingPane.test.ts` (find with `rg -l "ReadingPane" src --glob '*.test.ts'`), `TaskLinks.test.ts` + +**Interfaces:** +- Produces: `TaskLinks` bindable prop `linkedCount: number | undefined`, set + after its first successful lookup. + +- [ ] **Step 1: Write the failing tests** +- The `
`/`` "Tasks for this message" is replaced by a button + with the same accessible name and `aria-expanded`; clicking it shows the + "Linked tasks" section; after the lookup resolves with 2 linked tasks, the + button's visible text reads "Tasks 2". +- The close control is an icon button named "Close reading pane" with no + visible "Close" text. +- The meta strip shows readable kinds ("Email", "Chat", "Calendar event", + "Meeting") instead of raw `message_type` values; reuse the labels the + `RowKind` component already uses rather than adding a second map. + +- [ ] **Step 2: Run to verify they fail.** + +- [ ] **Step 3: Implement** + - Replace the disclosure with `Button size="sm" surface="outline"` named + "Tasks for this message", visible label `Tasks` plus the count when known, + toggling `tasksOpen`. Bind `linkedCount` from `TaskLinks`. The count + appears after the sheet has loaded once for the open message; it resets + when the selection changes (the existing effect that closes the sheet). + - Replace the Close `Button` with `IconButton` (lucide `x`) named "Close + reading pane". + - Build the meta strip from the readable kind label. + +- [ ] **Step 4: Run** `bunx vitest run src/lib/components/reader src/lib/components/tasks`. + +- [ ] **Step 5: Commit** + +Subject: `feat(web): make reading-pane actions visible and readable`. + +--- + +### Task 7: One Files view + +**Files:** +- Delete: `web/src/lib/components/explore/FilesPresentation.svelte` and its test +- Modify: `web/src/lib/components/files/FilesWorkspace.svelte` +- Modify: `web/src/lib/components/shell/AppShell.svelte` (files branch, + `currentGrid()`, file count reset) +- Modify: `web/src/lib/components/shell/EverythingWorkspace.svelte` + (remove the Files branch; Show as Files navigates) +- Test: `FilesWorkspace.test.ts`, `AppShell.test.ts`, `EverythingWorkspace.test.ts` + +**Interfaces:** +- Consumes: `ContextBar` `sort`/`countLabel` (Task 4), `fileTypeLabel`, + `FILE_FAMILY_LABELS` (Task 3), the Task 2 normalization. +- Produces: the Type menu lives in `FilesWorkspace`'s own `.file-controls` row, + so the shell, Relationships, and Directory embeds all get it. + +- [ ] **Step 1: Write the failing tests** +- `FilesWorkspace.test.ts`: + - The "MIME families" checkboxes are gone; a `FilterDropdown` named "Type" + lists "Images, PDFs, Audio, Video, Text, Documents, Archives, Other" + (person-scoped Media shows Images and Video only; person-scoped Files the + rest); choosing "PDFs" calls `onMIMEFamiliesChange(['pdf'])`. + - The Type column shows "PDF" for `application/pdf` with `title="application/pdf"`. + - Each row has a button named `Open containing item `; + clicking it calls `onOpenItem(entry_key)` and does not open the viewer. + - The toggle is named "Visual search". + - With `showHeader={false}`, no region named "Files" is rendered (the shell + owns the landmark). +- `AppShell.test.ts`: + - On Everything, choosing Show as "Files" lands in `main` "Files" with the + query and filters kept; in Files, Show as "Table" returns to Everything. + - In Files, the toolbar's Sort combobox lists "Newest first, Oldest first, + Filename A–Z, Filename Z–A, Largest first, Smallest first"; choosing + "Largest first" commits `fileSort` `{ field: 'size', direction: 'desc' }`. + - In Files, the toolbar count reads "N files" (not "Count pending"). + - Typing in the global search on Files changes the Files search request's + query (this also covers the PR 1 review gap on live typing in Files). + - Leaving Files and returning does not flash the previous count. +- `EverythingWorkspace.test.ts`: no grid named "Files in current context" is + ever rendered. + +- [ ] **Step 2: Run to verify they fail.** + +- [ ] **Step 3: Implement** + - Delete `FilesPresentation.svelte` and its test; remove its import and + branch from `EverythingWorkspace`. Remove `"Files in current context"` + from `currentGrid()`. Keep `openContextualFile` only if another caller + remains (`rg -n openContextualFile src`); delete it otherwise. + - Everything's `onPresentationChange('files')` commits + `{ workspace: 'files', presentation: 'files', activeRow: null, selectedRow: null, scrollAnchor: null }`. + - Files toolbar (AppShell `files-shell`): pass `sort` built from + `fileSort` with the six options above and `countLabel` from the bound + `fileCount` ("N files", "Counting…" while `null`). Reset `fileCount` to + `null` whenever the workspace changes. + - `FilesWorkspace`: + - Replace the MIME checkbox group with a kit `FilterDropdown` labelled + "Type" using `FILE_FAMILY_LABELS`, preserving `visibleMIMEFamilies` and + the effective-families logic. + - Rename the toggle to "Visual search"; keep its row and disclosure text. + - Type cell: `fileTypeLabel(row.mime_type, row.mime_family)` with + `title={row.mime_type || row.mime_family}`. + - Containing-item cell: keep the title text and add an `IconButton` or + text button named `Open containing item ${row.containing_title || row.entry_key}` + calling `onOpenItem?.(row.entry_key)`; the row click handler already + skips clicks inside buttons. + - Render the outer element as a `div` when `showHeader` is false and + `embedded` is true, so the shell's `main` "Files" is the only named + landmark. + - Source stays `row.source_identifier`; file rows carry no account display + name, which the spec allows ("when one is available"). + - Keep column-header sorting; it and the Sort menu drive the same `fileSort`. + +- [ ] **Step 4: Run** `bunx vitest run src/lib/components` and `bun run check`. + +- [ ] **Step 5: Commit** + +Subject: `feat(web): make Files the single file view`. + +--- + +### Task 8: Directory Media & files filters work + +**Files:** +- Modify: `web/src/lib/components/directory/PersonDetail.svelte` (~line 154) +- Test: `PersonDetail.test.ts` (find with `rg -l "PersonDetail" src --glob '*.test.ts'`) + +- [ ] **Step 1: Write the failing test** + +Render `PersonDetail` on the Media & files tab with a fetch mock for the +person files endpoint; type a filename, choose a Type, and click "Sort by +size"; assert each triggers a new files request carrying the filename, family, +and `sort` parameters. + +- [ ] **Step 2: Run to verify it fails** (no request changes today). + +- [ ] **Step 3: Implement** + +Hold `sort`, `filenameQuery`, and `mimeFamilies` in local `$state` in +`PersonDetail` and pass `onSortChange`, `onFilenameQueryChange` (debounced like +Relationships' embed; check `RelationshipsWorkspace.svelte` and copy its +approach), and `onMIMEFamiliesChange`. Reset them when `personID` changes. + +- [ ] **Step 4: Run** `bunx vitest run src/lib/components/directory`. + +- [ ] **Step 5: Commit** + +Subject: `fix(web): make Directory media and file filters take effect`. + +--- + +### Task 9: Save view dialog and saved-view library + +**Files:** +- Create: `web/src/lib/saved-views/canonical.ts` (+ `canonical.test.ts`) +- Create: `web/src/lib/components/saved-views/SaveViewDialog.svelte` (+ test) +- Modify: `web/src/lib/components/saved-views/SavedViewsWorkspace.svelte` +- Modify: `EverythingWorkspace.svelte` (header action), `AppShell.svelte` + (Files header action; stop passing `selection`) +- Test: `SavedViewsWorkspace.test.ts`, `AppShell.test.ts` + +**Interfaces:** +- Produces from `canonical.ts`: + +```ts +export type CanonicalState = /* moved from SavedViewsWorkspace unchanged */; +export function canonicalSavedViewState(state: ExploreURLState): CanonicalState; +export function exploreStateFromSavedView(saved: CanonicalState): Partial; +export function savedViewSummary(saved: CanonicalState): string[]; +``` + + `exploreStateFromSavedView` returns `workspace: 'files'` when + `presentation === 'files'`, else `'everything'`, with the same alias and + default handling `open()` has today. `savedViewSummary` returns readable + parts, for example `['Full text: “invoice”', 'Source: 7', 'Grouped by Year', 'Table']`. +- `SaveViewDialog` props: `client`, `state: ExploreURLState`, + `onSaved: (view: SavedView) => void`, `onclose: () => void`. Kit `Modal` + titled "Save view"; `TextInput` "Name" and "Description"; submit button + "Save" (`tone="info" surface="solid"`, disabled until a name is typed); on + Files (`state.workspace === 'files'`) a note "Filename, type, and file sort + aren’t saved with the view."; API errors shown in the dialog. + +- [ ] **Step 1: Write the failing tests** +- `canonical.test.ts`: round-trip an Everything state and a Files state; the + Files one opens in `files`; the summary reads as above; legacy aliases + `source_id` and `participant_id` map as today. +- `SaveViewDialog.test.ts`: Save is disabled with an empty name; saving posts + the canonical state and calls `onSaved`; the Files note shows only for Files. +- `SavedViewsWorkspace.test.ts`: no "Save this view" form; each view shows its + summary and the buttons "Open ", "Edit ", "Delete " + (names unchanged); the empty state reads "No saved views yet" and points to + "Save view…" in Everything and Files. +- `AppShell.test.ts`: "Save view…" exists in the Everything and Files headers + and opens the dialog; saving from Files and opening the view from Saved + views lands in Files. + +- [ ] **Step 2: Run to verify they fail.** + +- [ ] **Step 3: Implement** + - Move `CanonicalState`, `canonicalState()`, and the mapping in `open()` + into `canonical.ts`; `SavedViewsWorkspace` imports them. + - `SavedViewsWorkspace`: delete the "Save this view" card, the unused + `selection` and `currentState` props (confirm with `rg`), and their + styles; render the summary under each description; keep edit, delete, and + incompatibility behavior unchanged. + - Header actions: `Button surface="outline" label="Save view…"` in + Everything's `PageHeader` and the Files `PageHeader` in AppShell, opening + `SaveViewDialog` with `exploreState.current`. After saving, announce + "Saved view ." through the existing operation status region. + +- [ ] **Step 4: Run** `bunx vitest run src/lib/saved-views src/lib/components/saved-views src/lib/components/shell`. + +- [ ] **Step 5: Commit** + +Subject: `feat(web): save views from the page they describe`. + +--- + +### Task 10: Escape closes menus before panes + +Kit popovers let Escape reach the shell, which then closes the reading pane or +removes a grouping level while the user only meant to close a menu. Task 4 +and Task 7 add several menus to Everything and Files, so this becomes common. + +**Files:** +- Modify: `web/src/lib/components/shell/AppShell.svelte` (`handleEscape`) +- Test: `AppShell.test.ts`, `web/tests/shell-navigation.spec.ts` + +- [ ] **Step 1: Write the failing tests** +- Unit: with a message open in the reading pane, open "Columns", press + Escape: the menu closes and the reading pane stays. Same for the Sort + combobox and "More selection actions". +- Unit: after the narrow navigation menu unmounts, Escape closes the reading + pane again (the root scope is restored). +- Browser: the same Columns case in Chromium. + +- [ ] **Step 2: Run to verify they fail.** + +- [ ] **Step 3: Implement** + +At the start of `handleEscape`, return when the event target is inside an +open popover — the element that has focus is within `[role="menu"]`, +`[role="listbox"]`, `[role="dialog"]`, or a kit popover container — or when an +expanded trigger (`[aria-expanded="true"]`) has focus. Check how kit's +`dismissable` closes on Escape so the menu still closes; the shell must only +stop acting, not swallow the key. + +- [ ] **Step 4: Run** the unit tests and + `bunx playwright test tests/shell-navigation.spec.ts`. + +- [ ] **Step 5: Commit** + +Subject: `fix(web): let Escape close menus without closing panes`. + +--- + +### Task 11: Browser tests + +**Files:** +- Modify: `web/tests/presentations.spec.ts` (grid "Files in current context" + and region "Files presentation" → the Files workspace grid "Files results"), + `web/tests/archive-management.spec.ts` (open-in-source reason text, deletion + entry), `web/tests/theme-keyboard.spec.ts` (Saved views save flow and any + `kit-button--workflow` assertion), `web/tests/e2e/accessibility.spec.ts` + (add the Save view dialog, the Columns and Type menus, and a selected-row + selection bar), and any spec found in Step 1 + +- [ ] **Step 1: Find affected specs** + +```bash +rg -ln "Files in current context|Files presentation|Open in source:|Save this view|getByLabel\('Name'\)|kit-button--workflow|Columns|MIME families|Hosted visual search|No items selected|Count pending|Newest first" tests +``` + +- [ ] **Step 2: Update each spec to the new names and flows** without weakening + assertions. Add browser coverage for: Show as Files round trip and Back; + an old Everything-as-Files link; Save view from Files then open it; Review + for deletion from the selection bar. + +- [ ] **Step 3: Run** `make web-test-browser` from the repository root. + Expected: all pass, axe included. + +- [ ] **Step 4: Commit** + +Subject: `test(web): cover the Everything and Files toolbars and saved views`. +List every existing spec edit and why in the body. + +--- + +### Task 12: Verify and prepare the pull request + +- [ ] **Step 1:** From the repository root run + `make web-check && make web-test && make web-test-browser && make lint-ci`. +- [ ] **Step 2:** Rebuild and capture every workspace (controller helper) plus + Everything with a selection, the Columns menu open, the Save view dialog, + Files with the Type menu open, and the login screen at 420px. View every + image. Check: one toolbar row, count once, no raw codes, no purple or green + solid buttons on Everything, Files, or Saved views, and the login button is + not stretched oddly. +- [ ] **Step 3:** Read `git diff origin/main...HEAD` fully; remove leftovers. +- [ ] **Step 4:** Stop. The controller asks the user before pushing or opening + the pull request. + +## Self-review notes + +- Spec coverage (delivery item 2): toolbar, chips, Sort, Columns, Preview + position, count once, notices placement (Task 4); selection bar, Review for + deletion, Open in source overflow (Task 5); reading-pane header (Task 6); + one Files view, Show as symmetry, deletion of `FilesPresentation`, row + action, Type menu, readable Type, Sort, count, Visual search (Task 7); + normalization of old links, history, saved views (Task 2); Directory media + filters (Task 8); Save view dialog, library, Files note, unused prop + (Task 9). +- PR 1 follow-ups folded in: page gutter (Task 1), stale file count, nested + Files region, live typing on Files test (Task 7), Escape in kit popovers and + drawer-unmount Escape test (Task 10), purple Save button (Task 9), login + button width check (Task 12). +- Kit limits: `SegmentedControl` has no icons, so Preview position keeps its + text options "Below" and "Right" in the toolbar. Kit has no checkbox menu + item, so Columns and Type use `FilterDropdown`. +- The Tasks count appears after the sheet loads once for a message; fetching + on every row open only for a badge would add a request per selection. diff --git a/docs/internal/web-ui-overhaul-pr3-plan.md b/docs/internal/web-ui-overhaul-pr3-plan.md new file mode 100644 index 000000000..abc2a6efa --- /dev/null +++ b/docs/internal/web-ui-overhaul-pr3-plan.md @@ -0,0 +1,953 @@ +# Web UI overhaul PR 3: People — implementation plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use +> superpowers:subagent-driven-development (recommended) or +> superpowers:executing-plans to implement this plan task by task. Steps use +> checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Give Relationships, Directory, and Reviews the toolbar, chip, label, +and header patterns Everything and Files already use; split the person page +into sections; and link the three People workspaces to each other, without +removing any capability. + +**Architecture:** Two small modules hold the new shared logic: +`directory/labels.ts` (readable labels and status tones) and +`directory/dates.ts` (calendar-date and partial-date validation). Directory's +filters move into a disclosure panel with chips. `PersonDetail` gets a header +and a data-driven section list; rename, delete, and profile history move from +`StructuredProfileSection` into a new `PersonRecordActions`. Reviews keeps its +controllers and changes only its presentation, plus a person picker in Facts. + +**Tech stack:** Svelte 5 (runes), `@kenn-io/kit-ui` (`SelectDropdown`, +`Typeahead`, `Chip`, `Menu`, `IconButton`, `Button`), `@lucide/svelte`, Vitest +with Testing Library, Playwright. + +**Spec:** [PR 3 spec](web-ui-overhaul-pr3-spec.md), which refines the People +sections of the [Web UI overhaul design](web-ui-overhaul-design.md). Read the +spec before each task; it is the authority where this plan is silent. + +## Global constraints + +- Keep every capability. A control that moves keeps its accessible name unless + the spec names the change. +- Keep URL state keys, the `explore` JSON format, and API calls unchanged. The + only state change is dropping an impossible Directory date when the URL is + read (Task 2). +- No new npm dependencies. Icons come from `@lucide/svelte`. +- One solid primary button per screen: `tone="info" surface="solid"`. Purple + (`workflow`) and green (`success`) are not button colors. +- Sentence case for every visible label. Monospace only for identifiers, + hashes, code, keys, and cron text. +- Raw API codes are never the only text a person sees. Unknown codes fall back + to `sentenceCase` from `explore/labels.ts`. +- `kit-ui-check` forbids native date inputs. The two Directory date fields are + the approved exception (spec decision 1 and the design's Directory section); + each carries a `kit-ui-check-ignore` comment with that reason. +- Tests use Vitest and Testing Library or Playwright, query by role and + accessible name, and use failure-safe cleanup (`afterEach` or + `try`/`finally`). Synthetic names and `example.com` addresses only. +- Run commands from `web/` unless a step says otherwise. After each task run + `bun run check`, `bun run check:kit-ui`, and the focused tests. +- Commit after each task with the `kenn-io-commit` rules: new commits only, + conventional imperative subject, a short why body, the attribution trailer + the controller gives you, and a private-data check (public repository). Go + hooks run on commit; export `GOTOOLCHAIN=go1.27.1` if the shell pins an + older toolchain. + +## File structure + +| File | Responsibility | +|---|---| +| `web/src/lib/explore/labels.ts` | Export the existing `sentenceCase` | +| `web/src/lib/directory/labels.ts` (new) | Contact state, channel, and review-state labels and tones; date display | +| `web/src/lib/directory/dates.ts` (new) | `isCalendarDate`, `intervalDateError`, `profileDateError` | +| `web/src/lib/explore/state.svelte.ts` | Drop impossible Directory dates when parsing | +| `web/src/lib/components/directory/DirectoryWorkspace.svelte` | Toolbar, Filters panel, date fields, Sort, chips, count, primary Promote button | +| `web/src/lib/components/directory/DirectoryList.svelte` | Readable row text | +| `web/src/lib/components/directory/PersonDetail.svelte` | Header, seven sections, scrolling tablist | +| `web/src/lib/components/directory/PersonRecordActions.svelte` (new) | More actions menu, rename form, delete confirmation, profile history dialog | +| `web/src/lib/components/directory/StructuredProfileSection.svelte` | Loses rename, delete, and history | +| `web/src/lib/components/directory/PersonAgenda.svelte` | Hide the add form unless the integration is ready | +| `web/src/lib/components/directory/{EmploymentEditor,PersonRelationshipEditor,StructuredProfileEditor}.svelte` | Inline date validation | +| `web/src/lib/components/relationships/{RelationshipList,RelationshipHeader,RelationshipsWorkspace}.svelte` | Placeholder, header layout, counts line, People button | +| `web/src/lib/components/directory/{DirectoryReviewCentre,RelationshipReviewQueue,FactReviewPanel,IdentityCandidateCard,RelationshipReviewCard,DirectoryReviewWorkspace}.svelte` | Facts label, hidden headings with visible focus, Show selects, status chips, person picker | +| `web/src/lib/components/shell/AppShell.svelte` | Pass `openRelationship`, `openFacts`, and `client` down | + +--- + +### Task 1: People labels and date rules + +**Files:** +- Modify: `web/src/lib/explore/labels.ts:99` (export `sentenceCase`) +- Create: `web/src/lib/directory/labels.ts`, `web/src/lib/directory/labels.test.ts` +- Create: `web/src/lib/directory/dates.ts`, `web/src/lib/directory/dates.test.ts` + +**Interfaces:** +- Produces, in `directory/labels.ts`: + - `PRIMARY_CHANNELS: readonly ['email', 'phone', 'chat']` + - `contactStateLabel(code: string): string` + - `channelLabel(code: string): string` + - `reviewStateChip(code: string): { label: string; tone: ChipTone }` + - `formatContactDate(iso: string): string` — "Jan 2, 2002"; returns the + input unchanged when it does not parse. + - `formatDay(day: string): string` — formats a `YYYY-MM-DD` day in UTC, + "Jan 5, 2024". +- Produces, in `directory/dates.ts`: + - `isCalendarDate(value: string): boolean` — `YYYY-MM-DD` that exists. + - `intervalDateError(value: string): string | null` — `null` for empty or + valid. + - `profileDateError(value: string): string | null` + - `isTextProfileDate(value: string): boolean` — non-empty and not a + structured shape, so it will be saved as text. + +- [ ] **Step 1: Write the failing tests** + +`directory/dates.test.ts`: + +```ts +import { describe, expect, it } from 'vitest'; +import { intervalDateError, isCalendarDate, isTextProfileDate, profileDateError } from './dates'; + +describe('isCalendarDate', () => { + it.each([ + ['2024-02-29', true], ['2023-02-29', false], ['2026-02-31', false], + ['2026-13-01', false], ['2026-1-01', false], ['', false], ['last week', false], + ])('%s → %s', (value, want) => expect(isCalendarDate(value)).toBe(want)); +}); + +describe('intervalDateError', () => { + it.each(['', ' ', '2019', '2019-04', '2019-04-12', '20190412', '2024-02-29'])('accepts %j', (value) => { + expect(intervalDateError(value)).toBeNull(); + }); + it.each(['--04-12', '--04', '---12', 'spring 2019', '2019-13', '2019-02-30', '0000', '201904', '2019-4'])( + 'rejects %j', (value) => { + expect(intervalDateError(value)).toBe( + 'Use a year, year and month, or full date, like 2019, 2019-04, or 2019-04-12.'); + }); +}); + +describe('profileDateError', () => { + it.each(['', '2019', '2019-04', '2019-04-12', '--04-12', '--02-29', '--04', '---12', 'spring 2019'])( + 'accepts %j', (value) => expect(profileDateError(value)).toBeNull()); + it.each(['2024-13', '2023-02-29', '--02-30', '--13', '---32', '0000-01'])('rejects %j', (value) => { + expect(profileDateError(value)).toBe('This date does not exist. Check the month and day.'); + }); + it('marks free text as a text date only', () => { + expect(isTextProfileDate('spring 2019')).toBe(true); + expect(isTextProfileDate('--04-12')).toBe(false); + expect(isTextProfileDate('2024-13')).toBe(false); + expect(isTextProfileDate(' ')).toBe(false); + }); +}); +``` + +`directory/labels.test.ts`: + +```ts +import { describe, expect, it } from 'vitest'; +import { channelLabel, contactStateLabel, formatContactDate, formatDay, reviewStateChip } from './labels'; + +describe('directory labels', () => { + it('names known codes and sentence-cases unknown ones', () => { + expect(contactStateLabel('active')).toBe('Active'); + expect(contactStateLabel('needs_follow_up')).toBe('Needs follow up'); + expect(channelLabel('email')).toBe('Email'); + expect(channelLabel('carrier_pigeon')).toBe('Carrier pigeon'); + }); + it.each([ + ['candidate', 'Candidate', 'warning'], ['conflict', 'Conflict', 'warning'], + ['pending', 'Pending', 'warning'], ['accepted', 'Accepted', 'success'], + ['rejected', 'Rejected', 'muted'], ['superseded', 'Superseded', 'neutral'], + ])('review state %s', (code, label, tone) => { + expect(reviewStateChip(code)).toEqual({ label, tone }); + }); + it('formats dates for display', () => { + expect(formatDay('2024-01-05')).toBe('Jan 5, 2024'); + expect(formatContactDate('not a date')).toBe('not a date'); + expect(formatContactDate('2002-01-02T12:00:00Z')).toMatch(/2002/); + }); +}); +``` + +- [ ] **Step 2: Run** `bunx vitest run src/lib/directory/dates.test.ts src/lib/directory/labels.test.ts`. + Expected: FAIL, modules not found. + +- [ ] **Step 3: Implement** + +In `explore/labels.ts` change `function sentenceCase` to `export function sentenceCase`. + +`directory/dates.ts`: + +```ts +// Mirrors PartialDate.Validate (internal/store/partialdate.go): year 1–9999, +// month 1–12, and a day that exists in that month. A day without a year is +// checked against leap year 2000, as the store does. +const YEAR_FORMS = /^(\d{4})(?:-(\d{2})(?:-(\d{2}))?)?$|^(\d{4})(\d{2})(\d{2})$/; +const PROFILE_FORMS = /^(\d{4})(?:-(\d{2})(?:-(\d{2}))?)?$|^--(\d{2})(?:-(\d{2}))?$|^---(\d{2})$/; +const INTERVAL_MESSAGE = 'Use a year, year and month, or full date, like 2019, 2019-04, or 2019-04-12.'; +const RANGE_MESSAGE = 'This date does not exist. Check the month and day.'; + +function validParts(year: number | undefined, month: number | undefined, day: number | undefined): boolean { + if (year !== undefined && (year < 1 || year > 9999)) return false; + if (month !== undefined && (month < 1 || month > 12)) return false; + if (day !== undefined && (day < 1 || day > 31)) return false; + if (month === undefined || day === undefined) return true; + const probeYear = year ?? 2000; + const probe = new Date(Date.UTC(probeYear, month - 1, day)); + return probe.getUTCFullYear() === probeYear && probe.getUTCMonth() === month - 1 && probe.getUTCDate() === day; +} + +const num = (value: string | undefined): number | undefined => (value === undefined ? undefined : Number(value)); + +export function isCalendarDate(value: string): boolean { + const match = /^(\d{4})-(\d{2})-(\d{2})$/.exec(value); + return Boolean(match) && validParts(num(match![1]), num(match![2]), num(match![3])); +} + +export function intervalDateError(value: string): string | null { + const trimmed = value.trim(); + if (!trimmed) return null; + const match = YEAR_FORMS.exec(trimmed); + if (!match) return INTERVAL_MESSAGE; + const [year, month, day] = match[1] ? [match[1], match[2], match[3]] : [match[4], match[5], match[6]]; + return validParts(num(year), num(month), num(day)) ? null : INTERVAL_MESSAGE; +} + +function profileParts(value: string): [number | undefined, number | undefined, number | undefined] | undefined { + const match = PROFILE_FORMS.exec(value); + if (!match) return undefined; + if (match[1]) return [num(match[1]), num(match[2]), num(match[3])]; + if (match[4]) return [undefined, num(match[4]), num(match[5])]; + return [undefined, undefined, num(match[6])]; +} + +export function profileDateError(value: string): string | null { + const parts = profileParts(value.trim()); + if (!parts) return null; + return validParts(...parts) ? null : RANGE_MESSAGE; +} + +export function isTextProfileDate(value: string): boolean { + const trimmed = value.trim(); + return trimmed !== '' && profileParts(trimmed) === undefined; +} +``` + +`directory/labels.ts`: + +```ts +import type { ChipTone } from '@kenn-io/kit-ui'; +import { sentenceCase } from '../explore/labels'; + +export const PRIMARY_CHANNELS = ['email', 'phone', 'chat'] as const; +const CONTACT_STATES: Record = { active: 'Active', inactive: 'Inactive' }; +const CHANNELS: Record = { email: 'Email', phone: 'Phone', chat: 'Chat' }; +// Shared status vocabulary: amber needs attention, green finished, gray off. +const REVIEW_STATES: Record = { + candidate: { label: 'Candidate', tone: 'warning' }, + conflict: { label: 'Conflict', tone: 'warning' }, + pending: { label: 'Pending', tone: 'warning' }, + accepted: { label: 'Accepted', tone: 'success' }, + rejected: { label: 'Rejected', tone: 'muted' }, +}; +const DATE_FORMAT: Intl.DateTimeFormatOptions = { year: 'numeric', month: 'short', day: 'numeric' }; + +export const contactStateLabel = (code: string): string => CONTACT_STATES[code] ?? sentenceCase(code); +export const channelLabel = (code: string): string => CHANNELS[code] ?? sentenceCase(code); +export const reviewStateChip = (code: string): { label: string; tone: ChipTone } => + REVIEW_STATES[code] ?? { label: sentenceCase(code), tone: 'neutral' }; + +export function formatContactDate(iso: string): string { + const date = new Date(iso); + return Number.isNaN(date.valueOf()) ? iso : date.toLocaleDateString('en-US', DATE_FORMAT); +} + +export function formatDay(day: string): string { + return new Date(`${day}T00:00:00Z`).toLocaleDateString('en-US', { ...DATE_FORMAT, timeZone: 'UTC' }); +} +``` + +- [ ] **Step 4: Run the tests again.** Expected: PASS. +- [ ] **Step 5: Commit.** Subject: `feat(web): add People labels and partial-date rules`. + +--- + +### Task 2: Directory toolbar, filters panel, chips, and readable rows + +**Files:** +- Modify: `web/src/lib/explore/state.svelte.ts:463-464` +- Modify: `web/src/lib/components/directory/DirectoryWorkspace.svelte` +- Modify: `web/src/lib/components/directory/DirectoryList.svelte:93-94` +- Test: `web/src/lib/explore/state.test.ts`, + `web/src/lib/components/directory/DirectoryWorkspace.test.ts`, + `web/src/lib/components/directory/DirectoryList.test.ts` (create if absent; + otherwise extend) + +**Interfaces:** +- Consumes: `isCalendarDate`, `contactStateLabel`, `channelLabel`, + `PRIMARY_CHANNELS`, `formatContactDate`, `formatDay` (Task 1). +- Produces: no new exports. `.filters` is renamed `.directory-toolbar`; the + Playwright spec that used `.filters` is fixed in Task 8. + +Target markup and behavior (spec "Directory list"): + +- Toolbar row: `SearchInput` (unchanged), a `Button` "Filters" with + `ariaExpanded`, the Sort `SelectDropdown`, and a count `` at the right + edge with `aria-live="polite"`. +- Filters button: `surface={filtersOpen || activeFilterCount > 0 ? 'soft' : 'outline'}`, + matching `ContextBar.svelte`. +- Filter panel (`{#if filtersOpen}`, `
`): + the Contact state select, Category and Organization text fields, the Primary + channel select with `PRIMARY_CHANNELS.map((value) => ({ value, label: channelLabel(value) }))`, + and two date fields. +- Date field markup (one per boundary): + +```svelte + +``` + + Dates commit through `selectFilter` (no debounce): a native date input + yields a complete value or empty. Remove both keys from `TextFilterKey` and + `controllerTextFilters()`; read the values straight from `controller` instead. + Keep the visible label text above each input as shown. +- Sort: keep `title="Directory order"`; add + `triggerLabel: \`Sort: ${label}\`` to each `sortOptions` entry, as + `ContextBar.svelte` does. +- Chips (`
`), one per active filter, in panel order: + "Contact state: Active", "Category: ", "Organization: ", + "Primary channel: Email", "Last contacted after Jan 5, 2024", + "Last contacted before Jan 5, 2024". Each chip has an `IconButton` named + "Remove filter" that commits only that key as empty through + `selectFilter`. Reuse the `.chip` styling from `ContextBar.svelte` + (copy the three rules; do not import its private CSS). +- Count: `${controller.rows.length.toLocaleString()}${controller.cursor !== null ? '+' : ''} ${controller.rows.length === 1 ? 'person' : 'people'}`; + empty string while `controller.loading && controller.rows.length === 0`. +- Promote button: `tone="info" surface="solid"`. +- `DirectoryList.svelte`: line 93 becomes + `{person.primary_channel ? channelLabel(person.primary_channel) : 'No primary channel'} · {contactStateLabel(person.contact_state)}`; + line 94 becomes `Last contact ${formatContactDate(person.last_contact_at)}`. +- `state.svelte.ts:463-464`: keep a string only when it is empty or + `isCalendarDate(value)`; otherwise `''`. + +- [ ] **Step 1: Write the failing tests** + +`state.test.ts`: + +```ts +it('drops an impossible Directory date when reading the URL', () => { + window.history.replaceState(null, '', `/?explore=${encodeURIComponent(JSON.stringify({ + workspace: 'directory', directoryLastContactAfter: '2026-02-31', directoryLastContactBefore: '2026-03-01' + }))}`); + const state = new ExploreState(window); + try { + expect(state.current.directoryLastContactAfter).toBe(''); + expect(state.current.directoryLastContactBefore).toBe('2026-03-01'); + } finally { + state.destroy(); + } +}); +``` + +`DirectoryWorkspace.test.ts` (adapt the existing render helper and fetch stub +in that file; replace the assertions at :119-131 that typed into the old text +date fields): + +```ts +it('sets and clears each last-contacted boundary on its own', async () => { + const { controller } = renderDirectory(); // existing helper; rename if the file uses another + await fireEvent.click(screen.getByRole('button', { name: 'Filters' })); + const after = screen.getByLabelText('Last contacted after') as HTMLInputElement; + expect(after.type).toBe('date'); + await fireEvent.change(after, { target: { value: '2024-01-05' } }); + expect(controller.lastContactAfter).toBe('2024-01-05'); + expect(controller.lastContactBefore).toBe(''); + expect(screen.getByText('Last contacted after Jan 5, 2024')).toBeTruthy(); + await fireEvent.click(screen.getByRole('button', { name: 'Remove Last contacted after Jan 5, 2024 filter' })); + expect(controller.lastContactAfter).toBe(''); +}); + +it('removes only the chip that was cleared', async () => { + const { controller } = renderDirectory({ directoryContactState: 'active', directoryOrganization: 'Example Co' }); + await fireEvent.click(screen.getByRole('button', { name: 'Remove Contact state: Active filter' })); + expect(controller.contactState).toBe(''); + expect(controller.organization).toBe('Example Co'); +}); + +it('names primary channels and sorts with a visible label', async () => { + renderDirectory(); + expect(screen.getByRole('combobox', { name: /^Directory order:/ }).textContent).toContain('Sort: Name'); + await fireEvent.click(screen.getByRole('button', { name: 'Filters' })); + await fireEvent.click(screen.getByRole('combobox', { name: /^Primary channel/ })); + expect(screen.getByRole('option', { name: 'Email' })).toBeTruthy(); +}); +``` + +Also assert the request: after setting the after-boundary, the latest +`/api/v1/people/directory` request carries +`last_contact_after=2024-01-05T00:00:00Z` and no `last_contact_before`. +Existing tests that read `textbox "Category filter"` / `"Organization filter"` +/ `combobox "Primary channel"` now click "Filters" first. + +`DirectoryList` test: a row with `primary_channel: 'email'`, +`contact_state: 'active'`, `last_contact_at: '2002-01-02T12:00:00Z'` shows +"Email · Active" and a "Last contact" line containing "2002", and no `T12:00`. + +- [ ] **Step 2: Run** `bunx vitest run src/lib/explore/state.test.ts src/lib/components/directory`. + Expected: the new tests FAIL. +- [ ] **Step 3: Implement** the markup and behavior above. +- [ ] **Step 4: Run the tests, `bun run check`, and `bun run check:kit-ui`.** Expected: PASS, no findings. +- [ ] **Step 5: Commit.** Subject: `feat(web): give Directory one toolbar with filter chips`. + +--- + +### Task 3: Person header, sections, and record actions + +**Files:** +- Create: `web/src/lib/components/directory/PersonRecordActions.svelte`, `PersonRecordActions.test.ts` +- Modify: `web/src/lib/components/directory/StructuredProfileSection.svelte` + (remove `historyOpen`, `renaming`, `renameValue`, `confirmingDelete`, + `beginRename`, `saveRename`, `deletePerson`, the header buttons at + :236-257, the rename and delete groups at :261-310, and the + `ProfileHistoryDialog` block at :434-441; keep `reload()` but drop its + `confirmingDelete = false` line) +- Modify: `web/src/lib/components/directory/PersonDetail.svelte` +- Modify: `web/src/lib/components/directory/PersonAgenda.svelte:209-243` +- Modify: `web/src/lib/components/directory/DirectoryWorkspace.svelte` (new props, pass through) +- Modify: `web/src/lib/components/shell/AppShell.svelte` (DirectoryWorkspace mount at ~1379) +- Test: `PersonDetail.test.ts`, `StructuredProfileSection.test.ts`, + `PersonAgenda.test.ts`, `AppShell.test.ts` + +**Interfaces:** +- `PersonRecordActions` props: `{ client: APIClient; controller: DirectoryProfileController; personID: number }`. + Renders a kit `Menu` (`align="end"`) with `MenuTrigger ariaLabel="More actions"` + (`Ellipsis` icon) and `MenuItem`s "Rename person" (disabled when + `!controller.canWritePerson`), "View profile history", and "Delete person" + (disabled when `!controller.canWritePerson`). It also renders the moved + rename group, delete group, and `ProfileHistoryDialog` exactly as + `StructuredProfileSection` did, with the same names, labels, pending text, + and disabled rules. The groups render below the header row (the component + returns a fragment: the menu, then the groups). +- New `PersonDetail` props: `onOpenRelationship?: (participantID: number) => void` + and `onReviewFacts?: (personID: number) => void`. `DirectoryWorkspace` gets + the same two props and passes them to both `PersonDetail` mounts. +- AppShell passes `onOpenRelationship={openRelationship}` and + `onReviewFacts={(personID) => commitNavigation({ workspace: 'directory_review', reviewKind: 'fact', directoryPersonID: personID })}`. + +`PersonDetail` changes: + +- Replace the five tab buttons, five `bind:this` refs, and ten id constants + with one list: + +```ts +type DetailTab = 'overview' | 'profile' | 'organizations' | 'connections' | 'network' | 'media' | 'maintenance'; +const SECTIONS: ReadonlyArray<{ id: DetailTab; label: string }> = [ + { id: 'overview', label: 'Overview' }, + { id: 'profile', label: 'Profile' }, + { id: 'organizations', label: 'Organizations' }, + { id: 'connections', label: 'Connections' }, + { id: 'network', label: 'Network' }, + { id: 'media', label: 'Media & files' }, + { id: 'maintenance', label: 'Maintenance' }, +]; +const tabButtons: Partial> = $state({}); +const tabID = (tab: DetailTab) => `person-${personID}-${tab}-tab`; +const panelID = (tab: DetailTab) => `person-${personID}-${tab}-panel`; +``` + + Render with `{#each SECTIONS as section (section.id)}` and + `bind:this={tabButtons[section.id]}`. `handleTabKeydown` uses + `SECTIONS.map((s) => s.id)` for order and `'maintenance'` for End. + `selectTab` focuses `tabButtons[tab]`. +- Header above the tablist: + +```svelte +
+

{displayName}

+
+ {#if participantID !== undefined && onOpenRelationship} +
+
+``` + + with `displayName = bundle.person?.display_name ?? profile?.person?.display_name ?? \`Person ${personID}\`` + and `participantID = bundle.person?.participant_ids.length ? Math.min(...bundle.person.participant_ids) : undefined`. + The `h2` leaves the Overview panel. +- Panels, in the spec's order: + - Overview: `PersonBriefCard`, `PersonAgenda`, `AttributeSummary`, Contact + state, Activity, the organization and relationship summaries (the + "Relationships" summary heading becomes "Connections"), `MeetingPanel`. + Contact state shows + `{contactStateLabel(cadence_status)} · N interactions · last contact {formatContactDate(...)}`. + - Profile: `StructuredProfileSection` or the no-controller fallback + sections, then `AttributeSection`. + - Organizations, Connections (`RelationshipsTab`), Network, Media & files: + unchanged content. + - Maintenance: `PersonTrackingControl`, `CardDAVPublicationControl`, + `PersonMergeHistory`. +- `AttributeSummary` `onEdit`: `await selectTab('profile'); await tick();` + then scroll and focus `#person-attributes` as today. +- Tablist CSS: `overflow-x: auto; flex-wrap: nowrap;` and + `[role="tab"] { white-space: nowrap; flex: none; }`. + +`PersonAgenda`: wrap the `
` at :239-243 in `{#if mutationReady}`; give +the integration status paragraph `role="status"` instead of `role="alert"`. + +- [ ] **Step 1: Write the failing tests** + +`PersonDetail.test.ts` (replace the five-tab assertions at :258-284 and :473): + +```ts +it('lists seven person sections in order', async () => { + renderPersonDetail(); + expect(screen.getAllByRole('tab').map((tab) => tab.textContent?.trim())).toEqual( + ['Overview', 'Profile', 'Organizations', 'Connections', 'Network', 'Media & files', 'Maintenance']); + await fireEvent.click(screen.getByRole('tab', { name: 'Maintenance' })); + expect(screen.getByRole('heading', { name: 'Merge history' })).toBeTruthy(); +}); + +it('opens the relationship for the lowest participant ID and reviews facts', async () => { + const onOpenRelationship = vi.fn(); + const onReviewFacts = vi.fn(); + renderPersonDetail({ person: { ...person, participant_ids: [42, 7, 19] } }, { onOpenRelationship, onReviewFacts }); + await fireEvent.click(screen.getByRole('button', { name: 'Open relationship' })); + expect(onOpenRelationship).toHaveBeenCalledWith(7); + await fireEvent.click(screen.getByRole('button', { name: 'Review facts' })); + expect(onReviewFacts).toHaveBeenCalledWith(personID); +}); + +it('hides Open relationship when the person has no participants', () => { + renderPersonDetail({ person: { ...person, participant_ids: [] } }, { onOpenRelationship: vi.fn() }); + expect(screen.queryByRole('button', { name: 'Open relationship' })).toBeNull(); +}); +``` + +Use the file's existing bundle fixture and render helper names. The existing +"Contact state before Last time we talked" assertion (:467-469) is reversed: +the brief now comes first. + +`PersonRecordActions.test.ts`: move the rename, delete, and history tests +from `StructuredProfileSection.test.ts:256-307, 377`, opening each flow with +`fireEvent.click(screen.getByRole('button', { name: 'More actions' }))` then +the `menuitem` of the same name. Keep every assertion on group names, +buttons, and controller calls. Add: both write items are disabled when +`canWritePerson` is false. + +`PersonAgenda.test.ts`: with integration state `disabled`, the "New agenda +item" field is absent and the status line is present. + +`AppShell.test.ts`: from a Directory person, "Review facts" leaves +`state.current` with `workspace: 'directory_review'`, `reviewKind: 'fact'`, +and the person ID; "Open relationship" sets `relationshipTarget` to +`cluster:`. + +- [ ] **Step 2: Run** `bunx vitest run src/lib/components/directory src/lib/components/shell/AppShell.test.ts`. + Expected: new tests FAIL. +- [ ] **Step 3: Implement.** +- [ ] **Step 4: Run tests and checks.** Expected: PASS. +- [ ] **Step 5: Commit.** Subject: `feat(web): split the person page into sections with a header`. + +--- + +### Task 4: Partial-date validation in the person editors + +**Files:** +- Modify: `web/src/lib/components/directory/EmploymentEditor.svelte:182,189-190,207` +- Modify: `web/src/lib/components/directory/PersonRelationshipEditor.svelte:294-311,335` +- Modify: `web/src/lib/components/directory/StructuredProfileEditor.svelte:436-445` and its save button +- Test: `OrganizationEmploymentTab.test.ts` or `EmploymentEditor.test.ts`, + `RelationshipsTab.test.ts` or `PersonRelationshipEditor.test.ts`, + `StructuredProfileEditor.test.ts` (use whichever file already renders the + editor) + +**Interfaces:** +- Consumes: `intervalDateError`, `profileDateError`, `isTextProfileDate` (Task 1). + +Pattern for each interval field (employment start and end, relationship start +and end): + +```svelte + + +{#if startDateError}

{startDateError}

{/if} +``` + +Kit `TextInput` takes `ariaDescribedby`. Make each error id unique per editor +instance with `$props.id()` (for example `${uid}-start-date-error`). + +- Add `|| Boolean(startDateError) || Boolean(endDateError)` to each editor's + submit `disabled` expression. +- Profile "Date" field: `dateError = $derived(profileDateError(dateValue))`; + show the error the same way; show `

Saved as text

` + when `isTextProfileDate(dateValue)`. Disable the profile editor's save + button while `section === 'dates' && dateError`. +- `.field-error { color: var(--text-danger); font-size: var(--font-size-xs); margin: 0; }` + and `.field-hint { color: var(--text-muted); font-size: var(--font-size-xs); margin: 0; }`. + +- [ ] **Step 1: Write the failing tests** + +```ts +it('rejects a year-less employment date and keeps Save disabled', async () => { + renderEmploymentEditor(); // existing helper + await fireEvent.input(screen.getByRole('textbox', { name: 'Employment start date' }), { target: { value: '--04-12' } }); + expect(screen.getByText('Use a year, year and month, or full date, like 2019, 2019-04, or 2019-04-12.')).toBeTruthy(); + expect((screen.getByRole('button', { name: /Create employment|Save employment/ }) as HTMLButtonElement).disabled).toBe(true); + await fireEvent.input(screen.getByRole('textbox', { name: 'Employment start date' }), { target: { value: '20190412' } }); + expect(screen.queryByText(/Use a year/)).toBeNull(); +}); +``` + +Same shape for "Relationship start date" (`2019-02-30` rejected, `2019` +accepted). Profile: + +```ts +it.each(['--04-12', '--04', '---12'])('keeps the profile date form %s', async (value) => { + renderDateEditor(); + await fireEvent.input(screen.getByRole('textbox', { name: 'Date' }), { target: { value } }); + expect(screen.queryByText(/does not exist/)).toBeNull(); + expect(screen.queryByText('Saved as text')).toBeNull(); +}); +it('rejects an impossible profile date and marks free text', async () => { + renderDateEditor(); + const field = screen.getByRole('textbox', { name: 'Date' }); + await fireEvent.input(field, { target: { value: '--02-30' } }); + expect(screen.getByText('This date does not exist. Check the month and day.')).toBeTruthy(); + await fireEvent.input(field, { target: { value: 'spring 2019' } }); + expect(screen.getByText('Saved as text')).toBeTruthy(); +}); +``` + +Also assert a text date still submits `date_text: 'spring 2019'` with no +`date` field, so the existing text-date capability is covered. + +- [ ] **Step 2: Run** the three test files. Expected: FAIL. +- [ ] **Step 3: Implement.** +- [ ] **Step 4: Run tests and checks.** Expected: PASS. +- [ ] **Step 5: Commit.** Subject: `feat(web): validate partial dates in person editors`. + +--- + +### Task 5: Relationships header and labels + +**Files:** +- Modify: `web/src/lib/components/relationships/RelationshipList.svelte:168` +- Modify: `web/src/lib/components/relationships/RelationshipsWorkspace.svelte:494-506` +- Modify: `web/src/lib/components/relationships/RelationshipHeader.svelte:302-360` and styles near :515-545 +- Test: `RelationshipList.test.ts`, `RelationshipsWorkspace.test.ts`, `RelationshipHeader.test.ts` + +Changes: + +- Placeholder: `"Filter people and domains"`. +- Drawer toggle: `label="People"`, `ariaLabel` removed (the name becomes + "People"); drawer `title="People"`; drawer `ariaLabel` unchanged. +- Header: the title row keeps avatar, `h2`, and `.actions` with only + "Open in Directory" and "Same person…" (wrap them in + `
`; render the + group only for people). Move the `SegmentedControl ariaLabel="Relationship view"` + out of `.actions` into its own `
` after the title row + and any stale banner, left-aligned. Options and handler unchanged. +- Counts line: drop the files segment and `data-mono`: + `{activity_count} items · {formatDate(first_at)} – {formatDate(last_at)}`, + plus `· N people` for domains. Change the local `formatDate` to + `date.toLocaleDateString('en-US', { year: 'numeric', month: 'short', day: 'numeric' })`. +- Identity chip names already use the display name when known (`:376-378`, + `:425`); leave them. + +- [ ] **Step 1: Write the failing tests** + +```ts +// RelationshipList.test.ts +expect(screen.getByRole('searchbox', { name: 'Search people and domains' }).getAttribute('placeholder')) + .toBe('Filter people and domains'); + +// RelationshipsWorkspace.test.ts (replace "Show relationship list" at :617, :665) +await fireEvent.click(screen.getByRole('button', { name: 'People' })); +expect(screen.getByRole('dialog', { name: 'Relationship search and results' })).toBeTruthy(); + +// RelationshipHeader.test.ts +const actions = screen.getByRole('group', { name: 'Person actions' }); +expect(within(actions).getByRole('button', { name: 'Open in Directory' })).toBeTruthy(); +expect(within(actions).queryByRole('radio', { name: 'Messages' })).toBeNull(); +expect(screen.getByText(/items · /).textContent).not.toMatch(/files/); +``` + +- [ ] **Step 2: Run** `bunx vitest run src/lib/components/relationships`. Expected: FAIL. +- [ ] **Step 3: Implement.** +- [ ] **Step 4: Run tests and checks.** Expected: PASS. +- [ ] **Step 5: Commit.** Subject: `feat(web): separate Relationships actions from its view switch`. + +--- + +### Task 6: Reviews structure, Show selects, and status chips + +**Files:** +- Modify: `web/src/lib/components/directory/DirectoryReviewCentre.svelte` +- Modify: `web/src/lib/components/directory/RelationshipReviewQueue.svelte` +- Modify: `web/src/lib/components/directory/FactReviewPanel.svelte:17` (heading text only; Task 7 does the rest) +- Modify: `web/src/lib/components/directory/IdentityCandidateCard.svelte:22` +- Modify: `web/src/lib/components/directory/RelationshipReviewCard.svelte:26,34` +- Test: `DirectoryReviewCentre.test.ts`, `RelationshipReviewQueue.test.ts`, + `FactReviewPanel.test.ts`, `IdentityCandidateCard.test.ts`, + `RelationshipReviewCard.test.ts`, `AppShell.test.ts:1246-1251` + +**Interfaces:** +- Consumes: `reviewStateChip` (Task 1). +- Produces: CSS class `review-heading` on each review `h2`, and + `data-review-section` on each review section, used by Task 7's Facts panel. + +Changes: + +- `reviewKindOptions`: `{ value: 'fact', label: 'Facts' }`. +- Each review `h2` (`identity-review-heading`, `relationship-review-heading`, + `fact-review-heading`) keeps its id and `tabindex="-1"`, gets + `class="review-heading"`, and its text becomes the option label + ("Identity matches", "Imported relationships", "Facts"). Delete the + description paragraphs under the identity and imported headings. Keep the + imported read-only sentence as a one-line `

` in the + queue toolbar. +- Hidden heading with visible focus, in each component's styles: + +```css +.review-heading { + position: absolute; width: 1px; height: 1px; margin: -1px; padding: 0; + overflow: hidden; clip: rect(0 0 0 0); white-space: nowrap; border: 0; +} +[data-review-section] { position: relative; border-radius: var(--radius-md); } +[data-review-section]:has(> .review-heading:focus-visible), +[data-review-section]:has(> * > .review-heading:focus-visible) { + outline: 2px solid var(--focus-color); outline-offset: 4px; +} +``` + + `--focus-color` is the focus token in `src/styles/tokens.css`. Add `data-review-section` to the + identity `

`, the imported `
`, and the Facts `
`. +- Show selects replace both `SegmentedControl`s: + +```svelte + ({ ...option, triggerLabel: `Show: ${option.label}` }))} + onchange={selectIdentityState} + disabled={!!activeDecision} +/> +``` + + and the same for `title="Imported relationship review state"` with + `disabled={controller.loading}`. Place each at the start of its toolbar row. + Delete the now-unused `.kit-segmented` mobile rules. +- Cards: replace `

{candidate.state}

` and + `

{review.status}

` with + `{chip.label}` + where `chip = reviewStateChip(...)`. Delete the `Status` row from the + imported card's metadata list so the state shows once. +- The imported empty state uses `reviewStateChip(controller.state).label` + instead of the local `stateLabel` (delete `stateLabel`). + +- [ ] **Step 1: Write the failing tests** + +```ts +// DirectoryReviewCentre.test.ts (update :116-118, :153-162, :249-253) +expect(screen.getByRole('radio', { name: 'Facts' })).toBeTruthy(); +const show = screen.getByRole('combobox', { name: /^Identity review state/ }); +expect(show.textContent).toContain('Show: Candidate'); +await fireEvent.click(show); +await fireEvent.click(screen.getByRole('option', { name: 'Conflict' })); +expect(commit).toHaveBeenCalledWith(expect.objectContaining({ identityState: 'conflict' })); + +it('shows a focus ring on the section when its hidden heading takes keyboard focus', async () => { + renderCentre(); + const heading = screen.getByRole('heading', { name: 'Identity matches', level: 2 }); + expect(heading.className).toContain('review-heading'); + expect(heading.closest('[data-review-section]')).not.toBeNull(); +}); +``` + +jsdom cannot evaluate `:focus-visible`, so the unit test asserts the hook +(class and section attribute); Task 8 asserts the visible ring in a browser. + +```ts +// IdentityCandidateCard.test.ts +expect(screen.getByText('Candidate')).toBeTruthy(); +expect(screen.queryByText('candidate')).toBeNull(); +// RelationshipReviewCard.test.ts +expect(screen.getAllByText('Pending')).toHaveLength(1); +``` + +Update `RelationshipReviewQueue.test.ts:105` (radio "Pending" → combobox +`/^Imported relationship review state/` showing "Show: Pending") and +`AppShell.test.ts:1251` (radio "Accepted" → choose the "Accepted" option). + +- [ ] **Step 2: Run** `bunx vitest run src/lib/components/directory src/lib/components/shell/AppShell.test.ts`. Expected: FAIL. +- [ ] **Step 3: Implement.** +- [ ] **Step 4: Run tests and checks.** Expected: PASS. +- [ ] **Step 5: Commit.** Subject: `feat(web): give Reviews one heading and Show filters`. + +--- + +### Task 7: Facts person picker and person name + +**Files:** +- Modify: `web/src/lib/components/directory/FactReviewPanel.svelte` +- Modify: `web/src/lib/components/directory/DirectoryReviewCentre.svelte`, + `DirectoryReviewWorkspace.svelte` (new props, pass through) +- Modify: `web/src/lib/components/shell/AppShell.svelte` (~1400 mount) +- Test: `FactReviewPanel.test.ts`, `AppShell.test.ts` + +**Interfaces:** +- New props down the chain `AppShell → DirectoryReviewWorkspace → + DirectoryReviewCentre → FactReviewPanel`: `client: APIClient` and + `onSelectFactPerson: (personID: number) => void`. AppShell passes + `onSelectFactPerson={(personID) => commitNavigation({ directoryPersonID: personID })}`. +- `FactReviewPanel` drops `onOpenDirectory` (and the chain drops it too, with + AppShell's `onOpenDirectory` prop on this mount). + +`FactReviewPanel` behavior: + +- Always render a "Person" field above the ledger: kit `Typeahead` with + `title="Fact person"`, `placeholder="Search Directory people"`, + `fallbackLabel={personName ?? 'Choose a person'}`, `remote`, + `loading={searching}`, `loadingLabel="Searching…"`, + `emptyLabel="No matching people"`, `error={searchError}`, + `onquery={debouncedSearch}`, and `onselect={(value) => onSelectFactPerson(Number(value))}`. + Options are `people.map((person) => ({ value: String(person.id), label: person.display_name ?? \`Person ${person.id}\` }))`. +- Search copies `PersonRelationshipEditor.searchPeople` (`:88-121`): + `generatedListDirectoryPeople({ q, limit: 20 }, { ...client, signal })`, an + `AbortController` per query, a generation counter, a 250 ms `debounce` + imported the way that editor imports it, and abort plus `cancel()` in + `onDestroy`. Do not filter out the current person. +- With no person: the field and the sentence "Choose a person to see the facts + recorded about them." replace the EmptyState and "Open Directory". +- With a person: the context row shows `{personName ?? \`Person ${personID}\`}` + and "Open person profile". `personName` is set from the chosen option on + select; when `personID` changes and no name is known, load it with + `generatedGetPersonProfile({ id: personID }, { ...client, signal })` and use + `data.display_name`. Ignore a stale response with the same generation + pattern. +- Keep the two unavailable-feature notices and `FactLedger`. + +- [ ] **Step 1: Write the failing tests** + +```ts +it('picks a person for Facts from the Directory search', async () => { + const fetchFn = vi.fn(async (input) => { + const url = new URL(input instanceof Request ? input.url : String(input), 'http://localhost'); + if (url.pathname === '/api/v1/people/directory') return Response.json({ + people: [{ id: 12, display_name: 'Alex Example', categories: [], organizations: [], contact_state: 'active', revision: 1 }], + }); + return Response.json({}); + }); + const onSelectFactPerson = vi.fn(); + renderFactPanel({ client: createAPIClient(fetchFn), personID: null, onSelectFactPerson }); + expect(screen.queryByRole('button', { name: 'Open Directory' })).toBeNull(); + await fireEvent.input(screen.getByRole('combobox', { name: /Fact person/ }), { target: { value: 'Alex' } }); + await fireEvent.click(await screen.findByRole('option', { name: 'Alex Example' })); + expect(onSelectFactPerson).toHaveBeenCalledWith(12); +}); + +it('shows the selected person by name after a reload', async () => { + const fetchFn = vi.fn(async () => Response.json({ id: 12, display_name: 'Alex Example', participant_ids: [] })); + renderFactPanel({ client: createAPIClient(fetchFn), personID: 12 }); + expect(await screen.findByText('Alex Example')).toBeTruthy(); + expect(screen.queryByText('Person ID 12')).toBeNull(); +}); +``` + +Inspect kit `Typeahead`'s rendered roles first (`node_modules/@kenn-io/kit-ui/src/lib/components/Typeahead.svelte` +and its tests) and use the role and name it actually exposes; the +`PersonRelationshipEditor` tests show a working query. Use fake timers or +`waitFor` for the debounce. + +`AppShell.test.ts`: choosing a person in Facts sets +`state.current.directoryPersonID` and keeps `reviewKind: 'fact'`. + +- [ ] **Step 2: Run** `bunx vitest run src/lib/components/directory/FactReviewPanel.test.ts src/lib/components/shell/AppShell.test.ts`. Expected: FAIL. +- [ ] **Step 3: Implement.** +- [ ] **Step 4: Run tests and checks.** Expected: PASS. +- [ ] **Step 5: Commit.** Subject: `feat(web): choose a person from Facts`. + +--- + +### Task 8: Browser tests + +**Files:** +- Modify: `web/tests/e2e/directory.spec.ts` (`.filters` at :86, tabs at + :51-57 and :144, merge history region at :81 and maintenance switch at + :156-207 now under Maintenance), `web/tests/directory-review.spec.ts` + (:99-117 and :395-440 radios → Show selects; :301-308 "Fact review" → + "Facts"), `web/tests/e2e/accessibility.spec.ts` (region "Fact review" → + "Facts" at :248, :297, :315; tab "Network" at :187; maintenance switch at + :204), `web/tests/directory-network.spec.ts` (:44-58 tab order), and any + spec Step 1 finds + +- [ ] **Step 1: Find affected specs** + +```bash +rg -ln "Fact review|Identity review state|Imported relationship review state|Show relationship list|Media & Files|'Relationships' \}|\.filters|Last contacted (after|before)|Profile maintenance|Person merge history|Rename person|Delete person|View profile history" tests +``` + +- [ ] **Step 2: Update each spec** to the new names and flows without + weakening assertions. Add browser coverage for: + - Directory: set only "Last contacted after" with the native date field; the + chip appears; reload keeps it; removing the chip clears it. + - Person page: "Review facts" lands on Facts with the person's name; "Open + relationship" lands on the person's relationship. + - Reviews keyboard focus: change the imported Show select with the keyboard + so focus moves to the hidden heading, then assert the section's computed + `outline-style` is not `none`. + - Phone width (420px): the person tablist does not wrap (every tab's + `offsetTop` is equal). + - axe on Directory with the Filters panel open, a person page on each + section, and Facts with the picker, in both themes. +- [ ] **Step 3: Run** `make web-test-browser` from the repository root. + Expected: all pass. +- [ ] **Step 4: Commit.** Subject: `test(web): cover the People workspaces in the browser`. + List every existing spec edit and why in the body. + +--- + +### Task 9: Verify and open the pull request + +- [ ] **Step 1:** From the repository root run + `make web-check && make web-test && make web-test-browser && make lint-ci`. +- [ ] **Step 2:** Rebuild the fixture daemon and capture before and after + screenshots at 1440×900 and 420×860: Relationships with a person, + Directory with the Filters panel open and a chip, a person on Overview, + Profile, and Maintenance, and Reviews on each type including Facts with the + picker. View every image. Check: no purple button, no raw codes in rows or + chips, tabs on one line at 420px, one heading per review type. +- [ ] **Step 3:** Read `git diff origin/main...HEAD` fully; remove leftovers. +- [ ] **Step 4:** The user has authorized the pull request. Push the branch and + open it with the `kenn-io-commit-push-pr` and `kenn-io-pr-desc` rules, + attaching the cleared screenshots. + +## Self-review notes + +- Spec coverage: Relationships (Task 5); Directory toolbar, dates, chips, + count, rows, Promote (Task 2); person header, sections, menu, agenda, + narrow tablist, attributes Edit (Task 3); partial dates for intervals and + profiles (Task 4); Reviews label, headings, focus ring, Show selects, chips + (Task 6); Facts picker and name (Task 7); tests and screenshots (Tasks 8, 9). +- The spec's identity-chip naming item is already true in the code + (`RelationshipHeader.svelte:376-378, 425` use the display name when known), + so Task 5 leaves it. +- Task 2 renames `.filters`; Task 8 fixes the one Playwright locator that used + it. Tasks 3 and 6 both touch `AppShell.svelte` mounts but different blocks. diff --git a/docs/internal/web-ui-overhaul-pr3-spec.md b/docs/internal/web-ui-overhaul-pr3-spec.md new file mode 100644 index 000000000..345ebd6f8 --- /dev/null +++ b/docs/internal/web-ui-overhaul-pr3-spec.md @@ -0,0 +1,319 @@ +# Web UI overhaul, PR 3: People + +Status: approved 2026-09-30 after review; implemented and merged in #1019. This spec +refines the People sections of the [Web UI overhaul design](web-ui-overhaul-design.md) +(Relationships, Directory, Reviews) against the code on `main` at ef66efc1. The +design owns the shared rules (palette, page structure, toolbars, code labels, +delivery). This spec owns the exact PR 3 behavior, the decisions the design left +open, and the test changes. Where the two disagree, this spec records the +deviation under [Decisions for review](#decisions-for-review). + +## Outcome + +After PR 3, the three People workspaces follow the pattern Everything and Files +already use: a page header, one toolbar row, removable chips for active +filters, readable labels instead of codes, and one primary action. A person's +Directory page is split into sections that each answer one question, and the +three People workspaces link to each other. + +No capability is removed. Every control keeps its accessible name unless this +spec names the change. + +## What people see today + +Observed with the Enron docs fixture at 1440×900 and 420×860, light theme, +after promoting one participant to a Directory person. + +- **Directory toolbar** is eight inline controls that wrap to two rows at + 1440px. The date fields are free text with a truncated `YYYY-MM-DD` + placeholder. A value that is not shaped like `YYYY-MM-DD`, such as `last + week`, is silently dropped from the query but stays in the field and the URL + (`directory/controller.svelte.ts:576-580`). A well-shaped but impossible + value, such as `2026-02-31`, passes that check and reaches the backend's date + validation. +- **Directory list rows** show raw codes and ISO timestamps, such as + "No primary channel · inactive" and "Last contact" followed by an ISO + timestamp (`DirectoryList.svelte:93-94`). The Primary channel menu lists + `email`, `phone`, `chat`. +- **"Promote to person"** is a purple `workflow` button, the last purple button + outside Manage. +- **Person Overview** stacks 14 sections in one scroll: agenda, profile + maintenance with every eligible field definition, CardDAV publication, the + structured profile, attributes, summaries, contact state, meetings, the "Last + time we talked" brief, activity, and merge history. The brief, which is the + reason to open a person, sits near the bottom. +- **Rename, profile history, and delete** sit inside the Structured profile + section, far from the person's name. +- **At 420px** the five person tabs wrap to three lines ("Media / & / Files"). +- **No cross-links:** Directory cannot open the person's Relationships view or + their facts. The only way to see a person's facts is to type a URL. +- **Relationships** labels its narrow-screen list button "Contacts", while its + accessible name is "Show relationship list". The header shows the file count + twice ("Files 0" in the switch and "0 files" in the counts line). +- **Reviews** repeats the selected review type as a second heading with its own + segmented control ("Identity matches" and Candidate | Conflict | Accepted | + Rejected). The view option reads "Fact review" while the design says + "Facts". Facts with no person selected is a dead end with an "Open Directory" + button. Review cards show raw states (`candidate`, `pending`) and "Person ID + 42" instead of a name. + +## Relationships + +- **List search.** The placeholder becomes "Filter people and domains". The + accessible name stays "Search people and domains". +- **Narrow-screen list button.** The visible label and the drawer title become + "People". The button's accessible name becomes "People" too, so the visible + label is part of the name. The drawer keeps its accessible name + "Relationship search and results". +- **Person header.** The name and avatar sit on the left with "Open in + Directory" and "Same person…" as one action group on the right. The Messages + | Files switch moves to its own row under the header, left-aligned, like the + view row in `PageHeader`. It keeps the radio names "Messages" and "Files N". +- **Counts line.** Drops the file count, which the switch already shows: + "20 items · Apr 25, 2001 – Jan 2, 2002". Dates use the shared date format. +- Identity chips name the profile by display name, not "profile 123", when the + name is known. + +Out of scope: the meeting panel (shared with Directory, see +[Not in PR 3](#not-in-pr-3)), the "All senders" toggle, and the degraded-state +`msgvault build-cache` hint, which is actionable as written. + +## Directory list + +### Toolbar + +One row: search, **Filters**, **Sort**, and the result count at the right edge. + +- **Search** keeps its accessible name "Search directory", its placeholder, the + 250 ms debounce, and the `directoryQuery` key. +- **Filters** is a button that opens a panel under the toolbar, the same + pattern as Everything's Filters. The panel holds: + + | Control | Accessible name | URL key | Change | + |---|---|---|---| + | Contact state select | "Contact state" | `directoryContactState` | None | + | Category text field | "Category filter" | `directoryCategory` | None | + | Organization text field | "Organization filter" | `directoryOrganization` | None | + | Primary channel select | "Primary channel" | `directoryPrimaryChannel` | Options read Email, Phone, Chat | + | Last contacted after | "Last contacted after" | `directoryLastContactAfter` | Native `` with a clear button | + | Last contacted before | "Last contacted before" | `directoryLastContactBefore` | Native `` with a clear button | + + The API parameters are unchanged. The Filters button shows the soft style + while the panel is open or any filter is set, like Everything. +- **Dates.** Each boundary is independent, as today. The native input only + yields a valid `YYYY-MM-DD` or an empty value, so the silent-drop path goes + away for typed input. A URL that carries an invalid value (for example + `2026-02-31`) is dropped when the URL is read, so the field, the chip, and + the request agree. +- **Sort** is a select whose visible label reads "Sort: Name", "Sort: Most + recently contacted", or "Sort: Least recently contacted". It keeps the + accessible name that starts with "Directory order" and the `directorySort` + key. +- **Chips.** A second line shows each active filter as a removable chip, for + example "Contact state: Active", "Organization: Example Co", "Last contacted + after Jan 5, 2024". Removing a chip clears only that filter. A one-sided date + filter is one chip. +- **Count** reads "N people", or "N+ people" when more pages exist. + +### Rows and header + +- Rows show "Email · Active" and "Last contact Jan 2, 2002" instead of codes + and ISO timestamps. One label map covers contact state and channel; unknown + codes fall back to a sentence-cased form. +- "Promote to person" uses the standard primary button instead of the purple + `workflow` tone. It stays the only header action and appears only while a + participant is waiting to be promoted. + +## Person detail + +### Header + +The person's name moves out of Overview into a header above the section tabs, +with actions on the right: + +- **Open relationship** opens the person in Relationships through AppShell's + `openRelationship(participantID)`. It appears only when the person has at + least one participant ID. See [decision 2](#decisions-for-review) for which + ID it uses. +- **Review facts** opens Reviews → Facts for this person + (`workspace: 'directory_review'`, `reviewKind: 'fact'`, + `directoryPersonID`). +- **More actions** is a menu with "Rename person", "View profile history", and + "Delete person". It appears only when the structured profile controller is + available, as the buttons do today. "Rename person" and "Delete person" open + the existing inline rename form and delete confirmation directly under the + header, keeping their group names ("Rename person", "Confirm deleting + person"), buttons, disabled rules, and pending labels. "View profile history" + opens the existing Profile history dialog. + +### Sections + +The tablist keeps the name "Person detail sections" and the existing keyboard +pattern. Tab ids keep the `person--
-tab` form. + +| Section | Contents, in order | Source today | +|---|---|---| +| Overview | "Last time we talked" brief, agenda, attributes summary, contact state, activity, organization and connection summaries, meeting activity | Overview | +| Profile | Structured profile (names, contact points, addresses, dates, categories, media metadata), attributes | Overview | +| Organizations | Unchanged | Organizations tab | +| Connections | Unchanged content; the tab was "Relationships" | Relationships tab | +| Network | Unchanged | Network tab | +| Media & files | Unchanged content; the tab was "Media & Files" | Media & Files tab | +| Maintenance | Profile maintenance tracking, CardDAV publication, merge history and split | Overview | + +- The attributes summary's Edit button switches to Profile and scrolls to the + attributes section, which it does within Overview today. +- The Connections panel keeps its content; only the tab label and ids change + (`person--connections-tab`). +- **Agenda.** When the task integration is disabled or unavailable, the agenda + shows its status line and hides the "New agenda item" form, instead of + showing a disabled form under an error-like sentence. +- **Narrow screens.** The tablist scrolls horizontally instead of wrapping, so + every tab stays one line at 420px. + +Media metadata editing stays in Profile with the rest of the structured +profile. It is separate from the Media & files section, which lists archived +attachments. + +### Partial dates + +Date fields in the person editors stay text fields because a date picker cannot +express `YYYY` or `YYYY-MM`. They get inline validation that mirrors what the +store accepts, which differs between profile dates and interval dates. The +range rules come from `PartialDate.Validate` (`internal/store/partialdate.go`): +year 1–9999, month 1–12, and a day that exists in that month, checked against +leap year 2000 when no year is given. + +- **Interval dates** ("Employment start date", "Employment end date", + "Relationship start date", "Relationship end date") require a year, as + `validateEmploymentDate` and `ParseRelationshipDate` do. They accept `YYYY`, + `YYYY-MM`, `YYYY-MM-DD`, and the compact `YYYYMMDD`. Year-less forms such as + `--04-12` and any other text are invalid. An invalid value shows a message + under the field, such as "Use a year, year and month, or full date, like + 2019, 2019-04, or 2019-04-12", and disables that editor's save button. +- **Profile dates** (the "Date" field in the structured profile editor) keep + every form the editor accepts today. Values that `dateParts` recognizes as a + structured date (`YYYY`, `YYYY-MM`, `YYYY-MM-DD`, `--MM-DD`, `--MM`, and + `---DD`) must pass the range rules above; `2024-13` or `--02-30` shows a + message and disables Save. Any other text is still saved as a text date + (`date_text`), which the store accepts. The field shows the hint "Saved as + text" for such a value, so people can tell it will not sort or compare as a + date. + +## Reviews + +- **View switch.** The "Review type" radiogroup under the page header keeps its + name, the `reviewKind` key, and its disabled-while-deciding rule. The options + read "Identity matches", "Facts", and "Imported relationships". +- **One heading.** The second visible heading and description for each review + type are removed. Each type keeps an `h2` with the same text as a + visually-hidden heading, so screen readers still announce the section and the + existing focus targets after a change keep working. Focus on that heading + must stay visible: while the hidden heading has keyboard focus + (`:focus-visible`), its queue section shows the standard focus ring, so + keyboard users see where focus landed without the layout shifting. This + covers every place that focuses these headings, including the imported + relationship queue after a context change + (`RelationshipReviewQueue.svelte:35`). +- **Show menu.** Each queue's state filter moves from a segmented control to a + select at the start of the queue's toolbar, with visible labels "Show: + Candidate" and "Show: Pending". Like Directory Sort, the select's accessible + name starts with the old radiogroup name, for example "Identity review state: + Show: Candidate" and "Imported relationship review state: Show: Pending". + The keys `identityState` and `relationshipReviewState` are unchanged. + Choosing a state still invalidates an open decision, as today. +- **Cards.** States show as status chips with readable labels (Candidate, + Conflict, Accepted, Rejected, Pending) using the shared status tones: amber + for Candidate, Conflict, and Pending, green for Accepted, gray for Rejected. + The imported relationship card shows its state once. Diagnostic fields + (evidence IDs, basis, service, scope) stay as they are. +- **Facts person picker.** Facts always shows a "Person" picker above the + ledger. It searches the Directory with `listDirectoryPeople({ q, limit: 20 })` + through kit `Typeahead`, the same way `PersonRelationshipEditor` finds + people. Choosing a person commits `directoryPersonID`. With no person + selected, the picker and a one-line explanation replace the "Open Directory" + dead end. With a person selected, the ledger header shows the person's + display name with "Open person profile", instead of "Person ID 42". The name + comes from the picker selection or, after a reload, from the person endpoint + Directory already uses. + +## Decisions for review + +1. **Filters panel, not a popover.** The design says the Directory filters live + in a popover. This spec uses the disclosure panel Everything's Filters + button already opens (`ContextBar.svelte`), so both workspaces behave the + same and native date inputs are not nested inside a floating layer. +2. **Which participant "Open relationship" uses.** `relationshipTarget` is + `cluster:`, and the Relationships controller resolves any + member ID to its cluster (`relationships/controller.svelte.ts:363, 704`). + The button passes the lowest of the person's `participant_ids`, so the + choice is stable. If a Directory person's participants belong to more than + one relationship cluster, the button opens the cluster that contains the + lowest ID. Showing one button per cluster needs data the person response + does not carry. +3. **Visually hidden review headings.** Removing the second heading outright + would drop the focus targets that review decisions and state changes move + focus to. Hiding it visually keeps those targets and the heading outline + with less code. Because kit's `kit-sr-only` stays clipped when focused, the + section draws the focus ring instead, as described under Reviews. +4. **Rename and delete stay inline.** The header menu opens the existing + inline form and confirmation instead of new dialogs. This keeps their + roles, names, and flows, and their tests change only in how they are + opened. +5. **"Show" uses a select, like Sort.** PR 2 made Sort and Group by kit + `SelectDropdown` controls with "Sort: …" trigger labels. The review state + filters follow the same pattern rather than a separate menu component. +6. **"Fact review" becomes "Facts"**, as the design's Reviews section says. + Tests that name the radio or the region change. + +## Not in PR 3 + +- **Meeting panel.** `MeetingPanel` renders in Relationships and in the person + Overview with a heavy action-filter form. Restyling it affects both, and it + is not a People-only component. It moves to a follow-up. +- **Relationship types list** in Connections shows type codes beside their + labels ("acquaintance · acquaintance"). It stays as is. +- **Identity match and fact ledger diagnostics** (endpoint kinds, evidence and + candidate IDs, source classes) stay raw, because they identify records for + diagnosis. +- **Media & files filters and sort** already work; PR 2 wired them. + +## Tests + +Changed assertions, by file: + +| File | Change | +|---|---| +| `DirectoryWorkspace.test.ts` | Filters panel must be opened before the filter fields; date fields become date inputs; Sort visible label; chips | +| `PersonDetail.test.ts` | Seven tabs; "Connections", "Media & files", "Profile", "Maintenance"; Overview order; "Merge history" moves to Maintenance | +| `StructuredProfileSection.test.ts` | Rename, history, and delete open from the header menu | +| `DirectoryReviewCentre.test.ts`, `FactReviewPanel.test.ts` | "Facts"; hidden headings; state selects; picker | +| `RelationshipReviewQueue.test.ts` | State select | +| `RelationshipsWorkspace.test.ts` | "People" button name | +| `controller.svelte.test.ts` (directory) | Invalid URL dates are dropped | +| `tests/e2e/directory.spec.ts` | `.filters` locator; tabs; maintenance region moves to Maintenance | +| `tests/directory-review.spec.ts`, `tests/e2e/accessibility.spec.ts` | "Facts"; state selects instead of radios | +| `tests/directory-network.spec.ts` | Tab count and order | + +New tests cover the behavior this PR adds: + +- Directory: setting and clearing each date boundary alone; a one-sided date + chip; removing each chip clears only its filter; an invalid URL date is + dropped from the field, chip, and request. +- Person header: "Open relationship" commits the relationship target for the + lowest participant ID and is absent with no participants; "Review facts" + commits the Facts view for the person; each menu item opens its existing + flow. +- Interval dates: each year-bearing form is accepted; `--04-12`, free text, + an out-of-range month, and February 30 are rejected with Save disabled. +- Profile dates: `--MM-DD`, `--MM`, and `---DD` are still accepted; + `2024-13` and `--02-30` are rejected; free text saves as a text date and + shows the "Saved as text" hint. +- Reviews: the Show select commits its key; the Facts picker searches, + selects, and shows the name after a reload; a focused hidden heading gives + its section a visible focus ring. +- Accessibility (axe) on each People workspace in both themes, including a + person page and the Facts picker. + +The PR includes before and after screenshots at 1440×900 and 420×860 from the +docs fixture. diff --git a/docs/internal/web-ui-overhaul-pr4-plan.md b/docs/internal/web-ui-overhaul-pr4-plan.md new file mode 100644 index 000000000..c1be6919c --- /dev/null +++ b/docs/internal/web-ui-overhaul-pr4-plan.md @@ -0,0 +1,2701 @@ +# Web UI overhaul PR 4: Manage — implementation plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use +> superpowers:subagent-driven-development (recommended) or +> superpowers:executing-plans to implement this plan task by task. Steps use +> checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Give Sources, Operations, Deletions, and Settings the shared status +vocabulary and page structure, keep Settings' category in the URL, apply +saved appearance to the open tab, and bring the Web UI guide and its +screenshots up to date, without removing any capability. + +**Architecture:** Three small modules hold the new shared logic: +`util/format.ts` (one `formatBytes`, date-time, and relative-time format), +`sources/labels.ts` (source types, sync reasons, status chips), and +`operations/labels.ts` (lane, kind, state, trigger, counter, and Set up +labels that four Operations components duplicate today). Operations replaces +its lane cards with a new `OperationStatusList`, and its controller gains a +status-only refresh. Settings reads its category from a new `settingsCategory` +field in the `explore` URL state and reports saved appearance to `App`. + +**Tech stack:** Svelte 5 (runes), `@kenn-io/kit-ui` (`RefreshControl`, +`Chip`, `Notice`, `EmptyState`, `Table`, `IconButton`, `Card`), +`@lucide/svelte`, Vitest with Testing Library, Playwright with +`@axe-core/playwright`, Go for one metadata string. + +**Spec:** [PR 4 spec](web-ui-overhaul-pr4-spec.md), which refines the Manage +sections of the [Web UI overhaul design](web-ui-overhaul-design.md). Read the +spec before each task; it is the authority where this plan is silent. Exact +copy, accessible names, URL keys, and tones in this plan are copied from it. + +## Global constraints + +- Keep every capability. A control that moves keeps its accessible name unless + the spec names the change. The named changes are: "View source operations" + becomes "Sync history"; "Refresh operations" becomes "Refresh operation + status" (status) or "Reload run history" (history); "Save settings" becomes + "Save changes"; "Stage deletion" becomes "Stage deletion…"; "Open document + index settings" and "Open document vector settings" are replaced by a + host-configuration line and guide link. +- Keep URL state keys, the `explore` JSON format, and API calls unchanged, + except: the new `settingsCategory` field (Task 3) and the removal of the + `document_index` and `document_vector` values of `settingsAuthority` + (Task 4). An old link carrying one opens Settings on Appearance. +- No new npm dependencies. Icons come from `@lucide/svelte`. +- One solid primary button per screen: `tone="info" surface="solid"`. Purple + (`workflow`) and green (`success`) are not button colors. Only destructive + confirmation buttons ("Confirm stage deletion", "Confirm cancel manifest") + are red. +- Status tones: green (`success`) healthy or finished, blue (`info`) in + progress, amber (`warning`) attention, red (`danger`) failure, gray + (`muted`) off. Every Operations status is a kit `Chip` with a word, never a + dot alone (spec decision 2). +- Sentence case for every visible label. Monospace only for identifiers, + hashes, code, keys, and cron text. +- Raw API codes are never the only text a person sees. Unknown codes fall back + to `sentenceCase` from `explore/labels.ts`. Where the spec keeps a raw code + "in a tooltip", use the element's `title` attribute, as Sources and the + selection bar already do. +- `kit-ui-check` must pass. It forbids native date inputs, hand-rolled search + inputs, empty states, icon buttons, status dots, cards (background + border + + radius on one rule), tooltips, and `@media` widths other than 640, 760, + and 900px. Use the kit component instead. +- Tests use Vitest and Testing Library or Playwright, query by role and + accessible name, and use failure-safe cleanup (`afterEach` or + `try`/`finally`). Synthetic names and `example.com` addresses only; the + Enron fixture is for docs screenshots only. +- Run commands from `web/` unless a step says otherwise. After each task run + `bun run check`, `bun run check:kit-ui`, and the focused tests. +- Commit after each task with the `kenn-io-commit` rules: new commits only, + conventional imperative subject, a short why body, the attribution trailer + the controller gives you, and a private-data check (public repository). Go + hooks run on commit; export `GOTOOLCHAIN=go1.27.1` if the shell pins an + older toolchain. + +## File structure + +| File | Responsibility | +|---|---| +| `web/src/lib/util/format.ts` (new) | `formatBytes`, `formatDateTime`, `formatRelativeTime` | +| `web/src/lib/components/{files/FilesWorkspace,files/FileViewer,explore/EverythingTable,reader/ReadingPane}.svelte` | Use the shared `formatBytes` | +| `web/src/lib/sources/labels.ts` (new) | Source-type labels, sync-reason labels, sync status chip | +| `web/src/lib/operations/labels.ts` (new) | Lane, kind, related-status, action, state, trigger, counter, duration, and Set up maps | +| `web/src/lib/components/sources/SourcesWorkspace.svelte` | Columns, status chip, reasons, row detail, status sentences | +| `web/src/lib/carddav/navigation.ts` | Settings authorities: add `semantic_search`, `person_embeddings`; remove `document_index`, `document_vector` | +| `web/src/lib/explore/{models.ts,state.svelte.ts}` | `settingsCategory` URL field | +| `web/src/lib/components/settings/SettingsWorkspace.svelte` | Category from props, save bar, posture copy, own-save notes, Notice, appearance callback | +| `web/src/lib/components/operations/OperationStatusList.svelte` (new) | Status list replacing `OperationLaneCards.svelte` (deleted) | +| `web/src/lib/components/operations/OperationHostSetup.svelte` (new) | Host-configuration line and guide link | +| `web/src/lib/components/operations/{OperationsWorkspace,OperationRunTable,OperationRunDetail,OperationRelatedStatus}.svelte` | Refresh control, Reload run history, runs table, detail, document panels | +| `web/src/lib/operations/{controller.svelte.ts,models.ts}` | `refreshStatus`, `statusUpdatedAt`, `statusRefreshing` | +| `web/src/lib/components/shell/PageHeader.svelte` | Optional `descriptionContent` snippet | +| `web/src/lib/components/deletions/DeletionsWorkspace.svelte` | Empty states, review panel, manifests table and detail | +| `web/src/lib/components/explore/SelectionBar.svelte` | `stage_deletion` reason | +| `web/src/lib/theme/preferences.svelte.ts` | `SavedAppearance`, `mergeSavedAppearance` | +| `web/src/App.svelte`, `web/src/lib/components/shell/AppShell.svelte` | Category and appearance wiring; boot screens | +| `web/src/lib/components/auth/Login.svelte`, `web/src/app.css` | Shared sign-in and boot screen rule | +| `internal/api/settings_metadata.go:100` | Default search mode description | +| `web/tests/**` | Browser and axe coverage | +| `docs/web-ui.md` | Guide for the shipped UI | + +--- + +### Task 1: Shared format and label helpers + +**Files:** +- Create: `web/src/lib/util/format.ts`, `web/src/lib/util/format.test.ts` +- Create: `web/src/lib/sources/labels.ts`, `web/src/lib/sources/labels.test.ts` +- Create: `web/src/lib/operations/labels.ts`, `web/src/lib/operations/labels.test.ts` +- Modify (delete the local `formatBytes`, import the shared one): + `web/src/lib/components/files/FilesWorkspace.svelte:8-12`, + `web/src/lib/components/files/FileViewer.svelte:2-6` (the function lives in + the ` + +

+ {setup.text} + {setup.guideLabel} +

+ + +``` + +`OperationStatusList.svelte` (props `lanes`, `actionPending`, `onNavigate`, +`onAction` as `OperationLaneCards` had, plus `onSetUp`): + +```svelte + + +
+ {#each lanes as lane (lane.lane)} + {@const headingID = `operation-lane-${lane.lane}`} +
+

{OPERATION_LANE_LABELS[lane.lane]}

+ {#if lane.kinds.length === 0} + Status unavailable + {:else} +
    + {#each lane.kinds as kind (kind.kind)} + {@const nameID = `operation-kind-${kind.kind}`} + {@const chip = statusChip(kind)} + {@const run = kind.active ?? kind.latest} + {@const setup = kind.configured ? undefined : OPERATION_SETUP[kind.kind]} +
  • + {OPERATION_KIND_LABELS[kind.kind]} + + {chip.label} + {#if kind.history_availability !== 'available'}History unavailable{/if} + + + {#if run}{/if} + {#if kind.latest_successful && run?.state !== 'succeeded'} + Last succeeded + {/if} + + + {#if kind.related_status} +
  • + {/each} +
+ {/if} +
+ {/each} +
+``` + +Styles: lanes stack vertically (`display: grid; gap: var(--space-4)`); each +`li` is a four-column grid `minmax(10rem, 1fr) auto minmax(10rem, 1fr) auto` +with a `border-top: 1px solid var(--border-muted)`; under +`@media (max-width: 760px)` it becomes one column. `.history-note` uses +`color: var(--status-warning-ink)`. Do not give one rule background, border, +and radius together (`hand-rolled-card`). "Last succeeded" appears when the +row's shown run did not succeed and a successful run exists (state comparison; +run IDs are not stable to compare). + +`OperationsWorkspace.svelte`: import `OperationStatusList` instead of +`OperationLaneCards`; pass `onSetUp`; change `onConfigure` to `() => void` +and pass it through to `OperationRelatedStatus`. + +`OperationRelatedStatus.svelte`: + +- Delete `settingsLabels`. Props: `onConfigure?: () => void`. +- Configured-false branch: + +```svelte +{:else if configured === false} +
+

Off {configurationLabels[authority]}

+
+ {#if authority === 'getVisualAttachmentStatus'} +
+ {#if reviewed} + {@const stageReason = unavailableReason('stage_deletion')} +
+ {reviewed.count.toLocaleString()} {reviewed.count === 1 ? 'item' : 'items'} · {formatBytes(reviewed.estimated_bytes)} + {reviewed.deletable_count.toLocaleString()} can be staged · {(reviewed.count - reviewed.deletable_count).toLocaleString()} will be skipped.{selectionExclusions()} + Review expires + {#if reviewed.search_deletion_scope === 'active'}Semantic search covers active messages only.{/if} + {#if stageReason}{preflightReasonLabel('stage_deletion', stageReason)}{/if} +
+
+
+ {/if} + {#if preview} +

{resultSummary(preview)}

+ {#if stageCounts(preview).skipped > 0}{/if} + {/if} + + +{/if} +``` + + Arriving from Everything keeps `reviewOnMount` and the immediate + confirmation (spec decision 5); the summary renders behind the modal. +- Manifests (replaces :329-370): + +```svelte +{#if loading}

Loading deletion manifests…

+{:else if manifests.length === 0} + +{:else} +
+ + {#snippet header()} + + + {/snippet} + {#each manifests as manifest (manifest.id)} + {@const chip = manifestStatusChip(manifest.status)} + + + + + + + + + {/each} +
{manifest.id}{manifest.description}{manifest.message_count.toLocaleString()} {manifest.message_count === 1 ? 'item' : 'items'}{chip.label} +
+ {#if detail} + {@const detailChip = manifestStatusChip(detail.status)} + + + + {/if} +
+{/if} +``` + + "Confirm stage deletion" and "Confirm cancel manifest" keep + `tone="danger" surface="solid"`. +- Styles: `.manifests.has-detail { display: grid; grid-template-columns: minmax(0, 2fr) minmax(16rem, 1fr); gap: var(--space-4); align-items: start; }` + and `@media (max-width: 900px) { .manifests.has-detail { grid-template-columns: 1fr; } }`. + `.result` is neutral (`border: 1px solid var(--border-default); background: var(--bg-subtle)`, + no radius on the same rule as a background if `kit-ui-check` flags it). + `.warning` keeps the amber border. `.reason` uses `var(--text-secondary)`. + +- [ ] **Step 1: Write the failing tests** + +```ts +it('explains how to start when nothing is selected', async () => { + render(DeletionsWorkspace, { client: createAPIClient(vi.fn(async () => Response.json({ manifests: [] }))) }); + expect(screen.getByText('Nothing selected for deletion')).toBeDefined(); + expect(screen.getByText('Select items in Everything, then choose Review for deletion…')).toBeDefined(); + expect(screen.queryByRole('button', { name: 'Review selection' })).toBeNull(); + expect(await screen.findByText('No staged deletions')).toBeDefined(); + expect(screen.getByText('msgvault delete-staged').tagName).toBe('CODE'); +}); + +it('summarizes a review with size, relative expiry, and only the staging reason', async () => { + const fetchFn = vi.fn(async (input) => { + const request = input instanceof Request ? input : new Request(input); + if (new URL(request.url).pathname.endsWith('/explore/preflight')) return Response.json(preflight({ + count: 2, deletable_count: 2, estimated_bytes: 3 * 1024, expires_at: '2026-07-19T12:00:00Z', + unavailable_actions: [ + { action: 'stage_deletion', reason: 'selection_contains_items_that_cannot_be_deleted_from_source' }, + { action: 'export', reason: 'browser_export_requires_single_message' }, + { action: 'open_in_source', reason: 'trusted_source_link_unavailable' } + ] + })); + return Response.json({ manifests: [] }); + }); + render(DeletionsWorkspace, { client: createAPIClient(fetchFn), selection: explicit, now: () => new Date('2026-07-19T10:00:00Z') }); + await fireEvent.click(await screen.findByRole('button', { name: 'Review selection' })); + + expect(await screen.findByText('2 items · 3 KB')).toBeDefined(); + expect(screen.getByText('in 2 hours').closest('span')?.textContent).toBe('Review expires in 2 hours'); + const reason = screen.getByText('None of the selected items can be deleted from their source.'); + expect(reason.getAttribute('title')).toBe('selection_contains_items_that_cannot_be_deleted_from_source'); + expect(screen.queryByText(/Export works for one message/)).toBeNull(); + expect(screen.queryByText(/provide links to open/)).toBeNull(); + expect((screen.getByRole('button', { name: 'Stage deletion…' }) as HTMLButtonElement).disabled).toBe(true); +}); + +it('makes only the confirmation button red', async () => { + render(DeletionsWorkspace, { client: createAPIClient(vi.fn(async (input) => { + const request = input instanceof Request ? input : new Request(input); + return new URL(request.url).pathname.endsWith('/explore/preflight') ? Response.json(preflight()) : Response.json({ manifests: [] }); + })), selection: explicit }); + await fireEvent.click(await screen.findByRole('button', { name: 'Review selection' })); + const stage = await screen.findByRole('button', { name: 'Stage deletion…' }); + expect(stage.className).toContain('kit-button--info'); + expect(stage.className).toContain('kit-button--solid'); + expect(screen.getByRole('button', { name: 'Dry run' }).className).toContain('kit-button--outline'); + await fireEvent.click(stage); + expect(screen.getByRole('button', { name: 'Confirm stage deletion' }).className).toContain('kit-button--danger'); +}); + +it('lists manifests with status chips and closes the detail', async () => { + render(DeletionsWorkspace, { client: createAPIClient(vi.fn(async (input) => { + const request = input instanceof Request ? input : new Request(input); + if (new URL(request.url).pathname.endsWith('/batch-1')) return Response.json({ ...listResponse().manifests[0], account: 'archive@example.com' }); + return Response.json(listResponse()); + })) }); + const table = await screen.findByRole('table', { name: 'Deletion manifests' }); + expect(within(table).getAllByRole('columnheader').map((cell) => cell.textContent?.trim())) + .toEqual(['ID', 'Description', 'Items', 'Status', 'Created', 'Actions']); + expect(within(table).getByText('Pending')).toBeDefined(); + expect(within(table).getByRole('button', { name: 'Cancel batch-1' }).className).not.toContain('kit-button--danger'); + await fireEvent.click(within(table).getByRole('button', { name: 'Inspect batch-1' })); + expect(await screen.findByText('archive@example.com')).toBeDefined(); + await fireEvent.click(screen.getByRole('button', { name: 'Close manifest detail' })); + expect(screen.queryByText('archive@example.com')).toBeNull(); +}); +``` + +Existing tests: rename "Stage deletion" to "Stage deletion…" (:70, :102, :148, +:253, :399, :428); "1 item · 120 bytes" (:374) → "1 item · 120 B"; :398 +`/selection_contains_items_that_cannot_be_deleted_from_source/` → the +sentence; :359 `findAllByText('cancelled')` → `findAllByText('Cancelled')`. +The partial-staging assertions (:146, :156) still find `role="alert"`. Import +`within` from Testing Library. + +- [ ] **Step 2: Run** `bunx vitest run src/lib/components/deletions src/lib/components/shell`. + Expected: FAIL. +- [ ] **Step 3: Implement.** +- [ ] **Step 4: Run tests and checks.** Expected: PASS. +- [ ] **Step 5: Commit.** Subject: `feat(web): give Deletions empty states, a review panel, and a manifests table`. + +--- + +### Task 7: Selection bar staging reason + +**Files:** +- Modify: `web/src/lib/components/explore/SelectionBar.svelte:44-46,111-118` +- Test: `web/src/lib/components/explore/SelectionBar.test.ts` + +**Interfaces:** +- Consumes: `preflightReasonLabel` (already imported). + +Implementation: + +```ts +const stageReason = $derived(preflight?.unavailable_actions.find((item) => item.action === 'stage_deletion')?.reason); +const stageReasonID = $props.id(); +``` + +```svelte +{#if onReviewDeletion} +
- {#if reviewed} -
- {reviewed.count.toLocaleString()} - {reviewed.count === 1 ? 'item' : 'items'} · {reviewed.estimated_bytes.toLocaleString()} bytes - - {reviewed.deletable_count.toLocaleString()} can be staged · {(reviewed.count - reviewed.deletable_count).toLocaleString()} will be skipped.{selectionExclusions()} - - Authority expires {reviewed.expires_at} - {#if reviewed.search_deletion_scope === 'active'} - Semantic search covers active messages only. - {/if} - {#each reviewed.unavailable_actions as unavailable (`${unavailable.action}:${unavailable.reason}`)} - {unavailable.action}: {unavailable.reason} - {/each} -
-
-
- {/if} - {#if preview} -

{resultSummary(preview)}

- {#if stageCounts(preview).skipped > 0}{/if} - {/if} - - + {#if reviewed} + {@const stageReason = unavailableReason('stage_deletion')} +
+ {reviewed.count.toLocaleString()} + {reviewed.count === 1 ? 'item' : 'items'} · {formatBytes(reviewed.estimated_bytes)} + + {reviewed.deletable_count.toLocaleString()} can be staged · {(reviewed.count - reviewed.deletable_count).toLocaleString()} will be skipped.{selectionExclusions()} + + Review expires + {#if reviewed.search_deletion_scope === 'active'} + Semantic search covers active messages only. + {/if} + {#if stageReason} + {preflightReasonLabel('stage_deletion', stageReason)} + {/if} +
+
+
+ {/if} + {#if preview} +

{resultSummary(preview)}

+ {#if stageCounts(preview).skipped > 0}{/if} + {/if} + + + {/if} {#if loading}

Loading deletion manifests…

- {:else if manifests.length === 0}

No deletion manifests yet.

+ {:else if manifests.length === 0} + {:else} -
- {#each manifests as manifest (manifest.id)} -
-
{manifest.id}{manifest.description}
- {manifest.message_count.toLocaleString()} {manifest.message_count === 1 ? 'item' : 'items'} - {manifest.status} -
-
+ {detailChip.label} + {detail.account || 'Account unavailable'} + {detail.message_count.toLocaleString()} items · {detail.description} + {#if detail.execution} + {detail.execution.succeeded} succeeded · {detail.execution.failed} failed + {#each detail.execution.failed_ids ?? [] as id}{id}{/each} {/if} -
- - {/each} - - {/if} - - {#if detail} - - - + + + {/if} +
{/if} @@ -435,79 +511,76 @@ flex: 1; flex-direction: column; gap: var(--space-4); - padding: var(--space-5) var(--space-6); + padding: var(--space-5) var(--page-gutter) var(--space-4); } - header, - article, .actions, - .staging { + .staging-header, + .detail-header, + .row-actions { display: flex; align-items: center; gap: var(--space-3); } - header { + .staging { + display: grid; + gap: var(--space-3); + } + .staging-header, + .detail-header { justify-content: space-between; } - header p, - h1, h2, .staging p { margin: 0; } - header p { - color: var(--status-warning-ink); - font-size: var(--font-size-2xs); - font-weight: 800; - letter-spacing: 0.1em; - text-transform: uppercase; - } - header span, - article span, - .staging p, - .actions span, - aside span { + .detail span { color: var(--text-muted); font-size: var(--font-size-xs); } - .staging { - flex-wrap: wrap; - justify-content: space-between; - } .review { display: grid; gap: var(--space-1); } - .reason, + .reason { + color: var(--text-secondary); + } .notice--error { color: var(--text-danger); } - .manifest-list { + .manifests.has-detail { display: grid; - border-top: 1px solid var(--border-muted); - } - article { - justify-content: space-between; - padding: var(--space-3); - border-bottom: 1px solid var(--border-muted); + grid-template-columns: minmax(0, 2fr) minmax(16rem, 1fr); + gap: var(--space-4); + align-items: start; } - article > div:first-child, - aside { + .detail { display: grid; - gap: var(--space-1); + gap: var(--space-2); } .notice, - .preview { + .result, + .warning { padding: var(--space-3); border: 1px solid var(--accent-amber); - border-radius: var(--radius-md); background: var(--bg-subtle); } + .notice, + .warning { + border-radius: var(--radius-md); + } + .result { + border-color: var(--border-default); + } .notice--error { border-color: var(--accent-red); } + @media (max-width: 900px) { + .manifests.has-detail { + grid-template-columns: 1fr; + } + } @media (max-width: 760px) { - article, - .staging { + .staging-header { align-items: stretch; flex-direction: column; } diff --git a/web/src/lib/components/deletions/DeletionsWorkspace.test.ts b/web/src/lib/components/deletions/DeletionsWorkspace.test.ts index 2c8a506d5..fdee9da3b 100644 --- a/web/src/lib/components/deletions/DeletionsWorkspace.test.ts +++ b/web/src/lib/components/deletions/DeletionsWorkspace.test.ts @@ -1,8 +1,9 @@ -import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import { fireEvent, render, screen, waitFor, within } from '@testing-library/svelte'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { appShortcuts, initShortcuts } from '@kenn-io/kit-ui'; import { createAPIClient } from '../../api/client'; +import { formatDateTime } from '../../util/format'; import type { ExploreSelection as GeneratedExploreSelection } from '../../api/generated/models'; import DeletionsWorkspace from './DeletionsWorkspace.svelte'; @@ -56,6 +57,106 @@ function listResponse() { afterEach(() => document.body.replaceChildren()); describe('DeletionsWorkspace', () => { + it('explains how to start when nothing is selected', async () => { + render(DeletionsWorkspace, { + client: createAPIClient(vi.fn(async () => Response.json({ manifests: [] }))), + }); + expect(screen.getByText('Nothing selected for deletion')).toBeDefined(); + expect(screen.getByText('Select items in Everything, then choose Review for deletion…')).toBeDefined(); + expect(screen.queryByRole('button', { name: 'Review selection' })).toBeNull(); + expect(await screen.findByText('No staged deletions')).toBeDefined(); + expect(screen.getByText('msgvault delete-staged').tagName).toBe('CODE'); + }); + + it('summarizes a review with size, relative expiry, and only the staging reason', async () => { + const fetchFn = vi.fn(async (input) => { + const request = input instanceof Request ? input : new Request(input); + if (new URL(request.url).pathname.endsWith('/explore/preflight')) + return Response.json( + preflight({ + count: 2, + deletable_count: 2, + estimated_bytes: 3 * 1024, + expires_at: '2026-07-19T12:00:00Z', + unavailable_actions: [ + { action: 'stage_deletion', reason: 'selection_contains_items_that_cannot_be_deleted_from_source' }, + { action: 'export', reason: 'browser_export_requires_single_message' }, + { action: 'open_in_source', reason: 'trusted_source_link_unavailable' }, + ], + }), + ); + return Response.json({ manifests: [] }); + }); + render(DeletionsWorkspace, { + client: createAPIClient(fetchFn), + selection: explicit, + now: () => new Date('2026-07-19T10:00:00Z'), + }); + await fireEvent.click(await screen.findByRole('button', { name: 'Review selection' })); + + expect(await screen.findByText('2 items · 3 KB')).toBeDefined(); + expect(screen.getByText('in 2 hours').closest('span')?.textContent).toBe('Review expires in 2 hours'); + const reason = screen.getByText('None of the selected items can be deleted from their source.'); + expect(reason.getAttribute('title')).toBe('selection_contains_items_that_cannot_be_deleted_from_source'); + expect(screen.queryByText(/Export works for one message/)).toBeNull(); + expect(screen.queryByText(/provide links to open/)).toBeNull(); + expect((screen.getByRole('button', { name: 'Stage deletion…' }) as HTMLButtonElement).disabled).toBe(true); + }); + + it('makes only the confirmation button red', async () => { + render(DeletionsWorkspace, { + client: createAPIClient( + vi.fn(async (input) => { + const request = input instanceof Request ? input : new Request(input); + return new URL(request.url).pathname.endsWith('/explore/preflight') + ? Response.json(preflight()) + : Response.json({ manifests: [] }); + }), + ), + selection: explicit, + }); + await fireEvent.click(await screen.findByRole('button', { name: 'Review selection' })); + const stage = await screen.findByRole('button', { name: 'Stage deletion…' }); + expect(stage.className).toContain('kit-button--info'); + expect(stage.className).toContain('kit-button--solid'); + expect(screen.getByRole('button', { name: 'Dry run' }).className).toContain('kit-button--outline'); + await fireEvent.click(stage); + expect(screen.getByRole('button', { name: 'Confirm stage deletion' }).className).toContain('kit-button--danger'); + }); + + it('lists manifests with status chips and closes the detail back to its Inspect button', async () => { + render(DeletionsWorkspace, { + client: createAPIClient( + vi.fn(async (input) => { + const request = input instanceof Request ? input : new Request(input); + if (new URL(request.url).pathname.endsWith('/batch-1')) + return Response.json({ ...listResponse().manifests[0], account: 'archive@example.com' }); + return Response.json(listResponse()); + }), + ), + }); + const table = await screen.findByRole('table', { name: 'Deletion manifests' }); + expect(within(table).getAllByRole('columnheader').map((cell) => cell.textContent?.trim())).toEqual([ + 'ID', + 'Description', + 'Items', + 'Status', + 'Created', + 'Actions', + ]); + expect(within(table).getByText('Pending')).toBeDefined(); + expect(within(table).getByTitle(formatDateTime('2026-07-19T10:00:00Z', 'long'))).toBeDefined(); + expect(within(table).queryByTitle('2026-07-19T10:00:00Z')).toBeNull(); + expect(within(table).getByRole('button', { name: 'Cancel batch-1' }).className).not.toContain('kit-button--danger'); + await fireEvent.click(within(table).getByRole('button', { name: 'Inspect batch-1' })); + expect(await screen.findByText('archive@example.com')).toBeDefined(); + await fireEvent.click(screen.getByRole('button', { name: 'Close manifest detail' })); + expect(screen.queryByText('archive@example.com')).toBeNull(); + await waitFor(() => + expect(document.activeElement).toBe(within(table).getByRole('button', { name: 'Inspect batch-1' })), + ); + }); + it('requires the deletable-count contract before offering staging', async () => { const fetchFn = vi.fn(async (input) => { const request = input instanceof Request ? input : new Request(input); @@ -67,7 +168,7 @@ describe('DeletionsWorkspace', () => { render(DeletionsWorkspace, { client: createAPIClient(fetchFn), selection: explicit, reviewOnMount: true }); expect((await screen.findByRole('alert')).textContent).toContain('Upgrade the daemon and review again.'); - expect(screen.queryByRole('button', { name: 'Stage deletion' })).toBeNull(); + expect(screen.queryByRole('button', { name: 'Stage deletion…' })).toBeNull(); expect(screen.queryByRole('dialog')).toBeNull(); }); @@ -94,12 +195,12 @@ describe('DeletionsWorkspace', () => { await screen.findByText('batch-1'); await fireEvent.click(screen.getByRole('button', { name: 'Review selection' })); - expect(await screen.findByText('1 item · 120 bytes')).toBeDefined(); + expect(await screen.findByText('1 item · 120 B')).toBeDefined(); await fireEvent.click(screen.getByRole('button', { name: 'Dry run' })); expect(await screen.findByText(/Dry run: Matched: 1 · Staged: 1 · Skipped: 0 in archive@example.com/)).toBeDefined(); expect(screen.queryByRole('alert')).toBeNull(); - await fireEvent.click(screen.getByRole('button', { name: 'Stage deletion' })); + await fireEvent.click(screen.getByRole('button', { name: 'Stage deletion…' })); expect(screen.getByRole('dialog', { name: 'Confirm selected deletion' })).toBeDefined(); expect(deletionPosts).toBe(1); await fireEvent.click(screen.getByRole('button', { name: 'Confirm stage deletion' })); @@ -138,17 +239,17 @@ describe('DeletionsWorkspace', () => { }); render(DeletionsWorkspace, { client: createAPIClient(fetchFn), selection: explicit }); - await screen.findByText('No deletion manifests yet.'); + await screen.findByText('No staged deletions'); await fireEvent.click(screen.getByRole('button', { name: 'Review selection' })); - await screen.findByText('3 items · 120 bytes'); + await screen.findByText('3 items · 120 B'); await fireEvent.click(screen.getByRole('button', { name: 'Dry run' })); expect(await screen.findByText(/Dry run: Matched: 3 · Staged: 2 · Skipped: 1 in archive@example.com/)).toBeDefined(); - expect(screen.getByRole('alert').textContent).toMatch(/Partial staging.*deletable Gmail subset.*unsupported match will be skipped/); + expect(screen.getByRole('alert').textContent).toMatch(/Partial staging.*deletable Gmail and Microsoft Graph mail subset.*unsupported match will be skipped/); - await fireEvent.click(screen.getByRole('button', { name: 'Stage deletion' })); + await fireEvent.click(screen.getByRole('button', { name: 'Stage deletion…' })); const dialog = screen.getByRole('dialog', { name: 'Confirm selected deletion' }); expect(dialog.textContent).toMatch(/Dry run: Matched: 3 · Staged: 2 · Skipped: 1/); - expect(dialog.textContent).toMatch(/Only deletable Gmail messages will be staged/); + expect(dialog.textContent).toMatch(/Only deletable Gmail and Microsoft Graph mail messages will be staged/); expect(deletionPosts).toBe(1); await fireEvent.click(screen.getByRole('button', { name: 'Confirm stage deletion' })); await waitFor(() => expect(deletionPosts).toBe(2)); @@ -173,9 +274,9 @@ describe('DeletionsWorkspace', () => { }); render(DeletionsWorkspace, { client: createAPIClient(fetchFn), selection: explicit }); - await screen.findByText('No deletion manifests yet.'); + await screen.findByText('No staged deletions'); await fireEvent.click(screen.getByRole('button', { name: 'Review selection' })); - await screen.findByText('3 items · 120 bytes'); + await screen.findByText('3 items · 120 B'); await fireEvent.click(screen.getByRole('button', { name: 'Dry run' })); expect(await screen.findByText(/Dry run: Matched: 3 · Staged: 2 · Skipped: 1/)).toBeDefined(); await fireEvent.click(screen.getByRole('button', { name: 'Dry run' })); @@ -201,9 +302,9 @@ describe('DeletionsWorkspace', () => { }); const rendered = render(DeletionsWorkspace, { client: createAPIClient(fetchFn), selection: explicit }); - await screen.findByText('No deletion manifests yet.'); + await screen.findByText('No staged deletions'); await fireEvent.click(screen.getByRole('button', { name: 'Review selection' })); - await screen.findByText('1 item · 120 bytes'); + await screen.findByText('1 item · 120 B'); await fireEvent.click(screen.getByRole('button', { name: 'Dry run' })); await waitFor(() => expect(posts).toBe(1)); await rendered.rerender({ client: createAPIClient(fetchFn), selection: matching }); @@ -247,10 +348,10 @@ describe('DeletionsWorkspace', () => { }); const rendered = render(DeletionsWorkspace, { client: createAPIClient(fetchFn), selection: explicit }); - await screen.findByText('No deletion manifests yet.'); + await screen.findByText('No staged deletions'); await fireEvent.click(screen.getByRole('button', { name: 'Review selection' })); - await screen.findByText('1 item · 120 bytes'); - await fireEvent.click(screen.getByRole('button', { name: 'Stage deletion' })); + await screen.findByText('1 item · 120 B'); + await fireEvent.click(screen.getByRole('button', { name: 'Stage deletion…' })); await fireEvent.click(screen.getByRole('button', { name: 'Confirm stage deletion' })); await waitFor(() => expect(stagePosts).toBe(1)); await rendered.rerender({ client: createAPIClient(fetchFn), selection: matching }); @@ -291,7 +392,7 @@ describe('DeletionsWorkspace', () => { }); const rendered = render(DeletionsWorkspace, { client: createAPIClient(fetchFn), selection: matching }); try { - await screen.findByText('No deletion manifests yet.'); + await screen.findByText('No staged deletions'); await fireEvent.keyDown(window, { key: 'D', shiftKey: true }); expect(await screen.findByRole('dialog', { name: 'Confirm matching deletion' })).toBeDefined(); expect(screen.getByText(/Matched: 8 · Will stage: 6 · Will skip: 2.*After 1 exclusion/)).toBeDefined(); @@ -316,7 +417,7 @@ describe('DeletionsWorkspace', () => { }); const rendered = render(DeletionsWorkspace, { client: createAPIClient(fetchFn), selection: explicit }); try { - await screen.findByText('No deletion manifests yet.'); + await screen.findByText('No staged deletions'); await fireEvent.keyDown(window, { key: 'd' }); expect(await screen.findByRole('dialog', { name: 'Confirm selected deletion' })).toBeDefined(); expect(shellHandler).not.toHaveBeenCalled(); @@ -356,7 +457,7 @@ describe('DeletionsWorkspace', () => { expect(requests.some((request) => request.method === 'DELETE')).toBe(false); await fireEvent.click(screen.getByRole('button', { name: 'Confirm cancel manifest' })); await waitFor(() => expect(requests.some((request) => request.method === 'DELETE')).toBe(true)); - expect((await screen.findAllByText('cancelled')).length).toBeGreaterThan(0); + expect((await screen.findAllByText('Cancelled')).length).toBeGreaterThan(0); }); it('discloses the active-only deletion scope reported by the preflight review', async () => { @@ -371,7 +472,7 @@ describe('DeletionsWorkspace', () => { render(DeletionsWorkspace, { client: createAPIClient(fetchFn), selection: explicit }); await fireEvent.click(await screen.findByRole('button', { name: 'Review selection' })); - await screen.findByText('1 item · 120 bytes'); + await screen.findByText('1 item · 120 B'); expect(screen.queryByText(/active messages only/)).toBeNull(); scoped = true; @@ -395,8 +496,8 @@ describe('DeletionsWorkspace', () => { render(DeletionsWorkspace, { client: createAPIClient(fetchFn), selection: explicit }); await fireEvent.click(await screen.findByRole('button', { name: 'Review selection' })); - expect(await screen.findByText(/selection_contains_items_that_cannot_be_deleted_from_source/)).toBeDefined(); - expect((screen.getByRole('button', { name: 'Stage deletion' }) as HTMLButtonElement).disabled).toBe(true); + expect(await screen.findByText('None of the selected items can be deleted from their source.')).toBeDefined(); + expect((screen.getByRole('button', { name: 'Stage deletion…' }) as HTMLButtonElement).disabled).toBe(true); }); it('clears stale result counts when dry-run and create requests fail', async () => { @@ -414,9 +515,9 @@ describe('DeletionsWorkspace', () => { }); render(DeletionsWorkspace, { client: createAPIClient(fetchFn), selection: explicit }); - await screen.findByText('No deletion manifests yet.'); + await screen.findByText('No staged deletions'); await fireEvent.click(screen.getByRole('button', { name: 'Review selection' })); - await screen.findByText('1 item · 120 bytes'); + await screen.findByText('1 item · 120 B'); await fireEvent.click(screen.getByRole('button', { name: 'Dry run' })); expect(await screen.findByText(/Dry run: Matched: 1/)).toBeDefined(); await fireEvent.click(screen.getByRole('button', { name: 'Dry run' })); @@ -425,7 +526,7 @@ describe('DeletionsWorkspace', () => { await fireEvent.click(screen.getByRole('button', { name: 'Dry run' })); expect(await screen.findByText(/Dry run: Matched: 1/)).toBeDefined(); - await fireEvent.click(screen.getByRole('button', { name: 'Stage deletion' })); + await fireEvent.click(screen.getByRole('button', { name: 'Stage deletion…' })); await fireEvent.click(screen.getByRole('button', { name: 'Confirm stage deletion' })); expect(await screen.findByText('create failed')).toBeDefined(); expect(screen.queryByText(/Dry run: Matched: 1/)).toBeNull(); diff --git a/web/src/lib/components/directory/AttributeDefinitionDialog.test.ts b/web/src/lib/components/directory/AttributeDefinitionDialog.test.ts index ffc118cac..4f1a035ee 100644 --- a/web/src/lib/components/directory/AttributeDefinitionDialog.test.ts +++ b/web/src/lib/components/directory/AttributeDefinitionDialog.test.ts @@ -978,13 +978,20 @@ describe('AttributeDefinitionDialog', () => { is_sensitive: false, options, }); + const unrelated = definition({ + id: 8, + universal_id: 'unrelated-empty', + slug: 'unrelated_empty', + label: 'Unrelated empty field', + is_sensitive: false, + }); const fetchFn = vi.fn(async (input) => { const request = input instanceof Request ? input : new Request(input); return request.method === 'POST' ? Response.json(created, { status: 201 }) - : Response.json({ definitions: [created] }); + : Response.json({ definitions: [unrelated, created] }); }); - render(AttributeSection, { controller: controller(fetchFn) }); + render(AttributeSection, { controller: controller(fetchFn, [unrelated]) }); await fireEvent.click(screen.getByRole('button', { name: 'Create attribute field' })); await fireEvent.input(screen.getByLabelText('Label'), { target: { value: 'Option field' } }); await chooseSelectOption(screen.getByLabelText('Value type'), option); @@ -992,6 +999,7 @@ describe('AttributeDefinitionDialog', () => { await fireEvent.click(screen.getByRole('button', { name: 'Create field' })); await screen.findByRole('status'); await fireEvent.click(screen.getByRole('button', { name: 'Done' })); + expect(screen.queryByRole('heading', { name: 'Unrelated empty field' })).toBeNull(); await fireEvent.click(screen.getByRole('button', { name: 'Add Option field value' })); expect(screen.getByRole('form', { name: 'Add Option field value' }).textContent).toContain(visible); @@ -1018,6 +1026,7 @@ describe('AttributeDefinitionDialog', () => { }); const profile = controller(fetchFn, [existing]); render(AttributeSection, { controller: profile }); + await fireEvent.click(screen.getByRole('button', { name: 'Show empty fields (1)' })); await fireEvent.click(screen.getByRole('button', { name: 'Add Existing field value' })); await fireEvent.input(screen.getByRole('textbox', { name: 'Existing field' }), { target: { value: 'retained local draft' }, diff --git a/web/src/lib/components/directory/AttributeSection.svelte b/web/src/lib/components/directory/AttributeSection.svelte index b2fa05eca..4339391bc 100644 --- a/web/src/lib/components/directory/AttributeSection.svelte +++ b/web/src/lib/components/directory/AttributeSection.svelte @@ -4,17 +4,16 @@ import type { AttributeDefinition as GeneratedAttributeDefinition, - AttributeValue as GeneratedAttributeValue, PersonAttributeGroup as GeneratedPersonAttributeGroup, PersonAttributeValue as GeneratedPersonAttributeValue, } from '../../api/generated/models'; import type { DirectoryProfileController } from '../../directory/profile-controller.svelte'; import AttributeDefinitionDialog from './AttributeDefinitionDialog.svelte'; import AttributeEditor from './AttributeEditor.svelte'; + import { displayAttributeValue } from './attribute-value'; type AttributeDefinition = GeneratedAttributeDefinition; type AttributeGroup = GeneratedPersonAttributeGroup; - type AttributeValue = GeneratedAttributeValue; type PersonAttributeValue = GeneratedPersonAttributeValue; interface Props { @@ -41,9 +40,22 @@ confirming = undefined; revealed = {}; creatingDefinition = false; + showEmpty = false; + recentlyCreatedID = null; }); const fields = $derived.by(() => joinDefinitions(controller.definitions, controller.attributes?.attributes ?? [])); + let showEmpty = $state(false); + let recentlyCreatedID = $state(null); + const emptyCount = $derived(fields.filter((field) => field.current.length === 0).length); + const visibleFields = $derived(showEmpty ? fields : fields.filter((field) => + field.current.length > 0 || editing?.universalID === field.definition.universal_id || recentlyCreatedID === field.definition.universal_id + )); + + // Keep a newly created field actionable after its confirmation dialog closes. + $effect(() => { + if (controller.createdDefinition) recentlyCreatedID = controller.createdDefinition.universal_id; + }); function joinDefinitions(definitions: AttributeDefinition[], groups: AttributeGroup[]): JoinedAttribute[] { const byUniversalID = new Map(groups.map((group) => [group.definition.universal_id, group])); @@ -91,33 +103,6 @@ if (confirming?.universalID === definition.universal_id) confirming = undefined; } - function displayValue(definition: AttributeDefinition, value: AttributeValue): string { - const canonical = rawValue(value); - const choice = definition.options?.choices?.find((candidate) => candidate.value === canonical); - return choice?.label ?? canonical; - } - - function rawValue(value: AttributeValue): string { - switch (value.type) { - case 'text': - return value.text ?? '—'; - case 'integer': - return value.integer?.toString() ?? '—'; - case 'real': - return value.real?.toString() ?? '—'; - case 'boolean': - return value.boolean === undefined ? '—' : value.boolean ? 'Yes' : 'No'; - case 'date': - return value.date ?? '—'; - case 'timestamp': - return value.timestamp ?? '—'; - case 'record_reference': - return value.record_type === 'person' && value.record_id ? `Person ${value.record_id}` : '—'; - default: - return value.json === undefined ? '—' : JSON.stringify(value.json); - } - } - function provenance(value: PersonAttributeValue): string { return [ `Source: ${value.source}`, @@ -241,7 +226,7 @@ } -
+

Attributes

diff --git a/web/src/lib/components/directory/AttributeSummary.test.ts b/web/src/lib/components/directory/AttributeSummary.test.ts new file mode 100644 index 000000000..6e09134ed --- /dev/null +++ b/web/src/lib/components/directory/AttributeSummary.test.ts @@ -0,0 +1,53 @@ +import { cleanup, fireEvent, render, screen } from '@testing-library/svelte'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import type { AttributeValue, PersonAttributeGroup } from '../../api/generated/models'; +import AttributeSummary from './AttributeSummary.svelte'; + +afterEach(() => cleanup()); + +function group(label: string, value: AttributeValue | undefined, sensitive = false): PersonAttributeGroup { + return { + definition: { universal_id: `u-${label}`, slug: label.toLowerCase(), label, is_sensitive: sensitive, display_order: 0 }, + current: value === undefined ? [] : [{ value }], + history: [] + } as unknown as PersonAttributeGroup; +} + +describe('AttributeSummary', () => { + it('lists only current values, formats typed values and choice labels, and conceals sensitive values', async () => { + const onEdit = vi.fn(); + const status = group('Status', { type: 'text', text: 'active' }); + status.definition.options = { choices: [{ value: 'active', label: 'Active contact' }] }; + const subscribed = group('Subscribed', { type: 'boolean', boolean: true }); + subscribed.definition.options = { choices: [{ value: 'true', label: 'Opted in' }] }; + render(AttributeSummary, { + groups: [ + group('Birthday', { type: 'date', date: '1990-01-01' }), + subscribed, + status, + group('Employer', undefined), + group('Health', { type: 'text', text: 'synthetic private value' }, true) + ], + onEdit + }); + const region = screen.getByRole('region', { name: 'Attributes summary' }); + expect(region.textContent).toContain('Birthday'); + expect(region.textContent).toContain('1990-01-01'); + expect(region.textContent).toContain('Subscribed'); + expect(region.textContent).toContain('Opted in'); + expect(region.textContent).toContain('Active contact'); + expect(region.textContent).not.toContain('active'); + expect(region.textContent).not.toContain('Employer'); + expect(region.textContent).toContain('Health'); + expect(region.textContent).toContain('concealed'); + expect(region.textContent).not.toContain('synthetic private value'); + await fireEvent.click(screen.getByRole('button', { name: 'Edit attributes' })); + expect(onEdit).toHaveBeenCalledOnce(); + }); + + it('renders nothing when no attribute has a current value', () => { + const { container } = render(AttributeSummary, { groups: [group('Employer', undefined)] }); + expect(container.querySelector('section')).toBeNull(); + }); +}); diff --git a/web/src/lib/components/directory/CardDAVPublicationControl.svelte b/web/src/lib/components/directory/CardDAVPublicationControl.svelte index f81b71e56..ff616c873 100644 --- a/web/src/lib/components/directory/CardDAVPublicationControl.svelte +++ b/web/src/lib/components/directory/CardDAVPublicationControl.svelte @@ -145,7 +145,7 @@ /> {#if publication.conflict_id}
diff --git a/web/src/lib/components/directory/DirectoryList.test.ts b/web/src/lib/components/directory/DirectoryList.test.ts index a6290dc46..0455acdeb 100644 --- a/web/src/lib/components/directory/DirectoryList.test.ts +++ b/web/src/lib/components/directory/DirectoryList.test.ts @@ -56,7 +56,22 @@ describe('DirectoryList', () => { hasMore: false, selectedPersonID: null, onSelect: vi.fn(), onLoadMore: vi.fn(), onReload: vi.fn() }); - expect(screen.getByRole('row', { name: /Alpha Fixture/ }).textContent).toContain('Last contact 2026-08-20T10:00:00Z'); + expect(screen.getByRole('row', { name: /Alpha Fixture/ }).textContent).toContain('Last contact Aug 20, 2026'); expect(screen.getByRole('row', { name: /Bravo Fixture/ }).textContent).toContain('Never contacted'); }); + + it('names the channel and contact state and formats the last contact', () => { + render(DirectoryList, { + rows: [{ + ...rows[0]!, primary_channel: 'email', contact_state: 'active', last_contact_at: '2002-01-02T12:00:00Z' + }], + loading: false, loadingMore: false, error: null, pageError: null, pageRecovery: null, + hasMore: false, selectedPersonID: null, onSelect: vi.fn(), onLoadMore: vi.fn(), onReload: vi.fn() + }); + + const text = screen.getByRole('row', { name: /Alpha Fixture/ }).textContent ?? ''; + expect(text).toContain('Email · Active'); + expect(text).toContain('Last contact Jan 2, 2002'); + expect(text).not.toContain('T12:00'); + }); }); diff --git a/web/src/lib/components/directory/DirectoryReviewCentre.svelte b/web/src/lib/components/directory/DirectoryReviewCentre.svelte index 78d6b051f..191345878 100644 --- a/web/src/lib/components/directory/DirectoryReviewCentre.svelte +++ b/web/src/lib/components/directory/DirectoryReviewCentre.svelte @@ -1,7 +1,8 @@
- + + {#snippet view()} + + {/snippet} + {#if controller.reviewKind === 'identity'} -
+
+

Identity matches

-
-

Identity matches

-

Review server-supplied evidence before linking or separating identities.

-
- ({ ...option, triggerLabel: `Show: ${option.label}` }))} onchange={selectIdentityState} - ariaLabel="Identity review state" disabled={!!activeDecision} />
@@ -161,6 +160,10 @@

{controller.status}

{/if} + {#if controller.decisionError && !activeDecision} + + {/if} + {#if controller.loading && controller.rows.length === 0}

Loading identity matches… @@ -224,7 +227,13 @@

{:else if controller.reviewKind === 'fact'} {#if factController} - + {/if} {:else} @@ -252,12 +261,14 @@ {/if} diff --git a/web/src/lib/components/directory/DirectoryReviewCentre.test.ts b/web/src/lib/components/directory/DirectoryReviewCentre.test.ts index dc66e0c04..8700aa3de 100644 --- a/web/src/lib/components/directory/DirectoryReviewCentre.test.ts +++ b/web/src/lib/components/directory/DirectoryReviewCentre.test.ts @@ -1,4 +1,4 @@ -import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import { cleanup, fireEvent, render, screen, waitFor, within } from '@testing-library/svelte'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { createAPIClient } from '../../api/client'; @@ -23,6 +23,9 @@ function candidate(id: number, state = 'candidate'): IdentityMatchCandidate { basis: 'stable_provider_id', source: 'synthetic', state, + review_token: `token-${id}-${state}`, + actionable: state === 'candidate', + application_pending: false, evidence: [], created_at: '2026-08-01T10:00:00Z', updated_at: '2026-08-02T11:00:00Z' @@ -48,11 +51,53 @@ function renderReview(controller: DirectoryReviewController) { controller, relationshipController: new RelationshipReviewController(controller.apiClient), factController: new FactLedgerController(controller.apiClient), + client: controller.apiClient, directoryPersonID: null }); } describe('DirectoryReviewCentre', () => { + it.each([ + { action: 'Link identities', state: 'accepted' }, + { action: 'Keep separate', state: 'rejected' } + ])('closes a stale $action decision until the refreshed evidence is reviewed again', async ({ action, state }) => { + const tokens: string[] = []; + const refreshed = { ...candidate(17), review_token: 'fresh-token', source_ref: 'Refreshed source evidence' }; + const controller = new DirectoryReviewController(createAPIClient(vi.fn(async (input) => { + const request = requestOf(input); + if (request.method === 'POST') { + tokens.push((await request.json()).review_token); + if (tokens.length === 1) { + return Response.json({ error: 'identity_match_review_stale', message: 'Changed' }, { status: 409 }); + } + return Response.json({ candidate: candidate(17, state), identity_revision: 5, cache_state: 'ready' }); + } + return page([refreshed]); + }))); + controller.rows = [candidate(17)]; + renderReview(controller); + + await fireEvent.click(screen.getByRole('button', { name: action })); + await fireEvent.input(screen.getByRole('textbox', { name: 'Decision notes' }), { + target: { value: 'Retain this review note' } + }); + await fireEvent.click(within(screen.getByRole('dialog', { name: action })).getByRole('button', { name: action })); + + await waitFor(() => expect(screen.queryByRole('dialog')).toBeNull()); + expect(screen.getByRole('alert').textContent).toContain('Review the refreshed evidence before deciding.'); + expect(screen.getByText('Refreshed source evidence')).toBeDefined(); + expect(tokens).toEqual(['token-17-candidate']); + const trigger = screen.getByRole('button', { name: action }); + await waitFor(() => expect(document.activeElement).toBe(trigger)); + + await fireEvent.click(trigger); + expect(screen.getByRole('textbox', { name: 'Decision notes' })).toHaveProperty('value', 'Retain this review note'); + expect(tokens).toEqual(['token-17-candidate']); + await fireEvent.click(within(screen.getByRole('dialog', { name: action })).getByRole('button', { name: action })); + await waitFor(() => expect(screen.queryByRole('dialog')).toBeNull()); + expect(tokens).toEqual(['token-17-candidate', 'fresh-token']); + }); + it('selects the read-only imported relationship queue without identity requests', async () => { const calls: Array<{ method: string; path: string; status: string | null }> = []; const fetchFn = vi.fn(async (input) => { @@ -63,7 +108,11 @@ describe('DirectoryReviewCentre', () => { }); const review = new DirectoryReviewController(createAPIClient(fetchFn)); const relationships = new RelationshipReviewController(createAPIClient(fetchFn)); - render(DirectoryReviewCentre, { controller: review, relationshipController: relationships }); + render(DirectoryReviewCentre, { + controller: review, + relationshipController: relationships, + client: review.apiClient + }); await fireEvent.click(screen.getByRole('radio', { name: 'Imported relationships' })); @@ -100,6 +149,28 @@ describe('DirectoryReviewCentre', () => { expect(requests.filter((request) => new URL(request.url).pathname.endsWith('/accept'))).toHaveLength(1); }); + it('marks each hidden review heading as the focus target of its section', async () => { + const fetchFn = vi.fn(async () => page([candidate(17)])); + const apiClient = createAPIClient(fetchFn); + const controller = new DirectoryReviewController(apiClient); + await controller.loadIdentityPage(); + render(DirectoryReviewCentre, { + controller, + relationshipController: new RelationshipReviewController(apiClient), + factController: new FactLedgerController(apiClient), + client: apiClient, + directoryPersonID: null + }); + + for (const name of ['Identity matches', 'Facts', 'Imported relationships']) { + if (name !== 'Identity matches') await fireEvent.click(screen.getByRole('radio', { name })); + const heading = await screen.findByRole('heading', { name, level: 2 }); + expect(heading.className).toContain('review-heading'); + expect(heading.getAttribute('tabindex')).toBe('-1'); + expect(heading.closest('[data-review-section]')).not.toBeNull(); + } + }); + it('changes identity state through the controller and commits the URL filter', async () => { const requests: Request[] = []; const commit = vi.fn(); @@ -114,8 +185,10 @@ describe('DirectoryReviewCentre', () => { renderReview(controller); expect(screen.getByRole('radiogroup', { name: 'Review type' })).toBeDefined(); - expect(screen.getByRole('radiogroup', { name: 'Identity review state' })).toBeDefined(); - await fireEvent.click(screen.getByRole('radio', { name: 'Conflict' })); + const show = screen.getByRole('combobox', { name: /^Identity review state/ }); + expect(show.textContent).toContain('Show: Candidate'); + await fireEvent.click(show); + await fireEvent.click(screen.getByRole('option', { name: 'Conflict' })); await screen.findByRole('heading', { name: 'Identity match 22' }); expect(controller.reviewKind).toBe('identity'); @@ -146,15 +219,16 @@ describe('DirectoryReviewCentre', () => { controller, relationshipController: new RelationshipReviewController(apiClient), factController, + client: apiClient, directoryPersonID: null }); if (mode === 'selection') { - await fireEvent.click(screen.getByRole('radio', { name: 'Fact review' })); + await fireEvent.click(screen.getByRole('radio', { name: 'Facts' })); } - expect(screen.getByRole('region', { name: 'Fact review' })).toBeDefined(); - expect(screen.getByText('Choose a person in Directory to inspect their fact ledger')).toBeDefined(); + expect(screen.getByRole('region', { name: 'Facts' })).toBeDefined(); + expect(screen.getByText('Choose a person to see the facts recorded about them.')).toBeDefined(); expect(screen.queryByRole('button', { name: /accept|reject|unsure|link identities|keep separate/i })).toBeNull(); expect(fetchFn).not.toHaveBeenCalled(); if (mode === 'selection') expect(commit).toHaveBeenCalledWith({ reviewKind: 'fact' }); @@ -246,7 +320,7 @@ describe('DirectoryReviewCentre', () => { { name: 'identity review to fact review', target: { reviewKind: 'fact' as const, identityState: 'candidate' as const }, - focusHeading: 'Fact review' + focusHeading: 'Facts' }, { name: 'candidate review to conflict review', @@ -279,6 +353,7 @@ describe('DirectoryReviewCentre', () => { controller, relationshipController: new RelationshipReviewController(apiClient), factController, + client: apiClient, directoryPersonID: null }); @@ -370,7 +445,7 @@ describe('DirectoryReviewCentre', () => { await waitFor(() => expect(screen.queryByRole('dialog', { name: 'Link identities' })).toBeNull()); const row = screen.getByRole('article', { name: 'Identity match 17' }); - expect(row.textContent).toContain('accepted'); + expect(row.textContent).toContain('Accepted'); expect(screen.getByRole('status').textContent).toContain('Identity match accepted.'); expect(screen.getByRole('alert').textContent).toContain('Reload failed'); await waitFor(() => expect(document.activeElement).toBe(row)); diff --git a/web/src/lib/components/directory/DirectoryReviewWorkspace.svelte b/web/src/lib/components/directory/DirectoryReviewWorkspace.svelte index e6878f7e0..ab49da34c 100644 --- a/web/src/lib/components/directory/DirectoryReviewWorkspace.svelte +++ b/web/src/lib/components/directory/DirectoryReviewWorkspace.svelte @@ -1,4 +1,5 @@
-
-

Directory

Durable people and their recorded contact context.

- {#if promotionParticipantID !== undefined} -
-
- editTextFilter('directoryQuery', value)} /> - selectFilter({ directoryContactState: value })} /> - editTextFilter('directoryCategory', value)} /> - editTextFilter('directoryOrganization', value)} /> - selectFilter({ directoryPrimaryChannel: value })} /> - editTextFilter('directoryLastContactAfter', value)} /> - editTextFilter('directoryLastContactBefore', value)} /> - selectFilter({ directorySort: value as DirectoryURLState['directorySort'] })} />
{#if controller.promotionResult && !controller.promotionResult.ok} {#if controller.selectedPersonID !== null && narrow} void closeDetail()}> - {#if controller.detailLoading}

Loading person detail…

{:else if controller.detail} void controller.selectPerson(personID)} onSplitCommitted={(context) => controller.reconcilePersonSplit(context)} {onOpenCardDAVConflict} {onOpenCardDAVSettings} {onAnnounce} />{/if} + {#if controller.detailLoading}

Loading person detail…

{:else if controller.detail} void controller.selectPerson(personID)} onSplitCommitted={(context) => controller.reconcilePersonSplit(context)} {onOpenCardDAVConflict} {onOpenCardDAVSettings} {onAnnounce} {onOpenMeeting} {onOpenRelationship} {onReviewFacts} />{/if}
{/if}
diff --git a/web/src/lib/components/directory/DirectoryWorkspace.test.ts b/web/src/lib/components/directory/DirectoryWorkspace.test.ts index 817c98276..4a1ff0146 100644 --- a/web/src/lib/components/directory/DirectoryWorkspace.test.ts +++ b/web/src/lib/components/directory/DirectoryWorkspace.test.ts @@ -4,6 +4,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; import { createAPIClient } from '../../api/client'; import { DirectoryController } from '../../directory/controller.svelte'; import type { DirectoryURLState } from '../../directory/models'; +import { meetingFixtureResponse } from '../../meetings/fixtures.test-support'; import { chooseSelectOption } from '../../../test/kit-ui'; import DirectoryWorkspace from './DirectoryWorkspace.svelte'; @@ -53,8 +54,7 @@ describe('DirectoryWorkspace', () => { await vi.advanceTimersByTimeAsync(250); expect(commits).toEqual([[{ - directoryQuery: 'alice', directoryCategory: '', directoryOrganization: '', - directoryLastContactAfter: '', directoryLastContactBefore: '' + directoryQuery: 'alice', directoryCategory: '', directoryOrganization: '' }, 'replace']]); expect(requests).toHaveLength(2); expect(new URL(requests[1]!.url).searchParams.get('q')).toBe('alice'); @@ -74,6 +74,7 @@ describe('DirectoryWorkspace', () => { render(DirectoryWorkspace, { client, controller, state }); await vi.advanceTimersByTimeAsync(0); + await fireEvent.click(screen.getByRole('button', { name: 'Filters' })); await fireEvent.input(screen.getByRole('textbox', { name: 'Organization filter' }), { target: { value: 'Example Org' } }); await chooseSelectOption(screen.getByRole('combobox', { name: /^Contact state:/ }), 'Active'); await vi.advanceTimersByTimeAsync(0); @@ -97,6 +98,7 @@ describe('DirectoryWorkspace', () => { render(DirectoryWorkspace, { client, controller, state }); await screen.findByRole('row', { name: /Synthetic Person/ }); + await fireEvent.click(screen.getByRole('button', { name: 'Filters' })); await chooseSelectOption(screen.getByRole('combobox', { name: /^Contact state:/ }), 'Active'); await waitFor(() => expect(resolveFiltered).toBeDefined()); expect(screen.getByRole('row', { name: /Synthetic Person/ })).toBeDefined(); @@ -109,27 +111,132 @@ describe('DirectoryWorkspace', () => { expect(screen.queryByRole('status')).toBeNull(); }); - it('offers free-text category and organization filters when no server facet catalog exists', () => { + it('offers free-text category and organization filters when no server facet catalog exists', async () => { const client = createAPIClient(vi.fn(async () => directoryResponse())); const controller = new DirectoryController(client); render(DirectoryWorkspace, { client, controller, state }); + expect(screen.queryByRole('textbox', { name: 'Category filter' })).toBeNull(); + await fireEvent.click(screen.getByRole('button', { name: 'Filters' })); expect(screen.getByRole('textbox', { name: 'Category filter' })).toBeDefined(); expect(screen.getByRole('textbox', { name: 'Organization filter' })).toBeDefined(); }); - it('offers last-contact range and ordering controls', () => { + it('offers last-contact range and ordering controls', async () => { const client = createAPIClient(vi.fn(async () => directoryResponse())); const controller = new DirectoryController(client); render(DirectoryWorkspace, { client, controller, state }); + await fireEvent.click(screen.getByRole('button', { name: 'Filters' })); - expect(screen.getByRole('textbox', { name: 'Last contacted after' })).toBeDefined(); - expect(screen.getByRole('textbox', { name: 'Last contacted before' })).toBeDefined(); + expect((screen.getByLabelText('Last contacted after') as HTMLInputElement).type).toBe('date'); + expect((screen.getByLabelText('Last contacted before') as HTMLInputElement).type).toBe('date'); expect(screen.getByRole('combobox', { name: /^Directory order:/ })).toBeDefined(); }); + it('sets and clears each last-contacted boundary on its own', async () => { + const requests: Request[] = []; + const client = createAPIClient(vi.fn(async (input) => { + const request = input instanceof Request ? input : new Request(input); + requests.push(request); + return directoryResponse(); + })); + const controller = new DirectoryController(client); + render(DirectoryWorkspace, { client, controller, state }); + await fireEvent.click(screen.getByRole('button', { name: 'Filters' })); + + await fireEvent.change(screen.getByLabelText('Last contacted after'), { target: { value: '2024-01-05' } }); + expect(controller.lastContactAfter).toBe('2024-01-05'); + expect(controller.lastContactBefore).toBe(''); + expect(screen.getByText('Last contacted after Jan 5, 2024')).toBeDefined(); + await waitFor(() => { + const params = new URL(requests.at(-1)!.url).searchParams; + expect(params.get('last_contact_after')).toBe('2024-01-05T00:00:00Z'); + expect(params.has('last_contact_before')).toBe(false); + }); + + await fireEvent.click(screen.getByRole('button', { name: 'Remove Last contacted after Jan 5, 2024 filter' })); + expect(controller.lastContactAfter).toBe(''); + expect(screen.queryByText('Last contacted after Jan 5, 2024')).toBeNull(); + }); + + it('clears a boundary from the clear button beside its date field', async () => { + const client = createAPIClient(vi.fn(async () => directoryResponse())); + const controller = new DirectoryController(client); + render(DirectoryWorkspace, { client, controller, state: { ...state, directoryLastContactBefore: '2024-03-01' } }); + await fireEvent.click(screen.getByRole('button', { name: 'Filters' })); + + expect(screen.getByText('Last contacted before Mar 1, 2024')).toBeDefined(); + await fireEvent.click(screen.getByRole('button', { name: 'Clear last contacted before' })); + expect(controller.lastContactBefore).toBe(''); + expect((screen.getByLabelText('Last contacted before') as HTMLInputElement).value).toBe(''); + }); + + it('removes only the chip that was cleared', async () => { + const client = createAPIClient(vi.fn(async () => directoryResponse())); + const controller = new DirectoryController(client); + render(DirectoryWorkspace, { + client, controller, + state: { ...state, directoryContactState: 'active', directoryOrganization: 'Example Co', directoryPrimaryChannel: 'email' } + }); + + expect(screen.getByText('Primary channel: Email')).toBeDefined(); + await fireEvent.click(screen.getByRole('button', { name: 'Remove Contact state: Active filter' })); + expect(controller.contactState).toBe(''); + expect(controller.organization).toBe('Example Co'); + expect(controller.primaryChannel).toBe('email'); + expect(screen.getByText('Organization: Example Co')).toBeDefined(); + expect(screen.queryByText('Contact state: Active')).toBeNull(); + }); + + it('names primary channels and sorts with a visible label', async () => { + const client = createAPIClient(vi.fn(async () => directoryResponse())); + const controller = new DirectoryController(client); + render(DirectoryWorkspace, { client, controller, state }); + + expect(screen.getByRole('combobox', { name: /^Directory order:/ }).textContent).toContain('Sort: Name'); + await fireEvent.click(screen.getByRole('button', { name: 'Filters' })); + await fireEvent.click(screen.getByRole('combobox', { name: /^Primary channel/ })); + expect(screen.getAllByRole('option').map((option) => option.textContent?.trim())).toEqual([ + 'All channels', 'Email', 'Chat', 'Meeting', 'Other' + ]); + }); + + it.each([ + ['email', 'Email'], ['chat', 'Chat'], ['meeting', 'Meeting'], ['other', 'Other'] + ])('filters by the %s activity channel and clears it', async (channel, label) => { + const requests: Request[] = []; + const client = createAPIClient(vi.fn(async (input) => { + const request = input instanceof Request ? input : new Request(input); + requests.push(request); + return directoryResponse(); + })); + const controller = new DirectoryController(client); + render(DirectoryWorkspace, { client, controller, state }); + await screen.findByRole('row', { name: /Synthetic Person/ }); + await fireEvent.click(screen.getByRole('button', { name: 'Filters' })); + + await chooseSelectOption(screen.getByRole('combobox', { name: /^Primary channel/ }), label); + await waitFor(() => expect(new URL(requests.at(-1)!.url).searchParams.get('primary_channel')).toBe(channel)); + expect(screen.getByText(`Primary channel: ${label}`)).toBeDefined(); + + await fireEvent.click(screen.getByRole('button', { name: `Remove Primary channel: ${label} filter` })); + await waitFor(() => expect(new URL(requests.at(-1)!.url).searchParams.has('primary_channel')).toBe(false)); + expect(screen.queryByText(`Primary channel: ${label}`)).toBeNull(); + }); + + it('counts people and marks more pages with a plus', async () => { + const client = createAPIClient(vi.fn(async () => Response.json({ + people: [{ id: 7, revision: 2, display_name: 'Synthetic Person', contact_state: 'active', categories: [], organizations: [] }], + next_cursor: 'more' + }))); + const controller = new DirectoryController(client); + render(DirectoryWorkspace, { client, controller, state }); + + expect(await screen.findByText('1+ people')).toBeDefined(); + }); + it('offers only contact states accepted by the Directory handler contract', async () => { const requests: Request[] = []; const client = createAPIClient(vi.fn(async (input) => { @@ -140,6 +247,7 @@ describe('DirectoryWorkspace', () => { const controller = new DirectoryController(client); render(DirectoryWorkspace, { client, controller, state }); + await fireEvent.click(screen.getByRole('button', { name: 'Filters' })); const contactState = screen.getByRole('combobox', { name: /^Contact state:/ }); await chooseSelectOption(contactState, 'Active'); await waitFor(() => expect(new URL(requests.at(-1)!.url).searchParams.get('contact_state')).toBe('active')); @@ -153,9 +261,13 @@ describe('DirectoryWorkspace', () => { const commits: Array> = []; const fetchFn = vi.fn(async (input) => { const request = input instanceof Request ? input : new Request(input); - if (pathOf(request) === '/api/v1/people') return Response.json({ id: 42, revision: 1 }, { status: 201 }); - if (pathOf(request) === '/api/v1/people/directory') return directoryResponse(); - if (pathOf(request).endsWith('/files/search')) return Response.json({ files: [], total_count: 0, cache_revision: 'synthetic', search_provenance: {} }); + const path = pathOf(request); + if (path === '/api/v1/people') return Response.json({ id: 42, revision: 1 }, { status: 201 }); + if (path === '/api/v1/people/directory') return directoryResponse(); + if (path.endsWith('/files/search')) return Response.json({ files: [], total_count: 0, cache_revision: 'synthetic', search_provenance: {} }); + const meetingResponse = meetingFixtureResponse(path); + if (meetingResponse) return meetingResponse; + if (path.endsWith('/contact-state')) return Response.json({ person_id: 42, cadence_status: 'active', interaction_count: 0, computed_at: '2026-01-01T00:00:00Z', stale: false }); return Response.json({ id: 42, revision: 1, participant_ids: [], vcard_uid: '', created_at: '2026-01-01T00:00:00Z', updated_at: '2026-01-01T00:00:00Z' }); }); const client = createAPIClient(fetchFn); @@ -167,6 +279,8 @@ describe('DirectoryWorkspace', () => { await waitFor(() => expect(commits).toContainEqual({ directoryPersonID: 42 })); await waitFor(() => expect(controller.selectedPersonID).toBe(42)); + expect(await screen.findByText('4 meetings')).toBeDefined(); + expect(await screen.findByText('0 matching action items')).toBeDefined(); }); it('keeps loaded rows visible when loading another page fails and retries that page', async () => { @@ -266,6 +380,7 @@ describe('DirectoryWorkspace', () => { render(DirectoryWorkspace, { client, controller, state: selectedState }); + await fireEvent.click(await screen.findByRole('tab', { name: 'Profile' })); expect(await screen.findByRole('heading', { name: 'Structured profile' })).toBeDefined(); expect(screen.getByRole('button', { name: 'Add name' })).toHaveProperty('disabled', false); expect(controller.profile).not.toBeNull(); @@ -302,7 +417,8 @@ describe('DirectoryWorkspace', () => { const controller = new DirectoryController(createAPIClient(fetchFn), (patch) => commits.push(patch)); render(DirectoryWorkspace, { client: createAPIClient(fetchFn), controller, state: selectedState }); - await fireEvent.click(await screen.findByRole('button', { name: 'Delete person' })); + await fireEvent.click(await screen.findByRole('button', { name: 'More actions' })); + await fireEvent.click(screen.getByRole('menuitem', { name: 'Delete person' })); await fireEvent.click(screen.getByRole('button', { name: 'Confirm delete person' })); await waitFor(() => expect(controller.selectedPersonID).toBeNull()); @@ -357,6 +473,7 @@ describe('DirectoryWorkspace', () => { const controller = new DirectoryController(client); render(DirectoryWorkspace, { client, controller, state: selectedState }); + await fireEvent.click(await screen.findByRole('tab', { name: 'Profile' })); await fireEvent.click(await screen.findByRole('button', { name: 'Add category' })); await fireEvent.input(screen.getByRole('textbox', { name: 'Category' }), { target: { value: 'VIP' } }); await fireEvent.click(screen.getByRole('button', { name: 'Save category' })); @@ -414,6 +531,7 @@ describe('DirectoryWorkspace', () => { const controller = new DirectoryController(client); render(DirectoryWorkspace, { client, controller, state: selectedState }); + await fireEvent.click(await screen.findByRole('tab', { name: 'Profile' })); await fireEvent.click(await screen.findByRole('button', { name: 'Add category' })); await fireEvent.input(screen.getByRole('textbox', { name: 'Category' }), { target: { value: 'Straße' } }); await fireEvent.click(screen.getByRole('button', { name: 'Save category' })); @@ -483,6 +601,7 @@ describe('DirectoryWorkspace', () => { render(DirectoryWorkspace, { client, controller, state: selectedState }); const row = await screen.findByRole('row', { name: /Synthetic Person/ }); + await fireEvent.click(await screen.findByRole('tab', { name: 'Profile' })); await fireEvent.click(await screen.findByRole('button', { name: 'Add category' })); await fireEvent.input(screen.getByRole('textbox', { name: 'Category' }), { target: { value: 'Straße' } }); await fireEvent.click(screen.getByRole('button', { name: 'Save category' })); @@ -564,6 +683,7 @@ describe('DirectoryWorkspace', () => { const controller = new DirectoryController(client); render(DirectoryWorkspace, { client, controller, state: selectedState }); + await fireEvent.click(await screen.findByRole('tab', { name: 'Profile' })); await fireEvent.click(await screen.findByRole('button', { name: 'Edit Primary channel value 1' })); await fireEvent.change(screen.getByRole('combobox', { name: 'Primary channel' }), { target: { value: 'chat' } }); await fireEvent.click(screen.getByRole('button', { name: 'Save attribute' })); @@ -572,13 +692,13 @@ describe('DirectoryWorkspace', () => { await waitFor(() => expect( controller.profile?.attributes?.attributes?.[0]?.current?.[0]?.value ).toEqual({ type: 'text', text: 'chat' })); - expect(row.textContent).toContain('email · active'); + expect(row.textContent).toContain('Email · Active'); expect(requests.filter((request) => pathOf(request) === '/api/v1/people/directory')).toHaveLength(1); await fireEvent.click(screen.getByRole('button', { name: 'Close Primary channel value 1' })); await fireEvent.click(screen.getByRole('button', { name: 'Confirm close attribute' })); await waitFor(() => expect(controller.profile?.attributes?.attributes?.[0]?.current).toEqual([])); - expect(row.textContent).toContain('email · active'); + expect(row.textContent).toContain('Email · Active'); expect(requests.filter((request) => pathOf(request) === '/api/v1/people/directory')).toHaveLength(1); }); @@ -625,6 +745,7 @@ describe('DirectoryWorkspace', () => { publicationSignal = request.signal; return new Promise((resolve) => { resolvePublication = resolve; }); } + if (path.endsWith('/contact-state')) return Response.json({ person_id: 7, cadence_status: 'active', interaction_count: 0, computed_at: '2026-01-01T00:00:00Z', stale: false }); return Response.json({ id: 7, revision: 1, participant_ids: [], vcard_uid: '', created_at: '2026-01-01T00:00:00Z', updated_at: '2026-01-01T00:00:00Z' @@ -636,6 +757,7 @@ describe('DirectoryWorkspace', () => { await fireEvent.click(await screen.findByRole('row', { name: /Synthetic Person/ })); expect(await screen.findByRole('dialog', { name: 'Person detail' })).toBeDefined(); + await fireEvent.click(await screen.findByRole('tab', { name: 'Maintenance' })); await waitFor(() => expect(publicationSignal).toBeDefined()); await fireEvent.click(screen.getByRole('button', { name: 'Close' })); expect(publicationSignal?.aborted).toBe(true); diff --git a/web/src/lib/components/directory/EmploymentEditor.svelte b/web/src/lib/components/directory/EmploymentEditor.svelte index 1faaddd67..2c519244d 100644 --- a/web/src/lib/components/directory/EmploymentEditor.svelte +++ b/web/src/lib/components/directory/EmploymentEditor.svelte @@ -2,6 +2,7 @@ import { appShortcuts, Button, Checkbox, Modal, SelectDropdown, TextInput } from '@kenn-io/kit-ui'; import { onDestroy, onMount, untrack } from 'svelte'; + import { intervalDateError } from '../../directory/dates'; import type { DirectoryEntityController } from '../../directory/entity-controller.svelte'; import type { Employment, EmploymentBody, Organization } from '../../directory/models'; import EmploymentCurrentData from './EmploymentCurrentData.svelte'; @@ -28,6 +29,9 @@ let endDate = $state(partialDate(initialEmployment?.end_date)); let isCurrent = $state(initialEmployment?.is_current ?? true); let isPrimary = $state(initialEmployment?.is_primary ?? false); + const uid = $props.id(); + const startDateError = $derived(intervalDateError(startDate)); + const endDateError = $derived(intervalDateError(endDate)); let loading = $state(!!initialEmployment); let submitting = $state(false); let message = $state(''); @@ -179,15 +183,18 @@ {#if loading}

Loading current employment…

{:else if initialAction === 'end'} - + + {#if endDateError}

{endDateError}

{/if} {:else} - - + + {#if startDateError}

{startDateError}

{/if} + + {#if endDateError}

{endDateError}

{/if}
{ isCurrent = checked; }} disabled={submitting} /> { isPrimary = checked; }} disabled={submitting} />
{/if} @@ -204,7 +211,7 @@ - - - - + {#each SECTIONS as section (section.id)} + + {/each} {#each Object.entries(bundle.errors) as [section, message]} {/each} - {#if activeTab === 'media'} -
+
+ {#if activeTab === 'overview'} + + + { + await selectTab('profile'); + await tick(); + const section = document.getElementById('person-attributes'); + section?.scrollIntoView({ block: 'start', behavior: 'smooth' }); + section?.focus({ preventScroll: true }); + } : undefined} + /> + {#if bundle.contactState} +

Contact state

{contactStateText(bundle.contactState)}

+ {/if} + {#if bundle.activity} +

Activity

{bundle.activity.total_count} recorded days

+ {/if} + {#if entityController?.employments.length} +

Organizations and employment

    {#each entityController.employments as employment}
  • {employment.title ?? employment.role ?? 'Employment'} · {employmentOrganization(employment.id) ?? `Organization ${employment.organization_id}`}{#if employment.is_current} Current{/if}
  • {/each}
+ {/if} + {#if entityController?.relationships.length} +

Connections

    {#each entityController.relationships as view}
  • {view.counterpart_display_name?.trim() || view.counterpart_vcard_uid || `Person ${view.counterpart_person_id}`} · {view.counterpart_label}
  • {/each}
+ {/if} + {#if bundle.person?.id === personID} + a - b)])} + {onOpenMeeting} /> + {/if} + {:else if activeTab === 'profile'} + {#if profileController} + + {:else} + {#if profile?.names?.length} +

Names

    {#each profile.names as name}
  • {nameText(name)} {name.name_kind}
  • {/each}
+ {/if} + {#if groupedContacts().length} +

Contact observations

{#each groupedContacts() as [service, points]}

{service}

    {#each points as point}
  • {point.original_value} {point.address_kind}
  • {/each}
{/each}
+ {/if} + {#if profile?.addresses?.length} +

Addresses

    {#each profile.addresses as address}
  • {address.original_value} {address.address_kind}
  • {/each}
+ {/if} + {#if profile?.dates?.length} +

Dates

    {#each profile.dates as date}
  • {date.label ?? date.date_kind}: {date.date_text ?? valueText(date.date)}
  • {/each}
+ {/if} + {#if profile?.categories?.length} +

Categories

    {#each profile.categories as category}
  • {category.original_value}
  • {/each}
+ {/if} + {/if} + {#if profileController && profileController.attributes} + + {/if} + {:else if activeTab === 'organizations'} + {#if entityController} + {:else}

Organizations

Organizations are unavailable for this selection.

{/if} + {:else if activeTab === 'connections'} + {#if entityController} + {:else}

Connections

Connections are unavailable for this selection.

{/if} + {:else if activeTab === 'network'} + {#if entityController} + {:else}

Network

The curated network is unavailable for this selection.

{/if} + {:else if activeTab === 'media'} (fileSort = value)} + onFilenameQueryChange={debouncedFilenameQuery} + onMIMEFamiliesChange={(value) => (fileMIMEFamilies = value)} embedded /> -
- {:else if activeTab === 'network'} -
- {#if entityController} - {:else}

Network

The curated network is unavailable for this selection.

{/if} -
- {:else if activeTab === 'relationships'} -
- {#if entityController} - {:else}

Relationships

Relationships are unavailable for this selection.

{/if} -
- {:else if activeTab === 'organizations'} -
- {#if entityController} - {:else}

Organizations

Organizations are unavailable for this selection.

{/if} -
- {:else} -
- {#if bundle.person || profile} -

{bundle.person?.display_name ?? profile?.person?.display_name ?? `Person ${personID}`}

- {/if} + {:else} - {#if profileController} - - {:else if profile?.names?.length} -

Names

    {#each profile.names as name}
  • {nameText(name)} {name.name_kind}
  • {/each}
- {/if} - {#if !profileController && groupedContacts().length} -

Contact observations

{#each groupedContacts() as [service, points]}

{service}

    {#each points as point}
  • {point.original_value} {point.address_kind}
  • {/each}
{/each}
- {/if} - {#if !profileController && profile?.addresses?.length} -

Addresses

    {#each profile.addresses as address}
  • {address.original_value} {address.address_kind}
  • {/each}
- {/if} - {#if !profileController && profile?.dates?.length} -

Dates

    {#each profile.dates as date}
  • {date.label ?? date.date_kind}: {date.date_text ?? valueText(date.date)}
  • {/each}
- {/if} - {#if !profileController && profile?.categories?.length} -

Categories

    {#each profile.categories as category}
  • {category.original_value}
  • {/each}
- {/if} - {#if profileController && profileController.attributes} - - {:else if bundle.attributes?.attributes?.length} -

Attributes

    {#each bundle.attributes.attributes as group}
  • {group.definition.label}{#if group.definition.is_sensitive} Sensitive: concealed{:else}: {group.current?.map((value) => valueText(value.value)).join(', ')}{/if}
  • {/each}
- {/if} - {#if entityController?.employments.length} -

Organizations and employment

    {#each entityController.employments as employment}
  • {employment.title ?? employment.role ?? 'Employment'} · {employmentOrganization(employment.id) ?? `Organization ${employment.organization_id}`}{#if employment.is_current} Current{/if}
  • {/each}
- {/if} - {#if entityController?.relationships.length} -

Relationships

    {#each entityController.relationships as view}
  • {view.counterpart_display_name?.trim() || view.counterpart_vcard_uid || `Person ${view.counterpart_person_id}`} · {view.counterpart_label}
  • {/each}
- {/if} - {#if bundle.contactState} -

Contact state

{bundle.contactState.cadence_status} · {bundle.contactState.interaction_count} interactions{#if bundle.contactState.last_contact_at} · last contact {bundle.contactState.last_contact_at}{/if}

- {/if} - - {#if bundle.activity} -

Activity

{bundle.activity.total_count} recorded days

- {/if} -
- {/if} + {/if} +
diff --git a/web/src/lib/components/directory/PersonDetail.test.ts b/web/src/lib/components/directory/PersonDetail.test.ts index 1456300a9..f7e5fa33f 100644 --- a/web/src/lib/components/directory/PersonDetail.test.ts +++ b/web/src/lib/components/directory/PersonDetail.test.ts @@ -1,17 +1,54 @@ import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; import { describe, expect, it, vi } from 'vitest'; +import { meetingFixtureResponse } from '../../meetings/fixtures.test-support'; import { createAPIClient } from '../../api/client'; +import type { AttributeDefinition as GeneratedAttributeDefinition } from '../../api/generated/models'; import { DirectoryEntityController } from '../../directory/entity-controller.svelte'; +import { DirectoryProfileController } from '../../directory/profile-controller.svelte'; import type { DirectoryReadBundle } from '../../directory/models'; import PersonDetail from './PersonDetail.svelte'; +const when = '2026-01-01T00:00:00Z'; + +function nicknameDefinition(): GeneratedAttributeDefinition { + return { + id: 1, slug: 'nickname', label: 'Nickname', value_type: 'text', field_type: 'text', + api_mutable: true, cardinality: 'single', display_order: 0, history_exempt: false, + is_sensitive: false, is_active: true, is_audited: true, is_deletable: true, is_required: false, + is_searchable: false, object_type: 'person', ownership: 'user', revision: 1, + ui_creatable: true, ui_editable: true, universal_id: 'synthetic-nickname', + created_at: when, updated_at: when + }; +} + +function nicknameCurrent() { + return [{ id: 1, person_id: 7, definition_id: 1, definition_slug: 'nickname', ordinal: 0, source: 'user', active_from: when, created_at: when, value: { type: 'text', text: 'Synthetic nickname' } }]; +} + +// The Overview panel's self-loading cards must never decide one of these +// tests; answer their mount reads quietly and deterministically. +function quietOverviewFetch() { + return vi.fn(async (input) => { + const request = input instanceof Request ? input : new Request(input); + const path = new URL(request.url).pathname; + const overview = overviewCardResponse(request); + if (overview) return overview; + if (path === '/api/v1/carddav/publications/7') { + return Response.json({ error: 'carddav_unavailable', message: 'not rendered' }, { status: 503 }); + } + return Response.json({ merges: [], limit: 100, offset: 0 }); + }); +} + // The Overview panel mounts three self-loading cards. This answers the reads // they make on mount so each test only has to state what it is actually about. // An unenrolled brief enrollment is the quiet default: the brief card then // makes no further request. function overviewCardResponse(request: Request): Response | undefined { const path = new URL(request.url).pathname; + const meetingResponse = meetingFixtureResponse(path); + if (meetingResponse) return meetingResponse; if (path === '/api/v1/people/7/tracking') { return Response.json({ person_id: 7, tracked: false, tracked_at: null }); } @@ -25,8 +62,68 @@ function overviewCardResponse(request: Request): Response | undefined { return undefined; } +const personID = 7; +const person = { + id: personID, revision: 1, display_name: 'Synthetic Person', participant_ids: [11], vcard_uid: '', + created_at: when, updated_at: when +}; + +function renderPersonDetail( + bundle: Partial = {}, + props: Partial<{ onOpenRelationship: (participantID: number) => void; onReviewFacts: (personID: number) => void }> = {} +) { + return render(PersonDetail, { + client: createAPIClient(quietOverviewFetch()), personID, + bundle: { person, etags: {}, errors: {}, ...bundle }, ...props + }); +} + describe('PersonDetail', () => { - it('renders available read sections, marks sensitive attributes, and keeps Media & Files person-scoped', async () => { + it('lists seven person sections in order', async () => { + renderPersonDetail(); + expect(screen.getAllByRole('tab').map((tab) => tab.textContent?.trim())).toEqual( + ['Overview', 'Profile', 'Organizations', 'Connections', 'Network', 'Media & files', 'Maintenance']); + await fireEvent.click(screen.getByRole('tab', { name: 'Maintenance' })); + expect(await screen.findByRole('heading', { name: 'Merge history' })).toBeTruthy(); + }); + + it('names the person in a header above the sections', () => { + renderPersonDetail(); + const heading = screen.getByRole('heading', { level: 2, name: 'Synthetic Person' }); + const tablist = screen.getByRole('tablist', { name: 'Person detail sections' }); + expect(heading.compareDocumentPosition(tablist) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); + expect(screen.getByRole('tabpanel', { name: 'Overview' }).contains(heading)).toBe(false); + }); + + it('opens the relationship for the lowest participant ID and reviews facts', async () => { + const onOpenRelationship = vi.fn(); + const onReviewFacts = vi.fn(); + renderPersonDetail({ person: { ...person, participant_ids: [42, 7, 19] } }, { onOpenRelationship, onReviewFacts }); + await fireEvent.click(screen.getByRole('button', { name: 'Open relationship' })); + expect(onOpenRelationship).toHaveBeenCalledWith(7); + await fireEvent.click(screen.getByRole('button', { name: 'Review facts' })); + expect(onReviewFacts).toHaveBeenCalledWith(personID); + }); + + it('hides Open relationship when the person has no participants', () => { + renderPersonDetail({ person: { ...person, participant_ids: [] } }, { onOpenRelationship: vi.fn() }); + expect(screen.queryByRole('button', { name: 'Open relationship' })).toBeNull(); + }); + + it('offers record actions once, from the header menu, when the profile controller is available', async () => { + const client = createAPIClient(quietOverviewFetch()); + const bundle = { person, etags: { person: '"person-7-r1"' }, errors: {} } satisfies DirectoryReadBundle; + const profileController = new DirectoryProfileController(client, personID, bundle); + render(PersonDetail, { client, personID, bundle, profileController }); + + expect(screen.getAllByRole('button', { name: 'More actions' })).toHaveLength(1); + expect(screen.queryByRole('button', { name: 'Rename person' })).toBeNull(); + await fireEvent.click(screen.getByRole('button', { name: 'More actions' })); + await fireEvent.click(screen.getByRole('menuitem', { name: 'Rename person' })); + expect(screen.getByRole('group', { name: 'Rename person' })).toBeDefined(); + }); + + it('renders available read sections, marks sensitive attributes, and keeps Media & files person-scoped', async () => { const requestPaths: string[] = []; const client = createAPIClient(vi.fn(async (input) => { const request = input instanceof Request ? input : new Request(input); @@ -68,9 +165,7 @@ describe('PersonDetail', () => { void entityController.load(); render(PersonDetail, { client, bundle, personID: 7, entityController }); - expect(screen.getByText('Names')).toBeDefined(); - expect(screen.getByText('person@example.test')).toBeDefined(); - expect(screen.getByText('Sensitive')).toBeDefined(); + expect(document.querySelector('.attribute-summary .sensitive')?.textContent).toBe('concealed'); expect(document.body.innerHTML).not.toContain('Synthetic value'); expect(await screen.findByText(/Example Org/)).toBeDefined(); expect(screen.getByText('Synthetic Child · child')).toBeDefined(); @@ -80,13 +175,99 @@ describe('PersonDetail', () => { expect(requestPaths.filter((path) => path === '/api/v1/people/7/relationships')).toHaveLength(1); expect(requestPaths).not.toContain('/api/v1/people/7/network'); expect(screen.getByText('Activity')).toBeDefined(); + expect(screen.getByRole('heading', { name: 'Connections' })).toBeDefined(); expect(screen.queryByText('Provenance and history')).toBeNull(); - await fireEvent.click(screen.getByRole('tab', { name: 'Media & Files' })); + await fireEvent.click(screen.getByRole('tab', { name: 'Profile' })); + expect(screen.getByText('Names')).toBeDefined(); + expect(screen.getByText('person@example.test')).toBeDefined(); + await fireEvent.click(screen.getByRole('tab', { name: 'Media & files' })); await waitFor(() => expect(requestPaths).toContain('/api/v1/people/7/files/search')); expect(requestPaths).not.toContain('/api/v1/participants/7/files/search'); expect(requestPaths).not.toContain('/api/v1/files/search'); }); + it('applies Media & files filename, type, and sort choices to the person files request', async () => { + const filesBodies: Record[] = []; + const fetchFn = vi.fn(async (input) => { + const request = input instanceof Request ? input : new Request(input); + const overview = overviewCardResponse(request); + if (overview) return overview; + if (new URL(request.url).pathname === '/api/v1/people/7/files/search') { + filesBodies.push((await request.clone().json()) as Record); + return Response.json({ files: [], total_count: 0, cache_revision: 'synthetic', search_provenance: {} }); + } + return Response.json({ merges: [], limit: 100, offset: 0 }); + }); + const bundle = { etags: {}, errors: {} } satisfies DirectoryReadBundle; + render(PersonDetail, { client: createAPIClient(fetchFn), bundle, personID: 7 }); + + await fireEvent.click(screen.getByRole('tab', { name: 'Media & files' })); + await waitFor(() => expect(filesBodies).toHaveLength(1)); + + await fireEvent.input(screen.getByRole('searchbox', { name: 'Filter filename' }), { target: { value: 'invoice' } }); + await waitFor(() => expect(filesBodies.at(-1)).toMatchObject({ filename_query: 'invoice' })); + + await fireEvent.click(screen.getByRole('button', { name: 'Type' })); + // A person Files view starts with every file type included; unchecking PDFs leaves the rest. + const others = ['audio', 'text', 'document', 'archive', 'other']; + await fireEvent.click(screen.getByRole('button', { name: 'PDFs' })); + await waitFor(() => expect(filesBodies.at(-1)).toMatchObject({ filename_query: 'invoice', mime_families: others })); + + await fireEvent.click(screen.getByRole('button', { name: 'Sort by size' })); + await waitFor(() => expect(filesBodies.at(-1)).toMatchObject({ + filename_query: 'invoice', mime_families: others, sort: { field: 'size', direction: 'asc' } + })); + }); + + it('waits for filename typing to pause before requesting person files', async () => { + const filesBodies: Record[] = []; + const fetchFn = vi.fn(async (input) => { + const request = input instanceof Request ? input : new Request(input); + if (/^\/api\/v1\/people\/\d+\/files\/search$/.test(new URL(request.url).pathname)) { + filesBodies.push((await request.clone().json()) as Record); + return Response.json({ files: [], total_count: 0, cache_revision: 'synthetic', search_provenance: {} }); + } + return Response.json({ merges: [], limit: 100, offset: 0 }); + }); + const bundle = { etags: {}, errors: {} } satisfies DirectoryReadBundle; + const view = render(PersonDetail, { client: createAPIClient(fetchFn), bundle, personID: 7 }); + await fireEvent.click(screen.getByRole('tab', { name: 'Media & files' })); + await waitFor(() => expect(filesBodies).toHaveLength(1)); + + const search = screen.getByRole('searchbox', { name: 'Filter filename' }); + for (const value of ['i', 'in', 'inv']) await fireEvent.input(search, { target: { value } }); + await waitFor(() => expect(filesBodies.at(-1)).toMatchObject({ filename_query: 'inv' })); + expect(filesBodies.filter((body) => 'filename_query' in body)).toHaveLength(1); + + // A query still waiting when the person changes must not filter the next person's files. + await fireEvent.input(search, { target: { value: 'invoice' } }); + await view.rerender({ client: createAPIClient(fetchFn), bundle, personID: 8 }); + await new Promise((resolve) => setTimeout(resolve, 400)); + expect(filesBodies.at(-1)).not.toHaveProperty('filename_query'); + }); + + it('clears Media & files filters when the selected person changes', async () => { + const filesBodies: Record[] = []; + const fetchFn = vi.fn(async (input) => { + const request = input instanceof Request ? input : new Request(input); + if (/^\/api\/v1\/people\/\d+\/files\/search$/.test(new URL(request.url).pathname)) { + filesBodies.push((await request.clone().json()) as Record); + return Response.json({ files: [], total_count: 0, cache_revision: 'synthetic', search_provenance: {} }); + } + return Response.json({ merges: [], limit: 100, offset: 0 }); + }); + const bundle = { etags: {}, errors: {} } satisfies DirectoryReadBundle; + const view = render(PersonDetail, { client: createAPIClient(fetchFn), bundle, personID: 7 }); + await fireEvent.click(screen.getByRole('tab', { name: 'Media & files' })); + await fireEvent.input(await screen.findByRole('searchbox', { name: 'Filter filename' }), { target: { value: 'invoice' } }); + await waitFor(() => expect(filesBodies.at(-1)).toMatchObject({ filename_query: 'invoice' })); + + await view.rerender({ client: createAPIClient(fetchFn), bundle, personID: 8 }); + + await waitFor(() => expect(filesBodies.at(-1)).not.toHaveProperty('filename_query')); + expect((screen.getByRole('searchbox', { name: 'Filter filename' }) as HTMLInputElement).value).toBe(''); + }); + it('does not claim an organization name for an employment outside the primary projection', async () => { const client = createAPIClient(vi.fn(async (input) => { const request = input instanceof Request ? input : new Request(input); @@ -136,45 +317,38 @@ describe('PersonDetail', () => { const entityController = new DirectoryEntityController(client, 7); render(PersonDetail, { client, personID: 7, bundle: { etags: {}, errors: {} }, entityController }); - const overview = screen.getByRole('tab', { name: 'Overview' }); - const organizations = screen.getByRole('tab', { name: 'Organizations' }); - const relationships = screen.getByRole('tab', { name: 'Relationships' }); - const network = screen.getByRole('tab', { name: 'Network' }); - const media = screen.getByRole('tab', { name: 'Media & Files' }); - expect(overview.getAttribute('tabindex')).toBe('0'); - expect(screen.getAllByRole('tab')).toHaveLength(5); + const names = ['Overview', 'Profile', 'Organizations', 'Connections', 'Network', 'Media & files', 'Maintenance']; + const tabs = names.map((name) => screen.getByRole('tab', { name })); + const [overview, profileTab] = tabs; + const maintenance = tabs.at(-1)!; + expect(tabs.map((tab) => tab.getAttribute('tabindex'))).toEqual(['0', '-1', '-1', '-1', '-1', '-1', '-1']); + expect(tabs.map((tab) => tab.id)).toEqual( + ['overview', 'profile', 'organizations', 'connections', 'network', 'media', 'maintenance'].map((id) => `person-7-${id}-tab`)); + expect(overview!.getAttribute('aria-controls')).toBe(screen.getByRole('tabpanel', { name: 'Overview' }).id); + + overview!.focus(); + for (const [index, tab] of tabs.entries()) { + const next = tabs[(index + 1) % tabs.length]!; + await fireEvent.keyDown(tab, { key: 'ArrowRight' }); + expect(document.activeElement).toBe(next); + expect(next.getAttribute('aria-selected')).toBe('true'); + await waitFor(() => expect(next.getAttribute('aria-controls')).toBe(screen.getByRole('tabpanel', { name: names[(index + 1) % names.length] }).id)); + } + + await fireEvent.keyDown(overview!, { key: 'ArrowLeft' }); + expect(document.activeElement).toBe(maintenance); expect(await screen.findByRole('heading', { name: 'Merge history' })).toBeDefined(); - expect(organizations.getAttribute('tabindex')).toBe('-1'); - expect(relationships.getAttribute('tabindex')).toBe('-1'); - expect(network.getAttribute('tabindex')).toBe('-1'); - expect(media.getAttribute('tabindex')).toBe('-1'); - expect(overview.getAttribute('aria-controls')).toBe(screen.getByRole('tabpanel', { name: 'Overview' }).id); - - overview.focus(); - await fireEvent.keyDown(overview, { key: 'ArrowRight' }); - expect(document.activeElement).toBe(organizations); - expect(organizations.getAttribute('aria-selected')).toBe('true'); - await fireEvent.keyDown(organizations, { key: 'ArrowRight' }); - expect(document.activeElement).toBe(relationships); - await waitFor(() => expect(relationships.getAttribute('aria-controls')).toBe(screen.getByRole('tabpanel', { name: 'Relationships' }).id)); - await fireEvent.keyDown(relationships, { key: 'ArrowRight' }); - expect(document.activeElement).toBe(network); - await waitFor(() => expect(network.getAttribute('aria-controls')).toBe(screen.getByRole('tabpanel', { name: 'Network' }).id)); - await fireEvent.keyDown(network, { key: 'ArrowRight' }); - expect(document.activeElement).toBe(media); - expect(media.getAttribute('aria-controls')).toBe(screen.getByRole('tabpanel', { name: 'Media & Files' }).id); - - await fireEvent.keyDown(media, { key: 'ArrowRight' }); - expect(document.activeElement).toBe(overview); - await fireEvent.keyDown(overview, { key: 'ArrowLeft' }); - expect(document.activeElement).toBe(media); - await fireEvent.keyDown(media, { key: 'Home' }); + await fireEvent.keyDown(maintenance, { key: 'Home' }); expect(document.activeElement).toBe(overview); - expect(overview.getAttribute('aria-selected')).toBe('true'); + expect(overview!.getAttribute('aria-selected')).toBe('true'); - await fireEvent.keyDown(overview, { key: 'End' }); - expect(document.activeElement).toBe(media); + await fireEvent.keyDown(overview!, { key: 'End' }); + expect(document.activeElement).toBe(maintenance); + await fireEvent.keyDown(maintenance, { key: 'ArrowRight' }); + expect(document.activeElement).toBe(overview); + await fireEvent.keyDown(overview!, { key: 'ArrowRight' }); + expect(document.activeElement).toBe(profileTab); }); it('selects durable people and opens the exact organization editor from network actions', async () => { @@ -218,7 +392,7 @@ describe('PersonDetail', () => { expect(entityController.organizationETags.get(21)).toBe('"organization-21-r2"'); }); - it('mounts one compact profile-maintenance card before CardDAV without adding a tab', async () => { + it('mounts profile maintenance before CardDAV publication in Maintenance', async () => { const fetchFn = vi.fn(async (input) => { const request = input instanceof Request ? input : new Request(input); const path = new URL(request.url).pathname; @@ -248,14 +422,14 @@ describe('PersonDetail', () => { } }); + expect(screen.queryByRole('heading', { name: 'Profile maintenance' })).toBeNull(); + await fireEvent.click(screen.getByRole('tab', { name: 'Maintenance' })); const maintenance = await screen.findByRole('heading', { name: 'Profile maintenance' }); const publication = await screen.findByRole('heading', { name: 'CardDAV publication' }); expect(maintenance.compareDocumentPosition(publication) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); - expect(screen.getAllByRole('tab')).toHaveLength(5); - expect(screen.queryByRole('tab', { name: /maintenance/i })).toBeNull(); }); - it('mounts compact CardDAV publication in Overview and threads conflict and status callbacks', async () => { + it('mounts compact CardDAV publication in Maintenance and threads conflict and status callbacks', async () => { const onOpenCardDAVConflict = vi.fn(); const onAnnounce = vi.fn(); const fetchFn = vi.fn(async (input) => { @@ -288,15 +462,14 @@ describe('PersonDetail', () => { onAnnounce }); + await fireEvent.click(screen.getByRole('tab', { name: 'Maintenance' })); expect(await screen.findByRole('heading', { name: 'CardDAV publication' })).toBeDefined(); - expect(screen.getAllByRole('tab')).toHaveLength(5); - expect(screen.queryByRole('tab', { name: /CardDAV/i })).toBeNull(); await fireEvent.click(await screen.findByRole('button', { name: 'Review CardDAV conflict 41' })); expect(onOpenCardDAVConflict).toHaveBeenCalledWith(41); expect(onAnnounce).not.toHaveBeenCalled(); }); - it('mounts the brief card in Overview under contact state without adding a tab', async () => { + it('leads Overview with the brief card, then labeled contact state', async () => { const paths: string[] = []; const fetchFn = vi.fn(async (input) => { const request = input instanceof Request ? input : new Request(input); @@ -347,11 +520,11 @@ describe('PersonDetail', () => { const brief = await screen.findByRole('heading', { name: 'Last time we talked' }); const contactState = screen.getByText('Contact state'); - expect(contactState.compareDocumentPosition(brief) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); + expect(brief.compareDocumentPosition(contactState) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); + expect(screen.getByText('Active · 4 interactions · last contact Aug 29, 2026')).toBeDefined(); expect(await screen.findByRole('button', { name: 'Last time you talked (Aug 29, chat): they were preparing for a role change.' })).toBeDefined(); - expect(screen.getAllByRole('tab')).toHaveLength(5); expect(screen.queryByRole('tab', { name: /brief/i })).toBeNull(); expect(paths).toContain('/api/v1/people/7/brief'); expect(document.body.innerHTML).not.toContain('not-rendered.example.test'); @@ -380,6 +553,7 @@ describe('PersonDetail', () => { onOpenCardDAVSettings }); + await fireEvent.click(screen.getByRole('tab', { name: 'Maintenance' })); expect(await screen.findByText('CardDAV publication is unavailable. Configure or repair it in CardDAV settings.')).toBeDefined(); expect(screen.queryByRole('alert')).toBeNull(); expect(screen.queryByRole('button', { name: 'Retry CardDAV publication' })).toBeNull(); @@ -387,4 +561,44 @@ describe('PersonDetail', () => { await fireEvent.click(screen.getByRole('button', { name: 'Open CardDAV settings' })); expect(onOpenCardDAVSettings).toHaveBeenCalledOnce(); }); + + it('renders read-only bundle attributes once through the summary without a duplicate legacy list', () => { + const definition = nicknameDefinition(); + const client = createAPIClient(quietOverviewFetch()); + const bundle = { + person: { id: 7, revision: 2, display_name: 'Synthetic Person', participant_ids: [], vcard_uid: '', created_at: when, updated_at: when }, + attributes: { person_id: 7, attributes: [{ definition, current: nicknameCurrent() }] }, + etags: {}, errors: {} + } satisfies DirectoryReadBundle; + + render(PersonDetail, { client, personID: 7, bundle }); + + expect(document.querySelector('.attribute-summary')).not.toBeNull(); + expect(screen.getAllByText('Nickname')).toHaveLength(1); + expect(screen.getAllByText('Synthetic nickname')).toHaveLength(1); + expect(screen.queryByRole('heading', { name: 'Attributes' })).toBeNull(); + }); + + it('opens Profile and moves focus to the attributes section when Edit attributes is pressed', async () => { + const definition = nicknameDefinition(); + const client = createAPIClient(quietOverviewFetch()); + const bundle = { + person: { id: 7, revision: 2, display_name: 'Synthetic Person', participant_ids: [], vcard_uid: '', created_at: when, updated_at: when }, + attributes: { person_id: 7, attributes: [{ definition, current: nicknameCurrent() }] }, + definitions: { definitions: [definition] }, + etags: {}, errors: {} + } satisfies DirectoryReadBundle; + const profileController = new DirectoryProfileController(client, 7, bundle); + + render(PersonDetail, { client, personID: 7, bundle, profileController }); + + const button = await screen.findByRole('button', { name: 'Edit attributes' }); + await fireEvent.click(button); + + await waitFor(() => expect(document.getElementById('person-attributes')).not.toBeNull()); + expect(screen.getByRole('tab', { name: 'Profile' }).getAttribute('aria-selected')).toBe('true'); + const section = document.getElementById('person-attributes'); + expect(section!.getAttribute('tabindex')).toBe('-1'); + expect(document.activeElement).toBe(section); + }); }); diff --git a/web/src/lib/components/directory/PersonMergeHistory.svelte b/web/src/lib/components/directory/PersonMergeHistory.svelte index 4e486034f..66639fd81 100644 --- a/web/src/lib/components/directory/PersonMergeHistory.svelte +++ b/web/src/lib/components/directory/PersonMergeHistory.svelte @@ -85,7 +85,8 @@ {#if controller.history.length === 0}

No merge history on this page.

{:else} -
+ +
@@ -123,22 +124,27 @@ {#if controller.detailError}{/if} -
+ + +
MergeCreatedSurvivorAbsorbedCurrentParticipantsRowsRow actionsReviewSplitsAction
{#each controller.detail.participants ?? [] as participant}{/each}
ParticipantOriginDisposition
{participant.participant_id}{participant.origin_side}{disposition(participant.split_id)}
-
+ +
{#each controller.detail.rows ?? [] as row}{/each}
TableActionOriginProvenanceParticipantDisposition
{row.table_name}{row.action}{row.origin_side}{row.provenance_kind}{row.participant_id ?? 'None'}{disposition(row.split_id)}
-
+ +
{#each controller.detail.splits ?? [] as split}{/each}
SplitSourceCreated personRevision changeRestorationActorCreated
{split.id}Person {split.source_person_id}Person {split.new_person_id}{split.source_revision_before} → {split.source_revision_after}{split.exact_reversal ? 'Exact' : 'Partial'}{split.actor}{split.created_at}
-
+ +
{#each controller.detail.review_candidates ?? [] as candidate}{/each}
CandidatePersonDefinitionSurvivor valueAbsorbed valueResolutionStateReviewedReviewerCreated
{candidate.id}{candidate.person_id}{candidate.definition_id}{candidate.survivor_value_id}{candidate.absorbed_value_id}{candidate.resolution_value_id ?? 'None'}{candidate.state}{candidate.reviewed_at ?? 'Not reviewed'}{candidate.reviewed_by ?? 'None'}{candidate.created_at}
@@ -173,7 +179,7 @@ th, td { padding: var(--space-2); border-bottom: var(--border-width) solid var(--border-default); text-align: left; vertical-align: top; } thead th { color: var(--text-muted); white-space: nowrap; } .snapshot { max-height: 22rem; padding: var(--space-3); border: var(--border-width) solid var(--border-default); border-radius: var(--radius-sm); background: var(--bg-inset); } - .snapshot:focus-visible { outline: var(--focus-ring); outline-offset: var(--focus-ring-offset, 2px); } + .table-scroll:focus-visible, .snapshot:focus-visible { outline: var(--focus-ring); outline-offset: var(--focus-ring-offset, 2px); } .snapshot pre { white-space: pre-wrap; overflow-wrap: anywhere; } .message { display: flex; gap: var(--space-2); align-items: center; } .error { color: var(--text-danger); } diff --git a/web/src/lib/components/directory/PersonMergeHistory.test.ts b/web/src/lib/components/directory/PersonMergeHistory.test.ts index 647ce78d5..4cd1878d9 100644 --- a/web/src/lib/components/directory/PersonMergeHistory.test.ts +++ b/web/src/lib/components/directory/PersonMergeHistory.test.ts @@ -128,6 +128,9 @@ describe('PersonMergeHistory', () => { } expect(screen.getByRole('table', { name: 'Prior splits' })).toBeDefined(); expect(screen.getByRole('table', { name: 'Merge review candidates' })).toBeDefined(); + for (const name of ['Merge participants', 'Merge row dispositions', 'Prior splits', 'Merge review candidates']) { + expect(screen.getByRole('region', { name: `${name} table` }).tabIndex).toBe(0); + } expect(document.body.textContent).not.toContain('opaque-row-key-must-not-appear'); expect(document.body.textContent).not.toContain('private/snapshot/path-must-not-appear'); expect(requests.map((request) => new URL(request.url).pathname)).not.toContain('/api/v1/person-merges/41/snapshot'); diff --git a/web/src/lib/components/directory/PersonNetwork.svelte b/web/src/lib/components/directory/PersonNetwork.svelte index 78f7bb23e..3b85f9211 100644 --- a/web/src/lib/components/directory/PersonNetwork.svelte +++ b/web/src/lib/components/directory/PersonNetwork.svelte @@ -87,7 +87,9 @@
-
+ + +
{#if controller.network} + + + + + Rename person + + void openHistory()}>View profile history + + Delete person + + + + +{#if renaming} +
+ +
+
+
+{/if} +{#if confirmingDelete} +
+ Permanently delete this durable person profile? +
+
+
+{/if} + +{#if historyOpen} + { + historyOpen = false; + }} + /> +{/if} + + diff --git a/web/src/lib/components/directory/PersonRecordActions.test.ts b/web/src/lib/components/directory/PersonRecordActions.test.ts new file mode 100644 index 000000000..428bb100b --- /dev/null +++ b/web/src/lib/components/directory/PersonRecordActions.test.ts @@ -0,0 +1,144 @@ +import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import { describe, expect, it, vi } from 'vitest'; + +import { createAPIClient } from '../../api/client'; +import type { DirectoryReadBundle } from '../../directory/models'; +import { DirectoryProfileController } from '../../directory/profile-controller.svelte'; +import PersonRecordActions from './PersonRecordActions.svelte'; + +const when = '2026-08-01T00:00:00Z'; + +function person() { + return { id: 7, revision: 3, display_name: 'Test User', participant_ids: [], vcard_uid: 'person-7', created_at: when, updated_at: when }; +} + +function renderActions(fetchFn: typeof fetch, personETag: string | null = '"person-7-r3"') { + const bundle = { + person: person(), + etags: { person: personETag ?? undefined }, errors: {} + } satisfies DirectoryReadBundle; + const client = createAPIClient(fetchFn); + const controller = new DirectoryProfileController(client, 7, bundle); + render(PersonRecordActions, { client, controller, personID: 7 }); + return controller; +} + +async function choose(action: string): Promise { + await fireEvent.click(screen.getByRole('button', { name: 'More actions' })); + await fireEvent.click(screen.getByRole('menuitem', { name: action })); +} + +describe('PersonRecordActions', () => { + it('renames the selected person with the current strong ETag', async () => { + const requests: Request[] = []; + renderActions(vi.fn(async (input) => { + const request = input instanceof Request ? input : new Request(input); + requests.push(request); + return new Response(JSON.stringify({ ...person(), revision: 4, display_name: 'Renamed User' }), { + headers: { 'Content-Type': 'application/json', ETag: '"person-7-r4"' } + }); + })); + + await choose('Rename person'); + expect(screen.getByRole('group', { name: 'Rename person' })).toBeDefined(); + await fireEvent.input(screen.getByRole('textbox', { name: 'Display name' }), { target: { value: 'Renamed User' } }); + await fireEvent.click(screen.getByRole('button', { name: 'Save display name' })); + + await waitFor(() => expect(requests).toHaveLength(1)); + expect(requests[0]!.method).toBe('PATCH'); + expect(requests[0]!.headers.get('If-Match')).toBe('"person-7-r3"'); + await expect(requests[0]!.clone().json()).resolves.toEqual({ display_name: 'Renamed User' }); + }); + + it('locks rename controls while the non-abortable write is pending', async () => { + let resolveRename!: (response: Response) => void; + const pendingRename = new Promise((resolve) => { resolveRename = resolve; }); + const controller = renderActions(vi.fn(async () => pendingRename)); + + await choose('Rename person'); + await fireEvent.click(screen.getByRole('button', { name: 'Save display name' })); + + await waitFor(() => expect(controller.mutationPending).toBe(true)); + expect(screen.getByRole('button', { name: 'Cancel rename' })).toHaveProperty('disabled', true); + expect(screen.getByRole('button', { name: 'Renaming…' })).toHaveProperty('disabled', true); + + resolveRename(new Response(JSON.stringify({ ...person(), revision: 4 }), { + headers: { 'Content-Type': 'application/json', ETag: '"person-7-r4"' } + })); + await waitFor(() => expect(controller.mutationPending).toBe(false)); + }); + + it('requires confirmation and the current strong ETag before deleting a person', async () => { + const requests: Request[] = []; + renderActions(vi.fn(async (input) => { + requests.push(input instanceof Request ? input : new Request(input)); + return new Response(null, { status: 204 }); + })); + + await choose('Delete person'); + expect(requests).toHaveLength(0); + expect(screen.getByRole('group', { name: 'Confirm deleting person' })).toBeDefined(); + await fireEvent.click(screen.getByRole('button', { name: 'Confirm delete person' })); + + await waitFor(() => expect(requests).toHaveLength(1)); + expect(requests[0]!.method).toBe('DELETE'); + expect(requests[0]!.headers.get('If-Match')).toBe('"person-7-r3"'); + }); + + it('locks delete confirmation while the non-abortable delete is pending', async () => { + let resolveDelete!: (response: Response) => void; + const pendingDelete = new Promise((resolve) => { resolveDelete = resolve; }); + const controller = renderActions(vi.fn(async () => pendingDelete)); + + await choose('Delete person'); + await fireEvent.click(screen.getByRole('button', { name: 'Confirm delete person' })); + + await waitFor(() => expect(controller.mutationPending).toBe(true)); + expect(screen.getByRole('button', { name: 'Cancel delete' })).toHaveProperty('disabled', true); + expect(screen.getByRole('button', { name: 'Deleting…' })).toHaveProperty('disabled', true); + + resolveDelete(new Response(null, { status: 204 })); + await waitFor(() => expect(controller.mutationPending).toBe(false)); + }); + + it('shows one of rename or delete at a time', async () => { + renderActions(vi.fn()); + + await choose('Rename person'); + await choose('Delete person'); + expect(screen.queryByRole('group', { name: 'Rename person' })).toBeNull(); + expect(screen.getByRole('group', { name: 'Confirm deleting person' })).toBeDefined(); + + await choose('Rename person'); + expect(screen.queryByRole('group', { name: 'Confirm deleting person' })).toBeNull(); + expect(screen.getByRole('group', { name: 'Rename person' })).toBeDefined(); + }); + + it('disables both write actions when the person cannot be written', async () => { + renderActions(vi.fn(), null); + + await fireEvent.click(screen.getByRole('button', { name: 'More actions' })); + expect(screen.getByRole('menuitem', { name: 'Rename person' }).getAttribute('aria-disabled')).toBe('true'); + expect(screen.getByRole('menuitem', { name: 'Delete person' }).getAttribute('aria-disabled')).toBe('true'); + expect(screen.getByRole('menuitem', { name: 'View profile history' }).getAttribute('aria-disabled')).toBeNull(); + + await fireEvent.click(screen.getByRole('menuitem', { name: 'Rename person' })); + expect(screen.queryByRole('group', { name: 'Rename person' })).toBeNull(); + }); + + it('opens history and returns focus to the More actions trigger when it closes', async () => { + renderActions(vi.fn(async () => Response.json({ + person: person(), + names: [], contact_points: [], addresses: [], dates: [], categories: [], media: [], observations: [] + }))); + const trigger = screen.getByRole('button', { name: 'More actions' }); + + await choose('View profile history'); + const dialog = await screen.findByRole('dialog', { name: 'Profile history' }); + await waitFor(() => expect(dialog.contains(document.activeElement)).toBe(true)); + await fireEvent.click(screen.getByRole('button', { name: 'Close profile history' })); + + await waitFor(() => expect(screen.queryByRole('dialog', { name: 'Profile history' })).toBeNull()); + expect(document.activeElement).toBe(trigger); + }); +}); diff --git a/web/src/lib/components/directory/PersonRelationshipEditor.svelte b/web/src/lib/components/directory/PersonRelationshipEditor.svelte index fc024d94c..5536b763f 100644 --- a/web/src/lib/components/directory/PersonRelationshipEditor.svelte +++ b/web/src/lib/components/directory/PersonRelationshipEditor.svelte @@ -12,6 +12,7 @@ } from '@kenn-io/kit-ui'; import { onDestroy, onMount, untrack } from 'svelte'; import type { APIClient } from '../../api/client'; + import { intervalDateError } from '../../directory/dates'; import type { DirectoryEntityController } from '../../directory/entity-controller.svelte'; import type { CreatePersonRelationshipRequest, @@ -49,6 +50,9 @@ let relationshipTypeSlug = $state(initialRelationship?.type_slug ?? ''); let startDate = $state(partialDate(initialRelationship?.start_date)); let endDate = $state(partialDate(initialRelationship?.end_date)); + const uid = $props.id(); + const startDateError = $derived(intervalDateError(startDate)); + const endDateError = $derived(intervalDateError(endDate)); let notes = $state(initialRelationship?.notes ?? ''); let submitting = $state(false); let committed = $state(false); @@ -298,8 +302,10 @@ placeholder="YYYY, YYYY-MM, or YYYY-MM-DD" block disabled={submitting} + ariaDescribedby={startDateError ? `${uid}-start-date-error` : undefined} /> + {#if startDateError}

{startDateError}

{/if} {/if} + {#if endDateError}

{endDateError}

{/if} {#if message}
@@ -335,6 +343,8 @@ disabled={submitting || committed || !relationshipTypeSlug || + Boolean(startDateError) || + Boolean(endDateError) || (!initialRelationship && (counterpartID === null || counterpartID === personID)) || controller.createBlocked.relationships} /> @@ -357,6 +367,11 @@ p { margin: 0; } + .field-error { + margin: 0; + color: var(--text-danger); + font-size: var(--font-size-xs); + } .muted { color: var(--text-muted); font-size: var(--font-size-sm); diff --git a/web/src/lib/components/directory/RelationshipReviewCard.svelte b/web/src/lib/components/directory/RelationshipReviewCard.svelte index a485f4df3..f9bbee106 100644 --- a/web/src/lib/components/directory/RelationshipReviewCard.svelte +++ b/web/src/lib/components/directory/RelationshipReviewCard.svelte @@ -1,6 +1,7 @@ -
+
+

Imported relationships

-
-

Imported relationships

-

Imported relationship reviews are read-only in the browser until generated decision operations are available.

-
- ({ ...option, triggerLabel: `Show: ${option.label}` }))} onchange={selectState} - ariaLabel="Imported relationship review state" disabled={controller.loading} /> +

Imported relationship reviews are read-only in the browser until generated decision operations are available.

{#if controller.loading} @@ -98,7 +93,7 @@
{:else if controller.rows.length === 0} {:else} @@ -112,16 +107,18 @@ diff --git a/web/src/lib/components/directory/RelationshipReviewQueue.test.ts b/web/src/lib/components/directory/RelationshipReviewQueue.test.ts index f64516d28..9c13d9967 100644 --- a/web/src/lib/components/directory/RelationshipReviewQueue.test.ts +++ b/web/src/lib/components/directory/RelationshipReviewQueue.test.ts @@ -102,9 +102,10 @@ describe('RelationshipReviewQueue', () => { render(RelationshipReviewQueue, { controller }); await screen.findByRole('article', { name: 'Imported relationship review 41' }); - const pending = screen.getByRole('radio', { name: 'Pending' }); - pending.focus(); - await fireEvent.keyDown(pending, { key: 'ArrowRight' }); + const show = screen.getByRole('combobox', { name: /^Imported relationship review state/ }); + expect(show.textContent).toContain('Show: Pending'); + await fireEvent.click(show); + await fireEvent.click(screen.getByRole('option', { name: 'Accepted' })); expect(await screen.findByRole('article', { name: 'Imported relationship review 51' })).toBeDefined(); expect(states).toEqual(['pending', 'accepted']); diff --git a/web/src/lib/components/directory/RelationshipsTab.test.ts b/web/src/lib/components/directory/RelationshipsTab.test.ts index 141efc757..5f92ebfc5 100644 --- a/web/src/lib/components/directory/RelationshipsTab.test.ts +++ b/web/src/lib/components/directory/RelationshipsTab.test.ts @@ -199,6 +199,47 @@ describe('RelationshipsTab', () => { } }); + it('rejects an impossible relationship date and keeps Create disabled until it is fixed', async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }); + try { + const { client, controller } = controllerWith(vi.fn(async (input) => { + const request = requestOf(input); + if (pathOf(request) === '/api/v1/people/directory') return Response.json({ people: [ + { id: 8, revision: 1, display_name: 'Synthetic Counterpart', categories: [], contact_state: 'active', organizations: [] } + ] }); + throw new Error(`unexpected ${request.method} ${pathOf(request)}`); + })); + controller.relationshipTypes = [relationshipType()]; + render(RelationshipsTab, { client, controller, personID: 7 }); + + await fireEvent.click(screen.getByRole('button', { name: 'Add relationship' })); + const personSearch = await openTypeahead('Relationship counterpart'); + await fireEvent.input(personSearch, { target: { value: 'Synthetic' } }); + await vi.advanceTimersByTimeAsync(250); + await fireEvent.mouseDown(await screen.findByRole('option', { name: /Synthetic Counterpart/ })); + await chooseSelectOption(screen.getByRole('combobox', { name: /^Relationship type:/ }), 'mentors / is mentored by'); + const create = screen.getByRole('button', { name: 'Create relationship' }); + expect(create).toHaveProperty('disabled', false); + + const start = screen.getByRole('textbox', { name: 'Relationship start date' }); + await fireEvent.input(start, { target: { value: '2019-02-30' } }); + const message = screen.getByText('Use a year, year and month, or full date, like 2019, 2019-04, or 2019-04-12.'); + expect(start.getAttribute('aria-describedby')).toBe(message.id); + expect(create).toHaveProperty('disabled', true); + + await fireEvent.input(start, { target: { value: '2019' } }); + expect(screen.queryByText(/Use a year/)).toBeNull(); + expect(create).toHaveProperty('disabled', false); + + const end = screen.getByRole('textbox', { name: 'Relationship end date' }); + await fireEvent.input(end, { target: { value: '--04-12' } }); + expect(screen.getByText('Use a year, year and month, or full date, like 2019, 2019-04, or 2019-04-12.').id).toBe(end.getAttribute('aria-describedby')); + expect(create).toHaveProperty('disabled', true); + } finally { + vi.useRealTimers(); + } + }); + it('creates an outgoing edge at the collection endpoint with the selected person as source', async () => { const requests: Request[] = []; const created = relationship(45, 1, { source_person_id: 7, target_person_id: 9 }); diff --git a/web/src/lib/components/directory/StructuredProfileEditor.svelte b/web/src/lib/components/directory/StructuredProfileEditor.svelte index 469da8843..53aba1de4 100644 --- a/web/src/lib/components/directory/StructuredProfileEditor.svelte +++ b/web/src/lib/components/directory/StructuredProfileEditor.svelte @@ -13,6 +13,7 @@ PersonName as GeneratedPersonName, ValueEnvelopeInput as GeneratedValueEnvelopeInput, } from '../../api/generated/models'; + import { isTextProfileDate, profileDateError } from '../../directory/dates'; import type { PersonProfilePatchRequest } from '../../directory/models'; import type { DirectoryProfileController } from '../../directory/profile-controller.svelte'; @@ -81,6 +82,12 @@ (initialSection === 'dates' ? (initialCurrent?.original_value ?? '') : ''), ); let dateLabel = $state(currentDate?.label ?? ''); + const uid = $props.id(); + const dateError = $derived(profileDateError(dateValue)); + const textDate = $derived(isTextProfileDate(dateValue)); + const dateDescribedBy = $derived( + [dateError ? `${uid}-date-error` : '', textDate ? `${uid}-date-hint` : ''].filter(Boolean).join(' ') || undefined + ); let categoryValue = $state(initialSection === 'categories' ? (initialCurrent?.original_value ?? '') : ''); let mediaKind = $state(currentMedia?.media_kind ?? 'photo'); let mediaURI = $state( @@ -441,8 +448,11 @@ required block disabled={submitting} + ariaDescribedby={dateDescribedBy} /> + {#if dateError}

{dateError}

{/if} + {#if textDate}

Saved as text

{/if} {:else if section === 'categories'}
@@ -547,6 +557,16 @@ gap: var(--space-2); flex-wrap: wrap; } + .field-error { + margin: 0; + color: var(--text-danger); + font-size: var(--font-size-xs); + } + .field-hint { + margin: 0; + color: var(--text-muted); + font-size: var(--font-size-xs); + } .editor-error { margin: 0; color: var(--text-danger); diff --git a/web/src/lib/components/directory/StructuredProfileEditor.test.ts b/web/src/lib/components/directory/StructuredProfileEditor.test.ts index 3f056809c..89455b5c0 100644 --- a/web/src/lib/components/directory/StructuredProfileEditor.test.ts +++ b/web/src/lib/components/directory/StructuredProfileEditor.test.ts @@ -624,6 +624,48 @@ describe('StructuredProfileEditor', () => { }); }); + it.each(['--04-12', '--04', '---12'])('keeps the year-less profile date form %s', async (value) => { + const { controller } = requestHarness(); + render(StructuredProfileEditor, { controller, section: 'dates' }); + + await fireEvent.input(screen.getByRole('textbox', { name: 'Date' }), { target: { value } }); + + expect(screen.queryByText(/does not exist/)).toBeNull(); + expect(screen.queryByText('Saved as text')).toBeNull(); + expect(screen.getByRole('button', { name: 'Save date' })).toHaveProperty('disabled', false); + }); + + it('rejects an impossible profile date and marks free text', async () => { + const { controller } = requestHarness(); + render(StructuredProfileEditor, { controller, section: 'dates' }); + const field = screen.getByRole('textbox', { name: 'Date' }); + const save = screen.getByRole('button', { name: 'Save date' }); + + await fireEvent.input(field, { target: { value: '--02-30' } }); + const message = screen.getByText('This date does not exist. Check the month and day.'); + expect(field.getAttribute('aria-describedby')).toBe(message.id); + expect(save).toHaveProperty('disabled', true); + + await fireEvent.input(field, { target: { value: 'spring 2019' } }); + expect(screen.queryByText(/does not exist/)).toBeNull(); + const hint = screen.getByText('Saved as text'); + expect(field.getAttribute('aria-describedby')).toBe(hint.id); + expect(save).toHaveProperty('disabled', false); + }); + + it('submits a free-text profile date as date_text without a typed date', async () => { + const { controller, requests } = requestHarness(); + render(StructuredProfileEditor, { controller, section: 'dates' }); + + await fireEvent.input(screen.getByRole('textbox', { name: 'Date' }), { target: { value: 'spring 2019' } }); + await fireEvent.click(screen.getByRole('button', { name: 'Save date' })); + + await waitFor(() => expect(requests).toHaveLength(1)); + const body = (await requests[0]!.clone().json()) as { dates: { add: Array> } }; + expect(body.dates.add[0]).toMatchObject({ date_text: 'spring 2019' }); + expect(body.dates.add[0]).not.toHaveProperty('date'); + }); + it.each([ ['2000-02-03', { year: 2000, month: 2, day: 3 }], ['2000-02', { year: 2000, month: 2 }], diff --git a/web/src/lib/components/directory/StructuredProfileSection.svelte b/web/src/lib/components/directory/StructuredProfileSection.svelte index 5cffac052..a5ec77d8f 100644 --- a/web/src/lib/components/directory/StructuredProfileSection.svelte +++ b/web/src/lib/components/directory/StructuredProfileSection.svelte @@ -1,6 +1,6 @@
-
-

Structured profile

-
-
-
- - {#if renaming} -
- -
-
-
- {/if} - {#if confirmingDelete} -
- Permanently delete this durable person profile? -
-
-
- {/if} +

Structured profile

{#each sections as descriptor (descriptor.section)}
@@ -431,26 +335,14 @@ {/if}
-{#if historyOpen} - { - historyOpen = false; - }} - /> -{/if} - diff --git a/web/src/lib/components/explore/FilesPresentation.test.ts b/web/src/lib/components/explore/FilesPresentation.test.ts deleted file mode 100644 index 73e74ae64..000000000 --- a/web/src/lib/components/explore/FilesPresentation.test.ts +++ /dev/null @@ -1,141 +0,0 @@ -import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; -import { describe, expect, it, vi } from 'vitest'; - -import type { ExploreFileFact } from '../../explore/models'; -import FilesPresentation from './FilesPresentation.svelte'; - -function file(index: number): ExploreFileFact { - return { - id: index, - key: `message:${index}:file:${index}`, - entry_key: `message:${index}`, - message_id: index, - conversation_id: index + 1000, - occurred_at: '2026-07-18T12:00:00Z', - source_id: 1, - source_identifier: 'archive@example.com', - title: `Containing item ${index}`, - filename: `file-${index}.pdf`, - mime_type: 'application/pdf', - size: 2048 - }; -} - -describe('FilesPresentation', () => { - it('reports cache initialization and waits for automatic recovery', () => { - render(FilesPresentation, { - files: [], - unavailable: { - error: 'analytical_cache_unavailable', - message: 'The analytical cache is being prepared', - readiness: 'building', - recovery_action: '' - } - }); - - const alert = screen.getByRole('alert'); - expect(alert.textContent).toContain('Preparing analytical cache'); - expect(alert.textContent).toContain('This view will refresh automatically.'); - expect(screen.queryByRole('button', { name: 'Retry cache check' })).toBeNull(); - }); - - it('virtualizes a large loaded slice and implements bounded table navigation', async () => { - const files = Array.from({ length: 1_000 }, (_, index) => file(index + 1)); - const onActiveKey = vi.fn(); - const onLoadMore = vi.fn(); - render(FilesPresentation, { - files, hasMore: true, totalCount: 100_000, onActiveKey, onLoadMore - }); - const grid = screen.getByRole('grid', { name: 'Files in current context' }); - grid.focus(); - - await waitFor(() => expect(screen.getAllByRole('row').length).toBeLessThan(80)); - await fireEvent.keyDown(grid, { key: 'PageDown' }); - expect(onActiveKey).toHaveBeenLastCalledWith('message:10:file:10'); - await fireEvent.keyDown(grid, { key: 'PageUp' }); - expect(onActiveKey).toHaveBeenLastCalledWith('message:1:file:1'); - await fireEvent.keyDown(grid, { key: 'End' }); - expect(onActiveKey).toHaveBeenLastCalledWith('message:1000:file:1000'); - expect(onLoadMore).not.toHaveBeenCalled(); - }); - - it('loads at most one next page when keyboard movement crosses the loaded boundary', async () => { - let files = Array.from({ length: 600 }, (_, index) => file(index + 1)); - const onActiveKey = vi.fn(); - let rendered: ReturnType; - const onLoadMore = vi.fn(async () => { - files = Array.from({ length: 700 }, (_, index) => file(index + 1)); - await rendered.rerender({ - files, hasMore: true, totalCount: 10_000, - focusedKey: 'message:600:file:600', onActiveKey, onLoadMore - }); - return { status: 'advanced' }; - }); - rendered = render(FilesPresentation, { - files, hasMore: true, totalCount: 10_000, - focusedKey: 'message:600:file:600', onActiveKey, onLoadMore - }); - const grid = screen.getByRole('grid', { name: 'Files in current context' }); - grid.focus(); - - await fireEvent.keyDown(grid, { key: 'ArrowDown' }); - - expect(onLoadMore).toHaveBeenCalledTimes(1); - await waitFor(() => expect(onActiveKey).toHaveBeenLastCalledWith('message:601:file:601')); - expect(screen.getAllByRole('row').length).toBeLessThan(80); - }); - - it('keeps loaded files visible and exposes an accessible paging retry', async () => { - const files = Array.from({ length: 600 }, (_, index) => file(index + 1)); - const onLoadMore = vi.fn(async () => ({ status: 'advanced' })); - render(FilesPresentation, { - files, hasMore: true, totalCount: 10_000, - pageError: 'The next file page could not be loaded.', onLoadMore - }); - - expect(screen.getByText('file-1.pdf')).toBeDefined(); - expect(screen.getByRole('alert').textContent).toContain('The next file page could not be loaded.'); - await fireEvent.click(screen.getByRole('button', { name: 'Retry loading more files' })); - expect(onLoadMore).toHaveBeenCalledTimes(1); - }); - - it('restores a keyed scroll anchor and reports user scroll authority', async () => { - const onScrollAnchor = vi.fn(); - const files = Array.from({ length: 200 }, (_, index) => file(index + 1)); - const rendered = render(FilesPresentation, { - files, - focusedKey: 'message:150:file:150', - scrollAnchor: { key: 'message:140:file:140', offset: 5 }, - restoring: true, - onScrollAnchor - }); - const grid = screen.getByRole('grid', { name: 'Files in current context' }) as HTMLDivElement; - await waitFor(() => expect(grid.scrollTop).toBe(139 * 36 + 5)); - expect(grid.getAttribute('aria-activedescendant')).toContain('message-3a-150'); - - await rendered.rerender({ - files, focusedKey: 'message:150:file:150', - scrollAnchor: { key: 'message:140:file:140', offset: 5 }, - restoring: false, onScrollAnchor - }); - grid.scrollTop = 72; - await fireEvent.scroll(grid); - expect(onScrollAnchor).toHaveBeenLastCalledWith('message:3:file:3', 0); - }); - - it('opens the attachment while keeping the containing item a separate action', async () => { - const onOpenFile = vi.fn(); - const onOpenItem = vi.fn(); - const selected = file(7); - render(FilesPresentation, { files: [selected], onOpenFile, onOpenItem }); - const grid = screen.getByRole('grid', { name: 'Files in current context' }); - grid.focus(); - - await fireEvent.keyDown(grid, { key: 'Enter' }); - expect(onOpenFile).toHaveBeenCalledWith(selected); - expect(onOpenItem).not.toHaveBeenCalled(); - - await fireEvent.click(screen.getByRole('button', { name: 'Open containing item 7' })); - expect(onOpenItem).toHaveBeenCalledWith('message:7'); - }); -}); diff --git a/web/src/lib/components/explore/GroupTable.svelte b/web/src/lib/components/explore/GroupTable.svelte index af75d5203..b5e06262c 100644 --- a/web/src/lib/components/explore/GroupTable.svelte +++ b/web/src/lib/components/explore/GroupTable.svelte @@ -433,8 +433,6 @@ color: var(--text-muted); font-size: var(--font-size-2xs); font-weight: 600; - letter-spacing: 0.06em; - text-transform: uppercase; } .numeric { diff --git a/web/src/lib/components/explore/RowKind.svelte b/web/src/lib/components/explore/RowKind.svelte index 26d0c9eaa..7639983e5 100644 --- a/web/src/lib/components/explore/RowKind.svelte +++ b/web/src/lib/components/explore/RowKind.svelte @@ -1,27 +1,14 @@ - {presentation.label.replace(' item', '').replace('Archive', 'Item')} + {presentation.name} diff --git a/web/src/lib/components/explore/SelectionBar.test.ts b/web/src/lib/components/explore/SelectionBar.test.ts index 82b5c3ea2..73f2355ec 100644 --- a/web/src/lib/components/explore/SelectionBar.test.ts +++ b/web/src/lib/components/explore/SelectionBar.test.ts @@ -1,11 +1,13 @@ import { fireEvent, render, screen } from '@testing-library/svelte'; -import { describe, expect, it } from 'vitest'; +import { tick } from 'svelte'; +import { describe, expect, it, vi } from 'vitest'; +import { createAPIClient } from '../../api/client'; import type { ExplorePreflightResponse as GeneratedExplorePreflightResponse, + ExploreSelection as GeneratedExploreSelection, ExploreUnavailableAction as GeneratedExploreUnavailableAction, } from '../../api/generated/models'; - import { ExploreSelectionState } from '../../explore/state.svelte'; import { predicateFingerprint } from '../../explore/selection'; import SelectionBar from './SelectionBar.svelte'; @@ -35,6 +37,32 @@ describe('SelectionBar', () => { expect(selection.count).toBe(0); }); + it('announces the first selection through a live region that exists before it', async () => { + const selection = new ExploreSelectionState(); + render(SelectionBar, { selection, totalCount: 8 }); + const status = screen.getByRole('status'); + expect(status.textContent).toBe(''); + + selection.selectVisible(['message:1']); + await tick(); + + expect(screen.getByRole('status')).toBe(status); + expect(status.textContent).toBe('1 selected'); + }); + + it('hands focus back after Clear selection removes the bar', async () => { + const selection = new ExploreSelectionState(); + selection.selectVisible(['message:1']); + const onClear = vi.fn(() => expect(selection.count).toBe(0)); + render(SelectionBar, { selection, totalCount: 8, onClear }); + + await fireEvent.click(screen.getByRole('button', { name: 'Clear selection' })); + + expect(onClear).toHaveBeenCalledOnce(); + expect(screen.queryByRole('button', { name: 'Clear selection' })).toBeNull(); + expect(screen.getByRole('status').textContent).toBe(''); + }); + it('labels predicate selection as all matching rather than a finite URL selection', () => { const selection = new ExploreSelectionState(); selection.selectAllMatching({ @@ -96,8 +124,210 @@ describe('SelectionBar', () => { }); expect(screen.queryByRole('button', { name: 'Export selection' })).toBeNull(); - expect(screen.queryByRole('button', { name: 'Open selection in source' })).toBeNull(); - expect(screen.getByText('Export: selection_contains_items_without_exportable_files')).toBeDefined(); - expect(screen.getByText('Open in source: selection_contains_items_that_cannot_be_opened_in_source')).toBeDefined(); + expect(screen.getByText('Export unavailable: Selection contains items without exportable files.')).toBeDefined(); + }); + + it('lists a disabled open-in-source item with a plain reason and no raw code', async () => { + const selection = new ExploreSelectionState(); + selection.selectVisible(['message:1']); + render(SelectionBar, { + selection, + totalCount: 2, + preflight: preflight([{ action: 'open_in_source', reason: 'trusted_source_link_unavailable' }]), + onOpenInSource: () => undefined, + }); + + await fireEvent.click(screen.getByRole('button', { name: 'More selection actions' })); + + const item = screen.getByRole('menuitem', { + name: 'Open selection in source Your sources don’t provide links to open these items.' + }); + expect(item.getAttribute('aria-disabled')).toBe('true'); + expect(screen.queryByText(/trusted_source_link_unavailable/)).toBeNull(); + }); + + it('opens the selection in source from the overflow menu when preflight allows it', async () => { + const selection = new ExploreSelectionState(); + selection.selectVisible(['message:1']); + const onOpenInSource = vi.fn(); + render(SelectionBar, { selection, totalCount: 2, preflight: preflight(), onOpenInSource }); + + await fireEvent.click(screen.getByRole('button', { name: 'More selection actions' })); + await fireEvent.click(screen.getByRole('menuitem', { name: 'Open selection in source' })); + + expect(onOpenInSource).toHaveBeenCalledOnce(); + }); + + it('renders nothing while nothing is selected', () => { + const selection = new ExploreSelectionState(); + render(SelectionBar, { selection, totalCount: 8 }); + + expect(screen.getByRole('status').textContent).toBe(''); + expect(screen.queryByRole('button', { name: 'Clear selection' })).toBeNull(); + expect(screen.queryByText('No items selected')).toBeNull(); + expect(screen.queryByRole('button', { name: 'Review for deletion…' })).toBeNull(); + }); + + it('offers deletion review for an explicit and then an all-matching selection', async () => { + const selection = new ExploreSelectionState(); + selection.selectVisible(['message:1', 'message:2', 'message:3']); + const onReviewDeletion = vi.fn(); + const allMatching = { + mode: 'all_matching' as const, + predicate: { query: 'synthetic', search_mode: 'full_text' as const }, + exclusions: [], + cacheRevision: 'cache-1', + searchProvenance: { lexical_index_revision: 'fts-1' }, + predicateFingerprint: predicateFingerprint({ query: 'synthetic', search_mode: 'full_text' }), + resultGeneration: 1, + }; + render(SelectionBar, { selection, totalCount: 50, allMatching, onReviewDeletion }); + + expect(screen.getByRole('status').textContent).toContain('3 selected'); + await fireEvent.click(screen.getByRole('button', { name: 'Review for deletion…' })); + expect(onReviewDeletion).toHaveBeenLastCalledWith('explicit'); + + await fireEvent.click(screen.getByRole('button', { name: 'Select all 50 matching items' })); + await fireEvent.click(screen.getByRole('button', { name: 'Review for deletion…' })); + expect(onReviewDeletion).toHaveBeenLastCalledWith('all_matching'); + }); + + it('disables Review for deletion with the staging reason as a sentence', () => { + const selection = new ExploreSelectionState(); + selection.selectVisible(['message:1']); + render(SelectionBar, { + selection, + totalCount: 2, + onReviewDeletion: vi.fn(), + preflight: preflight([ + { action: 'stage_deletion', reason: 'selection_contains_items_that_cannot_be_deleted_from_source' }, + ]), + }); + + const review = screen.getByRole('button', { name: 'Review for deletion…' }) as HTMLButtonElement; + expect(review.disabled).toBe(true); + const reason = screen.getByText('None of the selected items can be deleted from their source.'); + expect(review.getAttribute('aria-describedby')).toBe(reason.id); + expect(reason.getAttribute('title')).toBe('selection_contains_items_that_cannot_be_deleted_from_source'); + }); + + it('keeps Review for deletion enabled when staging is available', () => { + const selection = new ExploreSelectionState(); + selection.selectVisible(['message:1']); + render(SelectionBar, { selection, totalCount: 2, onReviewDeletion: vi.fn(), preflight: preflight() }); + + const review = screen.getByRole('button', { name: 'Review for deletion…' }) as HTMLButtonElement; + expect(review.disabled).toBe(false); + expect(review.getAttribute('aria-describedby')).toBeNull(); + }); + + it('keeps meeting context independent from raw-export preflight eligibility', async () => { + const selection = new ExploreSelectionState(); + selection.selectVisible(['message:7', 'message:91']); + const meetingSelection: GeneratedExploreSelection = { + mode: 'explicit', + predicate: { filters: [], presentation: 'table' }, + row_keys: ['message:7', 'message:91'], + cache_revision: 'cache-1', + search_provenance: {}, + }; + render(SelectionBar, { + selection, + totalCount: 2, + preflight: preflight([ + { action: 'export', reason: 'selection_contains_items_without_exportable_files' }, + ]), + client: createAPIClient(async () => + Response.json( + { error: 'selection_not_all_meetings', message: 'Every selected row must be a meeting' }, + { status: 400 }, + ), + ), + meetingSelection, + canExportMeetings: true, + }); + + expect(screen.getByText('Export unavailable: Selection contains items without exportable files.')).toBeDefined(); + await fireEvent.click(screen.getByRole('button', { name: 'Meeting context…' })); + await fireEvent.click(screen.getByRole('button', { name: 'Export meeting context' })); + expect((await screen.findByRole('alert')).textContent).toContain('Select meetings only'); + }); + + it('explains an explicit selection over the 100-meeting context limit before request', async () => { + const rowKeys = Array.from({ length: 101 }, (_, index) => `message:${index + 1}`); + const selection = new ExploreSelectionState(); + selection.selectVisible(rowKeys); + const fetchFn = vi.fn(); + render(SelectionBar, { + selection, + totalCount: rowKeys.length, + client: createAPIClient(fetchFn), + canExportMeetings: true, + meetingSelection: { + mode: 'explicit', + predicate: { filters: [], presentation: 'table' }, + row_keys: rowKeys, + cache_revision: 'cache-1', + search_provenance: {}, + } satisfies GeneratedExploreSelection, + }); + + await fireEvent.click(screen.getByRole('button', { name: 'Meeting context…' })); + expect(screen.getByText('Meeting context accepts at most 100 meetings.')).toBeDefined(); + expect((screen.getByRole('button', { name: 'Export meeting context' }) as HTMLButtonElement).disabled).toBe(true); + expect(fetchFn).not.toHaveBeenCalled(); + }); + + describe('meeting context', () => { + const meetingSelection: GeneratedExploreSelection = { + mode: 'explicit', + predicate: { filters: [], presentation: 'table' }, + row_keys: ['message:1'], + cache_revision: 'cache-1', + search_provenance: {}, + }; + const renderBar = (canExportMeetings?: boolean) => { + const selection = new ExploreSelectionState(); + selection.selectVisible(['message:1']); + render(SelectionBar, { + selection, + totalCount: 2, + client: createAPIClient(vi.fn()), + meetingSelection, + canExportMeetings, + }); + return selection; + }; + + it('hides meeting export for selections that cannot contain meetings', () => { + renderBar(false); + + expect(screen.queryByRole('button', { name: 'Meeting context…' })).toBeNull(); + expect(screen.queryByRole('button', { name: 'Export meeting context' })).toBeNull(); + }); + + it('reveals the export controls from a disclosure button for meeting selections', async () => { + renderBar(true); + + expect(screen.queryByRole('button', { name: 'Export meeting context' })).toBeNull(); + const toggle = screen.getByRole('button', { name: 'Meeting context…' }); + expect(toggle.getAttribute('aria-expanded')).toBe('false'); + await fireEvent.click(toggle); + + expect(toggle.getAttribute('aria-expanded')).toBe('true'); + expect(screen.getByRole('button', { name: 'Export meeting context' })).toBeDefined(); + }); + + it('closes the meeting context controls when the bar hides', async () => { + const selection = renderBar(true); + await fireEvent.click(screen.getByRole('button', { name: 'Meeting context…' })); + await fireEvent.click(screen.getByRole('button', { name: 'Clear selection' })); + + selection.selectVisible(['message:1']); + await tick(); + + expect(screen.getByRole('button', { name: 'Meeting context…' }).getAttribute('aria-expanded')).toBe('false'); + expect(screen.queryByRole('button', { name: 'Export meeting context' })).toBeNull(); + }); }); }); diff --git a/web/src/lib/components/files/FileViewer.svelte b/web/src/lib/components/files/FileViewer.svelte index 0063128ae..433490393 100644 --- a/web/src/lib/components/files/FileViewer.svelte +++ b/web/src/lib/components/files/FileViewer.svelte @@ -1,11 +1,3 @@ - - - -
-

{personScoped ? 'Attachments' : 'Files'}

- {totalCount.toLocaleString()} {personPresentation === 'media' && personScoped ? 'media items' : 'files'} -
+ + {#if showHeader} +
+

{personScoped ? 'Attachments' : 'Files'}

+ {#if !loading && !error && !unavailable} + {totalCount.toLocaleString()} {personPresentation === 'media' && personScoped ? 'media items' : 'files'} + {/if} +
+ {/if}
{#if personScoped} @@ -762,7 +822,13 @@ oninput={(value) => onFilenameQueryChange?.(value)} /> - + + {#if hostedVisualSearch}
{#if personScoped && personPresentation === 'media'} @@ -864,7 +925,10 @@ {:else if error && rows.length === 0}
- +
{:else if loading && rows.length === 0} @@ -887,44 +951,58 @@
{:else}
-
+
{#each renderedRows as row, offset (row.key)} {@const index = slice.start + offset} - +
{ + onpointerdown={() => { activeKey = row.key; onActiveKey?.(row.key); grid?.focus(); - viewerReturnFocus = event.currentTarget as HTMLElement; }} onclick={(event) => { if (!(event.target as Element).closest('button')) { - open(row, event.currentTarget as HTMLElement); + open(row); } }} > - - {row.filename || '(unnamed)'} - {#if row.search_explain}RRF {row.search_explain.rrf.toFixed(4)}{/if} + + {row.filename || '(unnamed)'} - {row.mime_type || row.mime_family} + {fileTypeLabel(row.mime_type, row.mime_family)} {formatBytes(row.size_bytes)} {#if personScoped}{relationship(row)}{/if} {people(row)} {row.source_identifier} - {row.containing_title || row.entry_key} + + {#if onOpenItem} + {@const openItem = onOpenItem} + + {:else} + {row.containing_title || row.entry_key} + {/if} + {availability(row)}
{/each} @@ -1000,8 +1078,10 @@ {/if} diff --git a/web/src/lib/components/layout/SplitPane.test.ts b/web/src/lib/components/layout/SplitPane.test.ts index 3cc6ffe93..daf40307c 100644 --- a/web/src/lib/components/layout/SplitPane.test.ts +++ b/web/src/lib/components/layout/SplitPane.test.ts @@ -2,6 +2,7 @@ import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; import { afterEach, describe, expect, it, vi } from 'vitest'; import SplitPane from './SplitPane.svelte'; +import { splitHandleSize } from './split-handle'; const observers = new Map(); @@ -98,17 +99,21 @@ describe('SplitPane', () => { const host = container.querySelector('[data-split-pane]')!; const primary = container.querySelector('[data-pane="primary"]') as HTMLElement; + // The primary pane may take whatever the 320px secondary minimum and the + // kit handle leave over. + const wideMaximum = 700 - 320 - splitHandleSize; reportWidth(host, 700); - await waitFor(() => expect(primary.style.flexBasis).toBe('376px')); + await waitFor(() => expect(primary.style.flexBasis).toBe(`${wideMaximum}px`)); await fireEvent.keyDown(screen.getByRole('separator', { name: 'Resize result list' }), { key: 'ArrowRight' }); - expect(primary.style.flexBasis).toBe('376px'); + expect(primary.style.flexBasis).toBe(`${wideMaximum}px`); + const narrowMaximum = 500 - 320 - splitHandleSize; reportWidth(host, 500); - await waitFor(() => expect(primary.style.flexBasis).toBe('176px')); - expect(localStorage.getItem('archive:test-split')).toBe('176'); + await waitFor(() => expect(primary.style.flexBasis).toBe(`${narrowMaximum}px`)); + expect(localStorage.getItem('archive:test-split')).toBe(String(narrowMaximum)); }); it('caps a horizontal primary pane at maxPrimary even when more width is available', async () => { @@ -312,8 +317,8 @@ describe('SplitPane', () => { for (let step = 0; step < 30; step += 1) { await fireEvent.keyDown(handle, { key: 'ArrowUp' }); } - // 800 - 120 (list minimum) - 4 (Kit handle) = 676. - expect(secondary.style.flexBasis).toBe('676px'); + // 800 minus the 120px list minimum and the kit handle. + expect(secondary.style.flexBasis).toBe(`${800 - 120 - splitHandleSize}px`); for (let step = 0; step < 30; step += 1) { await fireEvent.keyDown(handle, { key: 'ArrowDown' }); diff --git a/web/src/lib/components/layout/split-handle.ts b/web/src/lib/components/layout/split-handle.ts new file mode 100644 index 000000000..a8228de7f --- /dev/null +++ b/web/src/lib/components/layout/split-handle.ts @@ -0,0 +1,6 @@ +import brand from '@kenn-io/kit-ui/brand.json'; + +/** Thickness of kit-ui's SplitResizeHandle in CSS pixels. kit-ui owns the + * value (`layout.splitHandleSize`, rendered as `--split-handle-size`), so + * split panes reserve exactly this much room for the handle. */ +export const splitHandleSize = parseFloat(brand.layout.splitHandleSize); diff --git a/web/src/lib/components/meetings/ArchivedMeetingReader.svelte b/web/src/lib/components/meetings/ArchivedMeetingReader.svelte new file mode 100644 index 000000000..5cf8d5c75 --- /dev/null +++ b/web/src/lib/components/meetings/ArchivedMeetingReader.svelte @@ -0,0 +1,43 @@ + + + +
+ {#if message} + + {:else if loading} +

Loading archived meeting…

+ {:else if error} +

{error}

+
+
+ + diff --git a/web/src/lib/components/meetings/MeetingActions.svelte b/web/src/lib/components/meetings/MeetingActions.svelte new file mode 100644 index 000000000..43eb32f71 --- /dev/null +++ b/web/src/lib/components/meetings/MeetingActions.svelte @@ -0,0 +1,179 @@ + + +
+

Archived action items

+ {#if controller.loading} +

Loading action evidence…

+ {/if} + {#if controller.error} +

{controller.error.message}

+ {#if controller.error.recovery === 'retry' || controller.error.recovery === 'reload'} +
+ + diff --git a/web/src/lib/components/meetings/MeetingActions.test.ts b/web/src/lib/components/meetings/MeetingActions.test.ts new file mode 100644 index 000000000..477b82ac0 --- /dev/null +++ b/web/src/lib/components/meetings/MeetingActions.test.ts @@ -0,0 +1,256 @@ +import { fireEvent, render, screen, waitFor, within } from '@testing-library/svelte'; +import { describe, expect, it, vi } from 'vitest'; + +import { createAPIClient } from '../../api/client'; +import type { ActionsPage, MeetingActionsRequest, MeetingRef } from '../../api/generated/models'; +import MeetingActions from './MeetingActions.svelte'; + +const request = { + scope: { message_ids: [42] }, + limit: 200, +} satisfies MeetingActionsRequest; +const meeting: MeetingRef = { + message_id: 42, + conversation_id: 8, + source_id: 3, + source_type: 'zoom', + source_identifier: 'synthetic-zoom', + source_message_id: 'meeting-42', + title: 'Café review', + occurred_at: '2026-09-12T10:00:00Z', + archive_path: '/api/v1/messages/42', +}; + +function page(overrides: Partial = {}): ActionsPage { + return { + schema_version: 1, + archive_uid: 'archive-test', + rows: [], + total_count: 0, + coverage: { + meeting_count: 1, + available: 1, + partial: 0, + unsupported: 0, + unavailable: 0, + }, + scope: { kind: 'direct' }, + ...overrides, + }; +} + +describe('MeetingActions', () => { + it('shows source evidence and passes authoritative meeting provenance to archive navigation', async () => { + const requests: Request[] = []; + const onOpenMeeting = vi.fn(); + const fetchFn = vi.fn(async (input) => { + requests.push(input instanceof Request ? input : new Request(input)); + return Response.json( + page({ + rows: [ + { + meeting, + action: { + ordinal: 0, + locator: 'action:0', + origin: 'source', + title: 'Send résumé', + description: 'Share the revised résumé without changing Unicode.', + status: 'completed', + source_status: 'true', + assignee_name: 'Jamie Example', + assignee_email: 'jamie@example.test', + due_date: 'After the next review', + }, + }, + ], + total_count: 1, + }), + ); + }); + render(MeetingActions, { + client: createAPIClient(fetchFn), + request, + onOpenMeeting, + }); + + expect(await screen.findByText('Send résumé')).toBeDefined(); + expect(screen.getByText('Share the revised résumé without changing Unicode.')).toBeDefined(); + const evidence = screen.getByText('Send résumé').closest('li')!; + expect(within(evidence).getByText('Source status')).toBeDefined(); + expect(within(evidence).getByText('completed (source: true)')).toBeDefined(); + expect(screen.getByText(/Jamie Example.*jamie@example\.test/)).toBeDefined(); + expect(within(evidence).getByText('Due')).toBeDefined(); + expect(within(evidence).getByText('After the next review')).toBeDefined(); + expect(screen.getByText('Coverage: 1 available · 0 partial · 0 unsupported · 0 unavailable')).toBeDefined(); + const archiveLink = screen.getByRole('link', { + name: 'Open archived meeting', + }); + expect(archiveLink.getAttribute('href')).toBe('/api/v1/messages/42'); + await fireEvent.click(archiveLink); + expect(onOpenMeeting).toHaveBeenCalledWith(meeting); + await expect(requests[0]!.clone().json()).resolves.toEqual(request); + }); + + it.each([ + [ + { + meeting_count: 1, + available: 1, + partial: 0, + unsupported: 0, + unavailable: 0, + }, + 'No recorded action items', + ], + [ + { + meeting_count: 1, + available: 0, + partial: 1, + unsupported: 0, + unavailable: 0, + }, + 'Action item evidence is partial', + ], + [ + { + meeting_count: 1, + available: 0, + partial: 0, + unsupported: 1, + unavailable: 0, + }, + 'Action items are not supported', + ], + [ + { + meeting_count: 1, + available: 0, + partial: 0, + unsupported: 0, + unavailable: 1, + }, + 'Action item evidence is unavailable', + ], + ])('uses coverage to distinguish empty evidence state %#', async (coverage, expected) => { + render(MeetingActions, { + client: createAPIClient(async () => Response.json(page({ coverage }))), + request, + }); + + expect(await screen.findByText(new RegExp(expected))).toBeDefined(); + }); + + it('aborts the old action request and never renders its stale response after scope changes', async () => { + let resolveFirst!: (response: Response) => void; + let firstRequest!: Request; + let requestCount = 0; + const fetchFn = vi.fn(async (input) => { + requestCount += 1; + const outgoing = input instanceof Request ? input : new Request(input); + if (requestCount === 1) { + firstRequest = outgoing; + return new Promise((resolve) => { + resolveFirst = resolve; + }); + } + return Response.json(page()); + }); + const client = createAPIClient(fetchFn); + const view = render(MeetingActions, { client, request }); + await waitFor(() => expect(firstRequest).toBeInstanceOf(Request)); + + await view.rerender({ + client, + request: { scope: { message_ids: [104] }, limit: 200 }, + }); + expect(firstRequest.signal.aborted).toBe(true); + resolveFirst( + Response.json( + page({ + rows: [ + { + meeting, + action: { + ordinal: 0, + locator: 'stale', + origin: 'source', + title: 'Stale action', + status: 'open', + }, + }, + ], + total_count: 1, + }), + ), + ); + + await screen.findByText('No recorded action items'); + expect(screen.queryByText('Stale action')).toBeNull(); + }); + + it('shows endpoint failures as explicit unavailable evidence', async () => { + render(MeetingActions, { + client: createAPIClient(async () => + Response.json( + { + error: 'meetings_unavailable', + message: 'Meeting intelligence is unavailable', + }, + { status: 503 }, + ), + ), + request, + }); + + expect((await screen.findByRole('alert')).textContent).toContain('Meeting intelligence is unavailable'); + }); +}); + + +it('continues reader pages, reconciles evidence and keeps a failed page visible', async () => { + let fail = true; + const row = (title: string, locator = 'action:0') => ({ meeting, action: { ordinal: 0, locator, origin: 'source', title, status: 'pending' } }); + const client = createAPIClient(async (input) => { + const outgoing = input as Request; + const body = await outgoing.json(); + if (!body.cursor) return Response.json(page({ rows: [row('Initial action')], total_count: 2, next_cursor: 'next' })); + if (fail) { fail = false; return Response.json({ error: 'unavailable', message: 'Continuation failed' }, { status: 503 }); } + return Response.json(page({ rows: [row('Updated action'), row('Final action', 'action:1')], total_count: 2 })); + }); + render(MeetingActions, { client, request }); + await screen.findByText('Initial action'); + expect(screen.getByText('Showing 1 of 2 action items')).toBeDefined(); + await fireEvent.click(screen.getByRole('button', { name: 'Load more action items' })); + expect((await screen.findByRole('alert')).textContent).toContain('Continuation failed'); + expect(screen.getByText('Initial action')).toBeDefined(); + await fireEvent.click(screen.getByRole('button', { name: 'Retry action items' })); + await screen.findByText('Final action'); + expect(screen.queryByText('Initial action')).toBeNull(); + expect(screen.getByText('Updated action')).toBeDefined(); + expect(screen.getByText('Showing 2 of 2 action items')).toBeDefined(); + expect(screen.queryByRole('button', { name: 'Load more action items' })).toBeNull(); +}); + +it('cancels a pending continuation and ignores its response after changing reader scope', async () => { + let resolveMore!: (response: Response) => void; + let pending!: Request; + const client = createAPIClient(async (input) => { + const outgoing = input as Request; + const body = await outgoing.json(); + if (body.cursor) { pending = outgoing; return new Promise((resolve) => { resolveMore = resolve; }); } + if (body.scope.message_ids[0] === 104) return Response.json(page()); + return Response.json(page({ rows: [{ meeting, action: { ordinal: 0, locator: 'first', origin: 'source', title: 'First action', status: 'pending' } }], total_count: 2, next_cursor: 'next' })); + }); + const view = render(MeetingActions, { client, request }); + await screen.findByText('First action'); + await fireEvent.click(screen.getByRole('button', { name: 'Load more action items' })); + await waitFor(() => expect(pending).toBeDefined()); + await view.rerender({ client, request: { scope: { message_ids: [104] }, limit: 200 } }); + expect(pending.signal.aborted).toBe(true); + resolveMore(Response.json(page({ rows: [{ meeting, action: { ordinal: 1, locator: 'stale', origin: 'source', title: 'Stale continuation', status: 'pending' } }], total_count: 2 }))); + await screen.findByText('No recorded action items'); + expect(screen.queryByText('First action')).toBeNull(); + expect(screen.queryByText('Stale continuation')).toBeNull(); +}); diff --git a/web/src/lib/components/meetings/MeetingContextExport.svelte b/web/src/lib/components/meetings/MeetingContextExport.svelte new file mode 100644 index 000000000..8aa05cfa9 --- /dev/null +++ b/web/src/lib/components/meetings/MeetingContextExport.svelte @@ -0,0 +1,164 @@ + + +
+ (format = value as MeetingContextRequestFormat)} + /> + (includeTranscript = checked)} + /> +
+ + diff --git a/web/src/lib/components/meetings/MeetingContextExport.test.ts b/web/src/lib/components/meetings/MeetingContextExport.test.ts new file mode 100644 index 000000000..2d1a4b277 --- /dev/null +++ b/web/src/lib/components/meetings/MeetingContextExport.test.ts @@ -0,0 +1,229 @@ +import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createAPIClient } from '../../api/client'; +import type { ExploreSelection, MeetingContextRequest } from '../../api/generated/models'; +import MeetingContextExport from './MeetingContextExport.svelte'; + +const selection: ExploreSelection = { + mode: 'explicit', + predicate: { + query: 'planning', + search_mode: 'full_text', + filters: [], + presentation: 'table', + }, + row_keys: ['message:7', 'message:91'], + cache_revision: 'cache-7', + search_provenance: { lexical_index_revision: 'fts-4' }, +}; + +describe('MeetingContextExport', () => { + let createObjectURL: ReturnType; + let revokeObjectURL: ReturnType; + let anchorClick: ReturnType; + + beforeEach(() => { + createObjectURL = vi.spyOn(URL, 'createObjectURL').mockReturnValue('blob:meeting-context'); + revokeObjectURL = vi.spyOn(URL, 'revokeObjectURL').mockImplementation(() => undefined); + anchorClick = vi.spyOn(HTMLAnchorElement.prototype, 'click').mockImplementation(() => undefined); + }); + + afterEach(() => vi.restoreAllMocks()); + + it('downloads exact Unicode content with transcript omitted by default and revokes the Blob URL', async () => { + const requests: Request[] = []; + const content = '{"title":"Café ☕","notes":"Привет 世界"}'; + const fetchFn = vi.fn(async (input) => { + requests.push(input instanceof Request ? input : new Request(input)); + return Response.json({ + schema_version: 1, + format: 'json', + content, + content_bytes: new TextEncoder().encode(content).byteLength, + truncated: false, + omitted_message_ids: [], + }); + }); + render(MeetingContextExport, { + client: createAPIClient(fetchFn), + request: { selection } satisfies MeetingContextRequest, + }); + + expect( + ( + screen.getByRole('checkbox', { + name: 'Include transcript', + }) as HTMLInputElement + ).checked, + ).toBe(false); + await fireEvent.click(screen.getByRole('button', { name: 'Export meeting context' })); + + await waitFor(() => expect(createObjectURL).toHaveBeenCalledOnce()); + await expect(requests[0]!.clone().json()).resolves.toEqual({ + selection, + format: 'json', + include_transcript: false, + }); + const blob = createObjectURL.mock.calls[0]![0] as Blob; + expect(blob.type).toBe('application/json'); + await expect(blob.text()).resolves.toBe(content); + expect((anchorClick.mock.instances[0] as HTMLAnchorElement).download).toBe('meeting-context.json'); + expect(revokeObjectURL).toHaveBeenCalledWith('blob:meeting-context'); + }); + + it('offers Markdown and reports explicit server truncation and omissions beside the download', async () => { + const content = '# Café\n\nCoverage: partial\n'; + const fetchFn = vi.fn(async () => + Response.json({ + schema_version: 1, + format: 'markdown', + content, + content_bytes: new TextEncoder().encode(content).byteLength, + truncated: true, + omitted_message_ids: [91, 104], + }), + ); + render(MeetingContextExport, { + client: createAPIClient(fetchFn), + request: { selection } satisfies MeetingContextRequest, + }); + anchorClick.mockImplementationOnce(() => { + expect(screen.getByText(/Export was truncated/)).toBeDefined(); + expect(screen.getByText(/91, 104/)).toBeDefined(); + }); + + await fireEvent.click(screen.getByRole('radio', { name: 'Markdown' })); + await fireEvent.click(screen.getByRole('checkbox', { name: 'Include transcript' })); + await fireEvent.click(screen.getByRole('button', { name: 'Export meeting context' })); + + expect(await screen.findByText(/Export was truncated/)).toBeDefined(); + expect(screen.getByText(/91, 104/)).toBeDefined(); + await waitFor(() => expect(createObjectURL).toHaveBeenCalledOnce()); + const blob = createObjectURL.mock.calls[0]![0] as Blob; + expect(blob.type).toBe('text/markdown'); + await expect(blob.text()).resolves.toBe(content); + expect((anchorClick.mock.instances[0] as HTMLAnchorElement).download).toBe('meeting-context.md'); + }); + + it('labels a delayed download with the format submitted before the control changes', async () => { + let resolveResponse!: (response: Response) => void; + let submittedRequest!: Request; + const content = '{"format":"json","title":"Caf\u00e9 \u2615"}'; + const fetchFn = vi.fn(async (input) => { + submittedRequest = input instanceof Request ? input : new Request(input); + return new Promise((resolve) => { + resolveResponse = resolve; + }); + }); + render(MeetingContextExport, { + client: createAPIClient(fetchFn), + request: { selection } satisfies MeetingContextRequest, + }); + + await fireEvent.click(screen.getByRole('button', { name: 'Export meeting context' })); + await waitFor(() => expect(resolveResponse).toBeTypeOf('function')); + await fireEvent.click(screen.getByRole('radio', { name: 'Markdown' })); + resolveResponse( + Response.json({ + schema_version: 1, + format: 'json', + content, + content_bytes: new TextEncoder().encode(content).byteLength, + truncated: false, + omitted_message_ids: [], + }), + ); + + await waitFor(() => expect(createObjectURL).toHaveBeenCalledOnce()); + await expect(submittedRequest.clone().json()).resolves.toMatchObject({ format: 'json' }); + const blob = createObjectURL.mock.calls[0]![0] as Blob; + expect(blob.type).toBe('application/json'); + await expect(blob.text()).resolves.toBe(content); + expect((anchorClick.mock.instances[0] as HTMLAnchorElement).download).toBe('meeting-context.json'); + }); + + it.each([ + ['selection_not_all_meetings', 400, 'Every row must be a meeting', 'Select meetings only'], + [ + 'meeting_selection_too_large', + 400, + 'Meeting context accepts at most 100 meetings', + 'Meeting context accepts at most 100 meetings', + ], + [ + 'not_found', + 404, + 'No route matches POST /api/v1/meetings/context', + 'Meeting context export is unavailable with this daemon', + ], + ['meeting_not_found', 404, 'Meeting 91 was deleted from this archive.', 'Meeting 91 was deleted from this archive.'], + ])('shows actionable state for %s', async (code, status, message, expected) => { + render(MeetingContextExport, { + client: createAPIClient(async () => Response.json({ error: code, message }, { status })), + request: { selection } satisfies MeetingContextRequest, + }); + + await fireEvent.click(screen.getByRole('button', { name: 'Export meeting context' })); + + expect((await screen.findByRole('alert')).textContent).toContain(expected); + expect(createObjectURL).not.toHaveBeenCalled(); + }); + + it('explains a known over-limit selection without sending it', () => { + const fetchFn = vi.fn(); + render(MeetingContextExport, { + client: createAPIClient(fetchFn), + request: { selection } satisfies MeetingContextRequest, + disabledReason: 'Meeting context accepts at most 100 meetings.', + }); + + expect(screen.getByText('Meeting context accepts at most 100 meetings.')).toBeDefined(); + expect( + ( + screen.getByRole('button', { + name: 'Export meeting context', + }) as HTMLButtonElement + ).disabled, + ).toBe(true); + expect(fetchFn).not.toHaveBeenCalled(); + }); + + it('aborts and discards a completed export after its selection authority changes', async () => { + let resolveFirst!: (response: Response) => void; + let firstRequest!: Request; + const fetchFn = vi.fn(async (input) => { + firstRequest = input instanceof Request ? input : new Request(input); + return new Promise((resolve) => { + resolveFirst = resolve; + }); + }); + const request = { selection } satisfies MeetingContextRequest; + const view = render(MeetingContextExport, { + client: createAPIClient(fetchFn), + request, + }); + await fireEvent.click(screen.getByRole('button', { name: 'Export meeting context' })); + await waitFor(() => expect(firstRequest).toBeInstanceOf(Request)); + + await view.rerender({ + client: createAPIClient(fetchFn), + request: { selection: { ...selection, row_keys: ['message:104'] } }, + }); + expect(firstRequest.signal.aborted).toBe(true); + resolveFirst( + Response.json({ + schema_version: 1, + format: 'json', + content: '{"stale":true}', + content_bytes: 14, + truncated: false, + omitted_message_ids: [], + }), + ); + await Promise.resolve(); + await Promise.resolve(); + + expect(createObjectURL).not.toHaveBeenCalled(); + }); +}); diff --git a/web/src/lib/components/meetings/MeetingMetrics.svelte b/web/src/lib/components/meetings/MeetingMetrics.svelte new file mode 100644 index 000000000..3eb854d42 --- /dev/null +++ b/web/src/lib/components/meetings/MeetingMetrics.svelte @@ -0,0 +1,63 @@ + + +
+

{metrics.totals.meeting_count.toLocaleString()} meetings

+

{metrics.totals.known_duration_count.toLocaleString()} known · {metrics.totals.unknown_duration_count.toLocaleString()} unknown duration

+
+
Total known meeting time
{duration(metrics.totals.total_known_seconds)}
+
Average known duration
{duration(metrics.totals.average_known_seconds)}
+
+

Time includes provider duration, scheduled, and transcript span evidence where available. Unknown durations are excluded from the average.

+ {#if metrics.duration_by_basis.length > 0} +
+ + + {#each metrics.duration_by_basis as basis (basis.basis)} + + {/each} +
EvidenceMeetingsKnown time
{basisLabels[basis.basis] ?? basis.basis}{basis.count}{basis.basis === 'unknown' ? 'Unavailable' : duration(basis.total_seconds)}
+
+ {/if} + {#if metrics.months.length > 0} +
+ + + {#each metrics.months as month (month.month)} + + {/each} +
MonthMeetingsKnownUnknownKnown timeAverage
{month.month}{month.totals.meeting_count}{month.totals.known_duration_count}{month.totals.unknown_duration_count}{duration(month.totals.total_known_seconds)}{duration(month.totals.average_known_seconds)}
+
+

Months without meetings are omitted.

+ {/if} + {#if metrics.undated_count > 0}

{metrics.undated_count} meetings have no date and are excluded from monthly rows.

{/if} +
+ + diff --git a/web/src/lib/components/meetings/MeetingMetrics.test.ts b/web/src/lib/components/meetings/MeetingMetrics.test.ts new file mode 100644 index 000000000..6bbbf010f --- /dev/null +++ b/web/src/lib/components/meetings/MeetingMetrics.test.ts @@ -0,0 +1,28 @@ +import { render, screen, within } from '@testing-library/svelte'; +import { describe, expect, it } from 'vitest'; +import { meetingMetrics } from '../../meetings/fixtures.test-support'; +import MeetingMetrics from './MeetingMetrics.svelte'; + +describe('MeetingMetrics', () => { + it('shows the 6000-second fixture, duration coverage, evidence bases, and monthly rows', () => { + render(MeetingMetrics, { metrics: meetingMetrics() }); + expect(screen.getByText('4 meetings')).toBeDefined(); + expect(screen.getByText('3 known · 1 unknown duration')).toBeDefined(); + expect(screen.getByLabelText('Total known meeting time').textContent).toContain('1h 40m'); + expect(screen.getByLabelText('Average known duration').textContent).toContain('33m 20s'); + expect(screen.queryByText(/recorded hours/i)).toBeNull(); + const bases = screen.getByRole('table', { name: 'Duration evidence' }); + expect(within(bases).getByRole('row', { name: /Scheduled.*1.*1h/ })).toBeDefined(); + expect(within(bases).getByRole('row', { name: /Transcript span.*1.*10m/ })).toBeDefined(); + const months = screen.getByRole('table', { name: 'Monthly meeting activity' }); + expect(within(months).getByRole('row', { name: /2026-01.*2.*2.*0.*1h 30m.*45m/ })).toBeDefined(); + expect(within(months).getByRole('row', { name: /2026-02.*2.*1.*1.*10m.*10m/ })).toBeDefined(); + }); + + it.each([0, 2])('keeps average unavailable for %s meetings without duration evidence', (count) => { + render(MeetingMetrics, { metrics: meetingMetrics({ totals: { meeting_count: count, known_duration_count: 0, + unknown_duration_count: count, total_known_seconds: 0, average_known_seconds: null }, duration_by_basis: [], months: [] }) }); + expect(screen.getByText(`${count} meetings`)).toBeDefined(); + expect(screen.getByLabelText('Average known duration').textContent).toContain('Unavailable'); + }); +}); diff --git a/web/src/lib/components/meetings/MeetingNavigation.test.ts b/web/src/lib/components/meetings/MeetingNavigation.test.ts new file mode 100644 index 000000000..2beb91899 --- /dev/null +++ b/web/src/lib/components/meetings/MeetingNavigation.test.ts @@ -0,0 +1,217 @@ +import { fireEvent, render, screen, waitFor, within } from '@testing-library/svelte'; +import { describe, expect, it, vi } from 'vitest'; +import { createAPIClient } from '../../api/client'; +import type { MessageDetail } from '../../api/generated/models'; +import { chooseSelectOption } from '../../../test/kit-ui'; +import { ExploreState } from '../../explore/state.svelte'; +import { meetingActions, meetingMetrics } from '../../meetings/fixtures.test-support'; +import AppShell from '../shell/AppShell.svelte'; + +const detail: MessageDetail = { id: 42, conversation_id: 8, message_type: 'meeting_transcript', subject: 'Archived review', sent_at: '2026-01-01T00:00:00Z', from: 'Example', to: [], body: 'Archived transcript body', labels: [], attachments: [], has_attachments: false, size_bytes: 10, snippet: 'Archived' }; +const action = { meeting: { message_id: 42, conversation_id: 8, source_id: 3, source_type: 'zoom', source_identifier: 'example-zoom', source_message_id: 'source-42', title: 'Action provenance title', occurred_at: detail.sent_at, archive_path: '/api/v1/messages/42' }, + action: { ordinal: 0, locator: 'action:0', origin: 'source', title: 'Prepare review', status: 'pending' } }; + +async function metricsResponse(request: Request): Promise { + const body = await request.clone().json(); + return Response.json(body.scope?.message_ids?.length === 1 + ? meetingMetrics({ totals: { meeting_count: 1, known_duration_count: 0, unknown_duration_count: 1, total_known_seconds: 0, average_known_seconds: null } }) + : meetingMetrics()); +} + +function handler(detailResponse: () => Response | Promise = () => Response.json(detail)) { + const requests: Request[] = []; + const client = createAPIClient(async (input) => { + const request = input instanceof Request ? input : new Request(input); requests.push(request); + const path = new URL(request.url).pathname; + if (path.endsWith('/meetings/metrics')) return metricsResponse(request); + if (path.endsWith('/meetings/actions')) return Response.json(meetingActions({ rows: [action], total_count: 1 })); + if (path.endsWith('/messages/42')) return detailResponse(); + if (path === '/api/v1/people/7') return Response.json({ id: 7, revision: 1, display_name: 'Example Person', participant_ids: [7], vcard_uid: 'person-7', created_at: '', updated_at: '' }); + if (path === '/api/v1/people/directory') return Response.json({ people: [] }); + if (path.endsWith('/brief-enrollment')) return Response.json({ person_id: 7, enrolled: false }); + if (path.endsWith('/tracking')) return Response.json({ person_id: 7, tracked: false }); + if (path.startsWith('/api/v1/people/') || path.startsWith('/api/v1/carddav/')) return Response.json({ error: 'not_found', message: 'Synthetic optional section unavailable.' }, { status: 404 }); + if (path.endsWith('/conversations/8')) return Response.json({ id: 8, messages: [detail], anchor_id: 42, total: 1, has_before: false, has_after: false }); + return Response.json({ rows: [], total_count: 0, cache_revision: 'cache-1', search_provenance: {} }); + }); + return { client, requests }; +} +function stateWithMeetingFilter(selectedRow: string | null = null): ExploreState { + window.history.replaceState(null, '', '/'); + const state = new ExploreState(window); + state.commitNavigation({ workspace: 'everything', selectedRow, filters: [{ dimension: 'message_type', values: ['meeting_transcript'] }, { dimension: 'source', values: ['3'] }] }); + return state; +} +const historyChange = () => new Promise((resolve) => window.addEventListener('popstate', () => resolve(), { once: true })); + +function groupNavigationClient() { + const second = { ...detail, id: 43, subject: 'Archived follow-up', body: 'Follow-up transcript body' }; + return createAPIClient(async (input) => { + const request = input instanceof Request ? input : new Request(input); + const url = new URL(request.url); + if (url.pathname.endsWith('/meetings/metrics')) return metricsResponse(request); + if (url.pathname.endsWith('/meetings/actions')) { + const body = await request.json(); + const messageID = body.scope?.message_ids?.[0]; + const rows = messageID === 43 ? [] : messageID === 42 + ? [{ ...action, meeting: { ...action.meeting, message_id: 43, archive_path: '/api/v1/messages/43' } }] : [action]; + return Response.json(meetingActions({ rows, total_count: rows.length })); + } + if (url.pathname.endsWith('/messages/42')) return Response.json(detail); + if (url.pathname.endsWith('/messages/43')) return Response.json(second); + if (url.pathname.endsWith('/conversations/8')) return Response.json({ id: 8, messages: [detail, second], + anchor_id: Number(url.searchParams.get('anchor')), total: 2, has_before: false, has_after: false }); + if (url.pathname.endsWith('/groups')) return Response.json({ rows: [{ key: 'exact.example', label: 'Exact domain', count: 2 }], + total_count: 1, cache_revision: 'group-cache', search_provenance: {} }); + if (url.pathname.endsWith('/files')) return Response.json({ files: [], total_count: 0, cache_revision: 'files-cache', search_provenance: {} }); + return Response.json({ rows: [], total_count: 0, cache_revision: 'outer-cache', search_provenance: {} }); + }); +} + +function stateWithGroup(): ExploreState { + window.history.replaceState(null, '', '/'); + const state = new ExploreState(window); + state.commitNavigation({ workspace: 'everything', selectedRow: 'group:domain:exact.example', + filters: [{ dimension: 'source', values: ['3'] }] }); + return state; +} + +describe('meeting archive reader navigation', () => { + it('preflights detail, opens the real reader, and restores scope and focus through Back/Forward and Close', async () => { + let resolveDetail!: (response: Response) => void; + const { client, requests } = handler(() => new Promise((resolve) => { resolveDetail = resolve; })); + const state = stateWithMeetingFilter(); + const view = render(AppShell, { client, state }); + await fireEvent.input(await screen.findByRole('textbox', { name: 'Assignee email' }), { target: { value: 'person@example.test' } }); + await chooseSelectOption(screen.getByRole('combobox', { name: /^Source status:/ }), 'Pending'); + await fireEvent.click(screen.getByRole('button', { name: 'Apply action filters' })); + const link = await screen.findByRole('link', { name: 'Open archived meeting' }); + const filters = state.current.filters; + link.focus(); + await fireEvent.click(link); + await waitFor(() => expect(resolveDetail).toBeDefined()); + expect(state.current.selectedRow).toBeNull(); + expect(screen.queryByRole('dialog', { name: 'Archived meeting' })).toBeNull(); + resolveDetail(Response.json(detail)); + const reader = await screen.findByRole('dialog', { name: 'Archived meeting' }); + expect(await within(reader).findByText('Archived transcript body')).toBeDefined(); + expect(within(reader).getByRole('complementary', { name: 'Reading pane: Archived review' })).toBeDefined(); + expect(state.current.workspace).toBe('everything'); + expect(state.current.filters).toEqual(filters); + expect(state.current.selectedRow).toBe('archive-meeting:42'); + const back = historyChange(); window.history.back(); await back; + await waitFor(() => expect(screen.queryByRole('dialog', { name: 'Archived meeting' })).toBeNull()); + expect(state.current.filters).toEqual(filters); + expect(screen.getByText('4 meetings')).toBeDefined(); + expect((screen.getByRole('textbox', { name: 'Assignee email' }) as HTMLInputElement).value).toBe('person@example.test'); + expect(screen.getByRole('combobox', { name: 'Source status: Pending' })).toBeDefined(); + const forward = historyChange(); window.history.forward(); await forward; + await screen.findByRole('dialog', { name: 'Archived meeting' }); + const close = historyChange(); + await fireEvent.click(screen.getByRole('button', { name: 'Close archived meeting' })); + await close; + await waitFor(() => expect(document.activeElement).toBe(link)); + const metricRequests = await Promise.all(requests.filter((request) => request.url.endsWith('/meetings/metrics')).map((request) => request.clone().json())); + expect(metricRequests.filter((body) => body.scope?.message_ids === undefined)).toHaveLength(2); + expect(metricRequests.filter((body) => body.scope?.message_ids?.[0] === 42)).toEqual([{ scope: { message_ids: [42] } }]); + view.unmount(); state.destroy(); + }); + + it('restores an archive marker from a deep URL without fabricating a list row or draining pages', async () => { + const { client, requests } = handler(); + window.history.replaceState(null, '', `/?explore=${encodeURIComponent(JSON.stringify({ workspace: 'everything', selectedRow: 'archive-meeting:42' }))}`); + const state = new ExploreState(window); + const view = render(AppShell, { client, state }); + const reader = await screen.findByRole('dialog', { name: 'Archived meeting' }); + expect(await within(reader).findByText('Archived transcript body')).toBeDefined(); + expect(requests.some((request) => request.url.endsWith('/messages/42'))).toBe(true); + await fireEvent.click(screen.getByRole('button', { name: 'Close archived meeting' })); + await waitFor(() => expect(state.current.selectedRow).toBeNull()); + view.unmount(); state.destroy(); + }); + + it('does not navigate after the originating scope changes while getMessage ignores abort', async () => { + let resolveDetail!: (response: Response) => void; + const { client, requests } = handler(() => new Promise((resolve) => { resolveDetail = resolve; })); + const state = stateWithMeetingFilter(); const view = render(AppShell, { client, state }); + await fireEvent.click(await screen.findByRole('link', { name: 'Open archived meeting' })); + await waitFor(() => expect(resolveDetail).toBeDefined()); + state.commitNavigation({ filters: [{ dimension: 'source', values: ['4'] }] }); + await waitFor(() => expect(requests.find((request) => request.url.endsWith('/messages/42'))!.signal.aborted).toBe(true)); + resolveDetail(Response.json(detail)); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(state.current.selectedRow).toBeNull(); + expect(screen.queryByRole('dialog', { name: 'Archived meeting' })).toBeNull(); + view.unmount(); state.destroy(); + }); + + it('shows missing archive detail without navigating to an invented row', async () => { + const { client } = handler(() => Response.json({ error: 'not_found', message: 'Archived meeting missing.' }, { status: 404 })); + const state = stateWithMeetingFilter(); const view = render(AppShell, { client, state }); + await fireEvent.click(await screen.findByRole('link', { name: 'Open archived meeting' })); + expect(await screen.findByText('Archived meeting missing.')).toBeDefined(); + expect(state.current.selectedRow).toBeNull(); + view.unmount(); state.destroy(); + }); + + it('returns to the selected group after expanding another message and transient reader updates', async () => { + const state = stateWithGroup(); + const view = render(AppShell, { client: groupNavigationClient(), state }); + try { + const source = await screen.findByRole('link', { name: 'Open archived meeting' }); + source.focus(); + await fireEvent.click(source); + const reader = await screen.findByRole('dialog', { name: 'Archived meeting' }); + await fireEvent.click(await within(reader).findByRole('button', { name: 'Expand message 43 from Example' })); + await waitFor(() => expect(state.current.conversationAnchor).toBe('43')); + expect(await within(reader).findByText('Follow-up transcript body')).toBeDefined(); + state.replaceTransient({ columnWidths: { title: 432 } }); + await fireEvent.click(within(reader).getByRole('button', { name: 'Close archived meeting' })); + await waitFor(() => expect(state.current.selectedRow).toBe('group:domain:exact.example')); + expect(screen.getByRole('complementary', { name: 'Reading pane: Exact domain' })).toBeDefined(); + expect(state.current.filters).toEqual([{ dimension: 'source', values: ['3'] }]); + expect(state.current.conversationAnchor).toBeNull(); + await waitFor(() => expect(document.activeElement).toBe(source)); + } finally { view.unmount(); state.destroy(); } + }); + + it('keeps the first reader history ownership through a nested reader, Back, and Close', async () => { + const state = stateWithGroup(); + const view = render(AppShell, { client: groupNavigationClient(), state }); + try { + await fireEvent.click(await screen.findByRole('link', { name: 'Open archived meeting' })); + const firstReader = await screen.findByRole('dialog', { name: 'Archived meeting' }); + await fireEvent.click(await within(firstReader).findByRole('link', { name: 'Open archived meeting' })); + await screen.findByRole('complementary', { name: 'Reading pane: Archived follow-up' }); + expect(state.current.selectedRow).toBe('archive-meeting:43'); + const back = historyChange(); window.history.back(); await back; + await screen.findByRole('complementary', { name: 'Reading pane: Archived review' }); + expect(state.current.selectedRow).toBe('archive-meeting:42'); + await fireEvent.click(screen.getByRole('button', { name: 'Close archived meeting' })); + await waitFor(() => expect(state.current.selectedRow).toBe('group:domain:exact.example')); + expect(screen.getByRole('complementary', { name: 'Reading pane: Exact domain' })).toBeDefined(); + expect(state.current.filters).toEqual([{ dimension: 'source', values: ['3'] }]); + } finally { view.unmount(); state.destroy(); } + }); +}); + + it.each([false, true])('opens Directory meeting actions through the shell and returns to the person (narrow=%s)', async (narrow) => { + vi.stubGlobal('matchMedia', () => ({ matches: narrow, addEventListener: vi.fn(), removeEventListener: vi.fn() })); + const { client } = handler(); + window.history.replaceState(null, '', '/'); + const state = new ExploreState(window); + state.commitNavigation({ workspace: 'directory', directoryPersonID: 7 }); + const view = render(AppShell, { client, state }); + try { + const link = await screen.findByRole('link', { name: 'Open archived meeting' }); + await fireEvent.click(link); + const reader = await screen.findByRole('dialog', { name: 'Archived meeting' }); + expect(await within(reader).findByText('Archived transcript body')).toBeDefined(); + expect(state.current.workspace).toBe('directory'); + expect(state.current.directoryPersonID).toBe(7); + const close = historyChange(); await fireEvent.click(within(reader).getByRole('button', { name: 'Close archived meeting' })); await close; + await waitFor(() => expect(screen.queryByRole('dialog', { name: 'Archived meeting' })).toBeNull()); + expect(state.current.directoryPersonID).toBe(7); + expect(screen.getByText('4 meetings')).toBeDefined(); + } finally { view.unmount(); state.destroy(); vi.unstubAllGlobals(); } + }); diff --git a/web/src/lib/components/meetings/MeetingPanel.svelte b/web/src/lib/components/meetings/MeetingPanel.svelte new file mode 100644 index 000000000..692a6f68b --- /dev/null +++ b/web/src/lib/components/meetings/MeetingPanel.svelte @@ -0,0 +1,77 @@ + + +
+

Meeting activity and follow-ups

+ {#if controller.metricsLoading}

Loading meeting metrics…

{/if} + {#each errors as error (error.message)}

{error.message}

{/each} + {#if canReload}
+ + diff --git a/web/src/lib/components/meetings/MeetingPanel.test.ts b/web/src/lib/components/meetings/MeetingPanel.test.ts new file mode 100644 index 000000000..d5dba4ecc --- /dev/null +++ b/web/src/lib/components/meetings/MeetingPanel.test.ts @@ -0,0 +1,71 @@ +import { fireEvent, render, screen, waitFor, within } from '@testing-library/svelte'; +import { describe, expect, it, vi } from 'vitest'; +import { createAPIClient } from '../../api/client'; +import { meetingAction, meetingActions, meetingMetrics } from '../../meetings/fixtures.test-support'; +import MeetingPanel from './MeetingPanel.svelte'; + +function response(request: Request): Response { + return Response.json(new URL(request.url).pathname.endsWith('/metrics') ? meetingMetrics() : meetingActions()); +} + +describe('MeetingPanel', () => { + it('shows metrics and scoped source evidence with page-one source filters', async () => { + const requests: Request[] = []; + render(MeetingPanel, { client: createAPIClient(async (input) => { + const request = input instanceof Request ? input : new Request(input); requests.push(request); return response(request); + }), scope: { kind: 'direct', scope: { person_id: 7 } } }); + expect(await screen.findByText('4 meetings')).toBeDefined(); + expect(await screen.findByText('Coverage: 1 available · 1 partial · 1 unsupported · 1 unavailable')).toBeDefined(); + expect(screen.getByRole('combobox', { name: 'Source status: All source statuses' })).toBeDefined(); + await fireEvent.input(screen.getByRole('textbox', { name: 'Assignee email' }), { target: { value: 'person@example.test' } }); + await fireEvent.click(screen.getByRole('button', { name: 'Apply action filters' })); + await waitFor(() => expect(requests.filter((request) => request.url.endsWith('/actions'))).toHaveLength(2)); + const last = requests.filter((request) => request.url.endsWith('/actions')).at(-1)!; + await expect(last.clone().json()).resolves.toEqual({ scope: { person_id: 7 }, assignee_email: 'person@example.test', limit: 50 }); + }); + + it('asks the owner for fresh Explore authority on explicit reload without repeating failed authority', async () => { + const onReloadScope = vi.fn(); + const fetchFn = vi.fn(async () => Response.json({ error: 'meeting_scope_changed', message: 'Meetings changed.' }, { status: 409 })); + const props = { client: createAPIClient(fetchFn), scope: { kind: 'explore' as const, explore: { + predicate: { filters: [] }, cache_revision: 'old-cache', search_provenance: {} + } }, onReloadScope }; + const view = render(MeetingPanel, props); + await screen.findByText(/Meetings changed.*Reload meeting activity/); + await fireEvent.click(screen.getByRole('button', { name: 'Reload meeting activity' })); + expect(onReloadScope).toHaveBeenCalledTimes(1); + expect(fetchFn).toHaveBeenCalledTimes(2); + await view.rerender({ ...props, scope: { ...props.scope, explore: { ...props.scope.explore, cache_revision: 'new-cache' } } }); + await waitFor(() => expect(fetchFn).toHaveBeenCalledTimes(4)); + }); + + it('keeps scope-too-large guidance visible and does not offer a looping reload', async () => { + const fetchFn = vi.fn(async () => Response.json({ error: 'meeting_scope_too_large', message: 'Over 10000.' }, { status: 400 })); + render(MeetingPanel, { client: createAPIClient(fetchFn), scope: { kind: 'direct', scope: { domains: ['example.test'] } } }); + expect(await screen.findByText(/Choose narrower filters/)).toBeDefined(); + expect(screen.queryByRole('button', { name: 'Reload meeting activity' })).toBeNull(); + expect(fetchFn).toHaveBeenCalledTimes(2); + }); + + it('renders updated evidence once when a source edit repeats an action on the next page', async () => { + const first = meetingAction(); + const updated = meetingAction({ action: { ...first.action, title: 'Review prepared', status: 'completed' } }); + const distinct = meetingAction({ action: { ...first.action, ordinal: 1, locator: 'action:1', title: 'Send notes' } }); + render(MeetingPanel, { client: createAPIClient(async (input) => { + const request = input instanceof Request ? input : new Request(input); + if (request.url.endsWith('/metrics')) return Response.json(meetingMetrics()); + const body = await request.json(); + return Response.json(body.cursor ? meetingActions({ rows: [updated, distinct], total_count: 2 }) + : meetingActions({ rows: [first], total_count: 2, next_cursor: 'second-page' })); + }), scope: { kind: 'direct', scope: { person_id: 7 } } }); + await screen.findByText('Prepare review'); + await fireEvent.click(screen.getByRole('button', { name: 'Load more action items' })); + await screen.findByText('Review prepared'); + const evidence = screen.getByRole('region', { name: 'Archived action items' }); + expect(within(evidence).getAllByRole('listitem')).toHaveLength(2); + expect(within(evidence).getByText('completed')).toBeDefined(); + expect(within(evidence).getByText('Send notes')).toBeDefined(); + expect(screen.queryByText('Prepare review')).toBeNull(); + expect(screen.queryByRole('button', { name: 'Load more action items' })).toBeNull(); + }); +}); diff --git a/web/src/lib/components/meetings/MeetingSurfaces.test.ts b/web/src/lib/components/meetings/MeetingSurfaces.test.ts new file mode 100644 index 000000000..b02ac0ae3 --- /dev/null +++ b/web/src/lib/components/meetings/MeetingSurfaces.test.ts @@ -0,0 +1,198 @@ +import { fireEvent, render, screen, waitFor, within } from '@testing-library/svelte'; +import { describe, expect, it, vi } from 'vitest'; +import { createAPIClient } from '../../api/client'; +import type { MeetingMetricsRequest } from '../../api/generated/models'; +import type { DirectoryReadBundle } from '../../directory/models'; +import { ExploreState } from '../../explore/state.svelte'; +import { meetingActions, meetingMetrics } from '../../meetings/fixtures.test-support'; +import { RelationshipsController } from '../../relationships/controller.svelte'; +import PersonDetail from '../directory/PersonDetail.svelte'; +import RelationshipsWorkspace from '../relationships/RelationshipsWorkspace.svelte'; +import AppShell from '../shell/AppShell.svelte'; + +const person = { id: 7, revision: 1, display_name: 'Example Person', participant_ids: [3, 7], vcard_uid: 'person-7', created_at: '', updated_at: '' }; +const at = '2026-01-01T00:00:00Z'; +function ancillary(path: string): Response { + if (path.endsWith('/tracking')) return Response.json({ person_id: 7, tracked: false }); + if (path.endsWith('/brief-enrollment')) return Response.json({ person_id: 7, enrolled: false }); + if (path.endsWith('/merges')) return Response.json({ merges: [], limit: 100, offset: 0 }); + return Response.json({ error: 'not_found', message: 'Synthetic unavailable resource.' }, { status: 404 }); +} + +describe('meeting panel surfaces', () => { + it('refreshes same Directory person identity revisions and ignores delayed pre-merge evidence', async () => { + const metrics: MeetingMetricsRequest[] = []; + let resolveOld!: (response: Response) => void; + let oldSignal!: AbortSignal; + const client = createAPIClient(async (input) => { + const request = input instanceof Request ? input : new Request(input); + const path = new URL(request.url).pathname; + if (path.endsWith('/meetings/metrics')) { + metrics.push(await request.clone().json()); + if (metrics.length === 1) { oldSignal = request.signal; return new Promise((resolve) => { resolveOld = resolve; }); } + return Response.json(meetingMetrics()); + } + if (path.endsWith('/meetings/actions')) return Response.json(meetingActions()); + return ancillary(path); + }); + const bundle: DirectoryReadBundle = { person, etags: {}, errors: {} }; + const view = render(PersonDetail, { client, personID: 7, bundle }); + await waitFor(() => expect(resolveOld).toBeDefined()); + await view.rerender({ client, personID: 7, bundle: { ...bundle, person: { ...person, revision: 2, participant_ids: [7, 9] } } }); + expect(await screen.findByText('4 meetings')).toBeDefined(); + expect(metrics).toEqual([{ scope: { person_id: 7 } }, { scope: { person_id: 7 } }]); + expect(oldSignal.aborted).toBe(true); + resolveOld(Response.json(meetingMetrics({ totals: { ...meetingMetrics().totals, meeting_count: 99 } }))); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(screen.queryByText('99 meetings')).toBeNull(); + }); + + it.each(['cluster:7', 'domain:exact.example'])('uses the active Relationships scope for %s', async (target) => { + const metrics: MeetingMetricsRequest[] = []; + const client = createAPIClient(async (input) => { + const request = input instanceof Request ? input : new Request(input); const path = new URL(request.url).pathname; + if (path.endsWith('/meetings/metrics')) { metrics.push(await request.clone().json()); return Response.json(meetingMetrics()); } + if (path.endsWith('/meetings/actions')) return Response.json(meetingActions()); + if (path.endsWith('/timeline')) return Response.json({ canonical_id: 7, identity_revision: 3, cache_revision: 'rel-cache', rows: [], total_count: 0 }); + if (path.endsWith('/summary')) return Response.json({ summary: { id: 7, display_label: 'Example Person', domain: 'exact.example', activity_count: 0, file_count: 0, identifiers: [], source_counts: [], cache_revision: 'rel-cache' } }); + if (path.endsWith('/participants/7')) return Response.json({ id: 7, display_label: 'Example Person', identifiers: [], activity_count: 0, file_count: 0, source_counts: [], first_at: at, last_at: at, cache_revision: 'rel-cache' }); + if (path.endsWith('/domains/exact.example')) return Response.json({ domain: 'exact.example', activity_count: 0, file_count: 0, source_counts: [], cache_revision: 'rel-cache' }); + if (path.endsWith('/relationships')) return Response.json({ rows: [] }); + return ancillary(path); + }); + const controller = new RelationshipsController(client, () => 'UTC'); + const predicate = { query: 'stale URL text', search_mode: 'hybrid' as const, filters: [ + { dimension: 'source' as const, values: ['2'] }, { dimension: 'after' as const, values: [at] } + ] }; + const view = render(RelationshipsWorkspace, { props: { client, controller, target, predicate, facet: target.startsWith('domain:') ? 'domains' : 'people', showAll: false, filesOpen: false, + onFacetChange: vi.fn(), onTargetChange: vi.fn(), onShowAllChange: vi.fn(), onFilesToggle: vi.fn() } }); + await controller.openTarget(target, predicate); + await screen.findByText('4 meetings'); + expect(metrics.at(-1)).toEqual({ scope: { ...(target.startsWith('domain:') ? { domains: ['exact.example'] } : { participant_id: 7 }), source_ids: [2], after: at } }); + view.unmount(); controller.destroy(); + }); + + it('uses separately loaded exact group authority across a workspace round-trip', async () => { + window.history.replaceState(null, '', '/'); + const metrics: MeetingMetricsRequest[] = []; + let groupLoads = 0; + const client = createAPIClient(async (input) => { + const request = input instanceof Request ? input : new Request(input); const path = new URL(request.url).pathname; + if (path.endsWith('/meetings/metrics')) { metrics.push(await request.clone().json()); return Response.json(meetingMetrics()); } + if (path.endsWith('/meetings/actions')) return Response.json(meetingActions()); + if (path.endsWith('/explore/groups')) { + const body = await request.clone().json(); + if (body.group_key) { + groupLoads += 1; + return Response.json({ rows: [ { key: 'frequent.example', label: 'Frequent domain', count: 20, estimated_bytes: 200 }, { key: 'exact.example', label: 'Exact domain', count: 2, estimated_bytes: 20 } ], total_count: 2, + cache_revision: 'detail-cache', search_provenance: { lexical_index_revision: 'detail-lex', vector_generation: 2 }, candidate_snapshot_id: 'detail-candidate' }); + } + } + if (path.endsWith('/explore/files')) return Response.json({ files: [], total_count: 0, cache_revision: 'files-cache', search_provenance: {} }); + if (path.endsWith('/explore')) return Response.json({ rows: [], total_count: 0, cache_revision: 'outer-cache', search_provenance: { lexical_index_revision: 'outer-lex', vector_generation: 1 }, candidate_snapshot_id: 'outer-candidate' }); + return ancillary(path); + }); + const state = new ExploreState(window); + const filters = [{ dimension: 'domain' as const, values: ['co.example'] }, { dimension: 'participant' as const, values: ['3'] }, { dimension: 'participant' as const, values: ['7'] }]; + state.commitNavigation({ workspace: 'everything', query: 'planning', searchMode: 'hybrid', filters, selectedRow: 'group:domain:exact.example' }); + const view = render(AppShell, { client, state }); + await screen.findByText('4 meetings'); + expect(metrics[0]).toMatchObject({ explore: { cache_revision: 'detail-cache', search_provenance: { lexical_index_revision: 'detail-lex', vector_generation: 2 }, candidate_snapshot_id: 'detail-candidate', + predicate: { query: 'planning', search_mode: 'hybrid', filters: [...filters, { dimension: 'domain', values: ['exact.example'] }] } } }); + await fireEvent.click(within(screen.getByRole('navigation', { name: 'Primary' })).getByRole('button', { name: 'Settings' })); + window.history.back(); + await new Promise((resolve) => window.addEventListener('popstate', resolve, { once: true })); + await screen.findByText('4 meetings'); + expect(metrics.at(-1)).toEqual(metrics[0]); + expect(groupLoads).toBe(1); + view.unmount(); state.destroy(); + }); + + it.each(['table', 'groups'])('exposes meeting-filtered Everything %s with the complete predicate and loaded authority', async (presentation) => { + window.history.replaceState(null, '', '/'); + const metrics: MeetingMetricsRequest[] = []; + const client = createAPIClient(async (input) => { + const request = input instanceof Request ? input : new Request(input); const path = new URL(request.url).pathname; + if (path.endsWith('/meetings/metrics')) { metrics.push(await request.clone().json()); return Response.json(meetingMetrics()); } + if (path.endsWith('/meetings/actions')) return Response.json(meetingActions()); + return Response.json({ rows: [], total_count: 0, cache_revision: 'everything-cache', search_provenance: { lexical_index_revision: 'lexical-1' } }); + }); + const state = new ExploreState(window); + const filters = [{ dimension: 'message_type' as const, values: ['meeting_transcript'] }, { dimension: 'source' as const, values: ['3'] }, { dimension: 'domain' as const, values: ['exact.example'] }]; + state.commitNavigation({ workspace: 'everything', groupingChain: presentation === 'groups' ? ['domain'] : [], filters, query: 'planning', searchMode: 'full_text' }); + const view = render(AppShell, { client, state }); + await screen.findByText('4 meetings'); + expect(metrics[0]).toMatchObject({ explore: { predicate: { ...state.predicate(), filters }, cache_revision: 'everything-cache', search_provenance: { lexical_index_revision: 'lexical-1' } } }); + view.unmount(); state.destroy(); + }); +}); + +it('aborts a prior group lookup immediately when the predicate changes while the new list is still loading', async () => { + window.history.replaceState(null, '', '/'); + let oldGroupSignal!: AbortSignal; + let groupRequests = 0; + let resolveOldGroup!: (response: Response) => void; + let resolveNewList!: (response: Response) => void; + let lists = 0; + const client = createAPIClient(async (input) => { + const request = input instanceof Request ? input : new Request(input); const path = new URL(request.url).pathname; + if (path.endsWith('/explore')) { + lists += 1; + if (lists > 1) return new Promise((resolve) => { resolveNewList = resolve; }); + return Response.json({ rows: [], total_count: 0, cache_revision: 'old-cache', search_provenance: {} }); + } + if (path.endsWith('/explore/groups')) { + groupRequests += 1; + oldGroupSignal = request.signal; + return new Promise((resolve) => { resolveOldGroup = resolve; }); + } + if (path.endsWith('/meetings/metrics')) return Response.json(meetingMetrics()); + if (path.endsWith('/meetings/actions')) return Response.json(meetingActions()); + return ancillary(path); + }); + const state = new ExploreState(window); + state.commitNavigation({ workspace: 'everything', selectedRow: 'group:domain:exact.example', filters: [{ dimension: 'source', values: ['2'] }] }); + const view = render(AppShell, { client, state }); + try { + await waitFor(() => expect(resolveOldGroup).toBeDefined()); + state.commitNavigation({ filters: [{ dimension: 'source', values: ['3'] }] }); + await waitFor(() => expect(resolveNewList).toBeDefined()); + expect(groupRequests).toBe(1); + expect(oldGroupSignal.aborted).toBe(true); + resolveOldGroup(Response.json({ rows: [{ key: 'exact.example', label: 'Old scope', count: 99, estimated_bytes: 0 }], total_count: 1, cache_revision: 'old-cache', search_provenance: {} })); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(screen.queryByRole('complementary', { name: 'Reading pane: Old scope' })).toBeNull(); + expect(screen.queryByText('4 meetings')).toBeNull(); + } finally { view.unmount(); state.destroy(); } +}); + +it('reloads a failed meeting overview from a freshly loaded matching authority even when the cache revision is unchanged', async () => { + window.history.replaceState(null, '', '/'); + const metrics: MeetingMetricsRequest[] = []; + let listLoads = 0; + let canLoad = false; + const client = createAPIClient(async (input) => { + const request = input instanceof Request ? input : new Request(input); const path = new URL(request.url).pathname; + if (path.endsWith('/explore')) { + listLoads += 1; + if (listLoads > 1) canLoad = true; + return Response.json({ rows: [], total_count: 0, cache_revision: 'same-cache', search_provenance: {} }); + } + if (path.endsWith('/meetings/metrics')) { + metrics.push(await request.clone().json()); + return canLoad ? Response.json(meetingMetrics()) : Response.json({ error: 'meeting_scope_changed', message: 'Archived scope changed.' }, { status: 409 }); + } + if (path.endsWith('/meetings/actions')) return Response.json(meetingActions()); + return ancillary(path); + }); + const state = new ExploreState(window); + state.commitNavigation({ workspace: 'everything', filters: [{ dimension: 'message_type', values: ['meeting_transcript'] }] }); + const view = render(AppShell, { client, state }); + try { + await fireEvent.click(await screen.findByRole('button', { name: 'Reload meeting activity' })); + expect(await screen.findByText('4 meetings')).toBeDefined(); + expect(listLoads).toBe(2); + expect(metrics).toHaveLength(2); + expect(metrics[1]).toEqual(metrics[0]); + } finally { view.unmount(); state.destroy(); } +}); diff --git a/web/src/lib/components/operations/OperationHostSetup.svelte b/web/src/lib/components/operations/OperationHostSetup.svelte new file mode 100644 index 000000000..78fbc6297 --- /dev/null +++ b/web/src/lib/components/operations/OperationHostSetup.svelte @@ -0,0 +1,14 @@ + + +

+ {setup.text} + {setup.guideLabel} +

+ + diff --git a/web/src/lib/components/operations/OperationLaneCards.svelte b/web/src/lib/components/operations/OperationLaneCards.svelte deleted file mode 100644 index 4daa01efb..000000000 --- a/web/src/lib/components/operations/OperationLaneCards.svelte +++ /dev/null @@ -1,203 +0,0 @@ - - -
- {#each lanes as lane (lane.lane)} -
-

{laneLabels[lane.lane]}

- {#if lane.kinds.length === 0} -

Status unavailable

- {:else} -
- {#each lane.kinds as kind (kind.kind)} -
-

{kindLabels[kind.kind]}

-
- - - {kind.configured ? 'Configured' : 'Not configured'} - - - - {kind.history_availability === 'available' ? 'History available' : 'History unavailable'} - -
-
- {#if kind.active} -
-
Active
-
- - {stateLabel(kind.active)} - -
-
- {/if} - {#if kind.latest} -
-
Latest
-
- - {stateLabel(kind.latest)} - -
-
- {/if} - {#if kind.latest_successful} -
-
Last successful
-
- - Succeeded - -
-
- {/if} -
- {#if kind.history_availability === 'available' && !kind.active && !kind.latest && !kind.latest_successful} -

No recorded runs

- {/if} - {#if kind.related_status || kind.supported_actions.length > 0} -
- {#if kind.related_status} -
- {/if} -
- {/each} -
- {/if} -
- {/each} -
- - diff --git a/web/src/lib/components/operations/OperationRelatedStatus.svelte b/web/src/lib/components/operations/OperationRelatedStatus.svelte index f4c604b4b..abc6ca60e 100644 --- a/web/src/lib/components/operations/OperationRelatedStatus.svelte +++ b/web/src/lib/components/operations/OperationRelatedStatus.svelte @@ -4,7 +4,7 @@ getDocumentVectorStatus as generatedGetDocumentVectorStatus, getVisualAttachmentStatus as generatedGetVisualAttachmentStatus, } from '../../api/generated/api/api'; - import { Button, StatusDot } from '@kenn-io/kit-ui'; + import { Button, Chip, StatusDot } from '@kenn-io/kit-ui'; import { onMount } from 'svelte'; import type { APIClient } from '../../api/client'; @@ -14,6 +14,8 @@ Status as GeneratedVisualStatus, } from '../../api/generated/models'; import type { OperationStatusAuthority } from '../../explore/models'; + import { DOCUMENT_INDEX_SETUP, DOCUMENT_SEARCH_SETUP } from '../../operations/labels'; + import OperationHostSetup from './OperationHostSetup.svelte'; type DocumentStatus = GeneratedDocumentIndexStatusResponse; type DocumentVectorStatus = GeneratedDocumentVectorOperationsResponse; @@ -30,7 +32,7 @@ authority: OperationStatusAuthority; configured?: boolean; onClose?: () => void; - onConfigure?: (authority: OperationStatusAuthority) => void; + onConfigure?: () => void; } = $props(); let loading = $state(true); @@ -51,12 +53,6 @@ getVisualAttachmentStatus: 'Visual attachments need configuration' }; - const settingsLabels: Record = { - getDocumentIndexStatus: 'Open document index settings', - getDocumentVectorStatus: 'Open document vector settings', - getVisualAttachmentStatus: 'Open visual attachment settings' - }; - onMount(() => { if (configured === false) { loading = false; @@ -99,7 +95,7 @@ diff --git a/web/src/lib/components/operations/OperationRelatedStatus.test.ts b/web/src/lib/components/operations/OperationRelatedStatus.test.ts index d0d71e16e..257c582a9 100644 --- a/web/src/lib/components/operations/OperationRelatedStatus.test.ts +++ b/web/src/lib/components/operations/OperationRelatedStatus.test.ts @@ -58,36 +58,71 @@ describe('OperationRelatedStatus', () => { expect(document.body.textContent).not.toContain('private'); }); + it('does not fetch unconfigured visual attachments and instead opens their Settings', async () => { + const onConfigure = vi.fn(); + const fetchFn = vi.fn(); + render(OperationRelatedStatus, { + client: createAPIClient(fetchFn), authority: 'getVisualAttachmentStatus', configured: false, + onClose: vi.fn(), onConfigure + }); + + await fireEvent.click(await screen.findByRole('button', { name: 'Open visual attachment settings' })); + expect(onConfigure).toHaveBeenCalledWith(); + expect(fetchFn).not.toHaveBeenCalled(); + }); + it.each([ - ['missing or disabled document configuration', 'getDocumentIndexStatus', 'Open document index settings'], - ['unconfigured visual attachments', 'getVisualAttachmentStatus', 'Open visual attachment settings'] - ] as const)('does not fetch %s and instead opens its Settings authority', async ( - _case, authority, settingsLabel + ['getDocumentIndexStatus', 'Configured in config.toml on the daemon host.', 'Document indexing setup'], + ['getDocumentVectorStatus', 'Configured in config.toml on the daemon host. Also needs semantic search.', + 'Document search setup'] + ] as const)('explains host configuration for unconfigured %s without a request', async ( + authority, text, linkName ) => { - const onConfigure = vi.fn(); const fetchFn = vi.fn(); render(OperationRelatedStatus, { - client: createAPIClient(fetchFn), authority, configured: false, onClose: vi.fn(), onConfigure + client: createAPIClient(fetchFn), authority, configured: false, onClose: vi.fn(), onConfigure: vi.fn() }); - await fireEvent.click(await screen.findByRole('button', { name: settingsLabel })); - expect(onConfigure).toHaveBeenCalledWith(authority); + expect(await screen.findByText('Off')).toBeDefined(); + expect(screen.getByText(text, { exact: false })).toBeDefined(); + const link = screen.getByRole('link', { name: linkName }); + expect(link.getAttribute('target')).toBe('_blank'); + expect(link.getAttribute('rel')).toBe('noreferrer'); + expect(screen.queryByRole('button', { name: /settings/i })).toBeNull(); expect(fetchFn).not.toHaveBeenCalled(); }); - it('offers Settings only when a status response proves configuration needs attention', async () => { - const onConfigure = vi.fn(); - const fetchFn = vi.fn(async () => Response.json({ - ...documentStatus, - status: { ...documentStatus.status, profile_enabled: false, exact_consent: false } - })); + it.each([ + ['getDocumentIndexStatus', { + ...documentStatus, status: { ...documentStatus.status, profile_enabled: false } + }, 'Document indexing setup', + 'https://msgvault.io/docs/usage/document-indexing/#configure-the-policy'], + ['getDocumentVectorStatus', { enabled: false, configured: true }, 'Document search setup', + 'https://msgvault.io/docs/usage/document-indexing/#semantic-and-hybrid-document-search'] + ] as const)('links %s to its host setup guide when the status needs configuration', async ( + authority, response, linkName, href + ) => { + const fetchFn = vi.fn(async () => Response.json(response)); render(OperationRelatedStatus, { - client: createAPIClient(fetchFn), authority: 'getDocumentIndexStatus', - configured: true, onClose: vi.fn(), onConfigure + client: createAPIClient(fetchFn), authority, configured: true, onClose: vi.fn(), onConfigure: vi.fn() + }); + + const link = await screen.findByRole('link', { name: linkName }); + expect(link.getAttribute('href')).toBe(href); + expect(link.getAttribute('target')).toBe('_blank'); + expect(link.getAttribute('rel')).toBe('noreferrer'); + expect(screen.queryByRole('button', { name: /settings/i })).toBeNull(); + }); + + it('omits the host setup guide when document status is fully configured', async () => { + const fetchFn = vi.fn(async () => Response.json(documentStatus)); + render(OperationRelatedStatus, { + client: createAPIClient(fetchFn), authority: 'getDocumentIndexStatus', configured: true, + onClose: vi.fn(), onConfigure: vi.fn() }); - await fireEvent.click(await screen.findByRole('button', { name: 'Open document index settings' })); - expect(onConfigure).toHaveBeenCalledWith('getDocumentIndexStatus'); + expect(await screen.findByText('4 of 5 owners ready')).toBeDefined(); + expect(screen.queryByRole('link', { name: 'Document indexing setup' })).toBeNull(); }); it('keeps endpoint failures on the status authority with fixed retry and no Settings redirect', async () => { diff --git a/web/src/lib/components/operations/OperationRunDetail.svelte b/web/src/lib/components/operations/OperationRunDetail.svelte index 634aae95e..71fb52a3e 100644 --- a/web/src/lib/components/operations/OperationRunDetail.svelte +++ b/web/src/lib/components/operations/OperationRunDetail.svelte @@ -1,14 +1,19 @@

Run detail

-

{kindLabels[detail.kind]}

+

{OPERATION_KIND_LABELS[detail.kind]}

{#if showClose}
-
State
{titleCase(detail.state)}
-
Trigger
{titleCase(detail.trigger)}
-
Started
-
Finished
{#if detail.finished_at}{:else}Not available{/if}
-
Duration
{duration(detail)}
+ {#if detail.connection}
Connection
{detail.connection}
{/if} +
State
{stateChip.label}
+
Trigger
{triggerLabel(detail.trigger)}
+
Started
+
Finished
{#if detail.finished_at}{:else}Not available{/if}
+
Duration
{operationDuration(detail)}
@@ -109,7 +59,7 @@ {:else}
{#each detail.counters as counter (`${counter.name}:${counter.unit}`)} -
{counter.name.replaceAll('_', ' ')}
{counter.value.toLocaleString()} {counter.unit}
+
{counterLabel(counter.name)}
{counterValue(counter)}
{/each}
{/if} @@ -117,8 +67,8 @@ {#if detail.error} {/if} @@ -128,7 +78,7 @@
{/each} @@ -107,15 +73,15 @@ class="run-link" type="button" data-run-id={run.id} - aria-label={`Open ${kindLabels[run.kind]} run`} + aria-label={`Open ${OPERATION_KIND_LABELS[run.kind]} run`} onclick={(event) => onSelect(run.id, event.currentTarget)} - >{kindLabels[run.kind]} + >{OPERATION_KIND_LABELS[run.kind]}{#if run.connection} · {run.connection}{/if} - {titleCase(run.trigger)} - {titleCase(run.state)} - - {duration(run)} - {counters(run)} + {triggerLabel(run.trigger)} + {@render runState(run)} + + {operationDuration(run)} + {counterSummary(run.counters)} {/each} @@ -138,7 +104,8 @@ cursor: pointer; } .run-link:focus-visible { outline: var(--focus-ring); outline-offset: 2px; } - .state { display: inline-flex; align-items: center; gap: var(--space-2); white-space: nowrap; } + .state { display: grid; justify-items: start; gap: var(--space-1); } + .state .error { color: var(--status-error-ink); } .run-list { display: grid; gap: var(--space-2); } .run-list [role="listitem"] { border: 1px solid var(--border-default); border-radius: var(--radius-md); background: var(--bg-surface); } .run-list [role="listitem"].selected { border-color: var(--accent-blue); } diff --git a/web/src/lib/components/operations/OperationStatusList.svelte b/web/src/lib/components/operations/OperationStatusList.svelte new file mode 100644 index 000000000..f60fed1d7 --- /dev/null +++ b/web/src/lib/components/operations/OperationStatusList.svelte @@ -0,0 +1,139 @@ + + +
+ {#each lanes as lane (lane.lane)} + {@const headingID = `operation-lane-${lane.lane}`} +
+

{OPERATION_LANE_LABELS[lane.lane]}

+ {#if lane.kinds.length === 0} + Status unavailable + {:else} +
    + {#each lane.kinds as kind (kind.kind)} + {@const nameID = `operation-kind-${kind.kind}`} + {@const chip = statusChip(kind)} + {@const run = kind.active ?? kind.latest} + {@const setup = kind.configured ? undefined : OPERATION_SETUP[kind.kind]} +
  • + {OPERATION_KIND_LABELS[kind.kind]} + + {#if chip}{chip.label}{/if} + {#if kind.history_availability !== 'available'} + History unavailable + {/if} + + + {#if run}{/if} + {#if kind.latest_successful && run?.state !== 'succeeded'} + + Last succeeded + + + {/if} + + + {#if kind.related_status} +
  • + {/each} +
+ {/if} +
+ {/each} +
+ + diff --git a/web/src/lib/components/operations/OperationsRefresh.test.ts b/web/src/lib/components/operations/OperationsRefresh.test.ts new file mode 100644 index 000000000..4d5afd7bf --- /dev/null +++ b/web/src/lib/components/operations/OperationsRefresh.test.ts @@ -0,0 +1,120 @@ +import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { createAPIClient } from '../../api/client'; +import { OperationsController } from '../../operations/controller.svelte'; +import type { OperationRunSummary, OperationsURLState } from '../../operations/models'; +import OperationsWorkspace from './OperationsWorkspace.svelte'; + +// The daemon encrypts run IDs with a fresh nonce per response +// (internal/api/operation_tokens.go), so the same run never carries the same +// ID twice. These fixtures do the same; a test cannot pass by matching IDs. +let encodings = 0; +const encode = (run: number) => `op2.${String(++encodings).padStart(32, '0')}.run${run}`; + +function summary(run: number): OperationRunSummary { + return { + id: encode(run), kind: 'source_sync', lane: 'messages', trigger: 'manual', state: 'succeeded', + started_at: `2026-08-30T1${run}:00:00Z`, finished_at: `2026-08-30T1${run}:01:00Z`, + counters: [{ name: 'processed', unit: 'messages', value: run }] + }; +} + +const state = (overrides: Partial = {}): OperationsURLState => ({ + operationLane: '', operationKind: '', operationState: '', operationStartedFrom: '', + operationStartedBefore: '', operationRunID: null, operationStatus: '', ...overrides +}); + +function daemon(pageTwo: () => Response = () => Response.json({ + runs: [summary(3), summary(4)], membership_revision: 7, unavailable_kinds: [] +})) { + const requests: string[] = []; + const fetchFn = vi.fn(async (input) => { + const url = new URL(input instanceof Request ? input.url : String(input)); + requests.push(`${url.pathname}${url.search}`); + if (url.pathname === '/api/v1/operations/status') { + return Response.json({ lanes: [{ lane: 'messages', kind: 'source_sync', configured: true, + history_availability: 'available', supported_actions: [], latest: summary(1) }] }); + } + if (url.pathname === '/api/v1/operations/runs') { + if (url.searchParams.get('cursor') === 'page-two') return pageTwo(); + return Response.json({ + runs: [summary(1), summary(2)], membership_revision: 7, unavailable_kinds: [], next_cursor: 'page-two' + }); + } + return Response.json({ ...summary(3), related_status: 'listSourceStatus', supported_actions: [] }); + }); + return { requests, client: createAPIClient(fetchFn) }; +} + +const runButtons = () => screen.getAllByRole('button', { name: 'Open Source sync run' }); + +afterEach(() => vi.useRealTimers()); + +describe('Operations refresh', () => { + it('refreshes only status on its timer and on click, keeping paged rows, detail, and focus', async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }); + const { requests, client } = daemon(); + const controller = new OperationsController(client); + try { + await controller.applyURLState(state()); + await controller.loadMore(); + const selected = controller.snapshot.rows[2]!.id; + await controller.applyURLState(state({ operationRunID: selected })); + render(OperationsWorkspace, { controller, state: state({ operationRunID: selected }) }); + expect(runButtons()).toHaveLength(4); + expect(screen.getByRole('region', { name: 'Operation run detail' })).toBeDefined(); + runButtons()[2]!.focus(); + const focused = document.activeElement; + + requests.length = 0; + await vi.advanceTimersByTimeAsync(5 * 60 * 1000); + await waitFor(() => expect(requests).toEqual(['/api/v1/operations/status'])); + expect(runButtons()).toHaveLength(4); + expect(screen.getByRole('region', { name: 'Operation run detail' })).toBeDefined(); + expect(document.activeElement).toBe(focused); + + requests.length = 0; + await fireEvent.click(screen.getByRole('button', { name: 'Refresh operation status' })); + await waitFor(() => expect(requests).toEqual(['/api/v1/operations/status'])); + expect(runButtons()).toHaveLength(4); + expect(screen.getByRole('region', { name: 'Operation run detail' })).toBeDefined(); + } finally { + controller.destroy(); + } + }); + + it('reloads status and page one of runs from Reload run history', async () => { + const { requests, client } = daemon(); + const controller = new OperationsController(client); + try { + await controller.applyURLState(state()); + await controller.loadMore(); + const before = controller.snapshot.rows.map((row) => row.id); + render(OperationsWorkspace, { controller, state: state() }); + requests.length = 0; + await fireEvent.click(screen.getByRole('button', { name: 'Reload run history' })); + await waitFor(() => expect(runButtons()).toHaveLength(2)); + expect(requests.sort()).toEqual(['/api/v1/operations/runs?limit=25', '/api/v1/operations/status']); + expect(controller.snapshot.rows.some((row) => before.includes(row.id))).toBe(false); + } finally { + controller.destroy(); + } + }); + + it('keeps the conflict notice when Load more meets a changed history', async () => { + const { client } = daemon(() => Response.json( + { error: 'operation_history_conflict', message: 'Operation history changed.' }, { status: 409 })); + const controller = new OperationsController(client); + try { + await controller.applyURLState(state()); + render(OperationsWorkspace, { controller, state: state() }); + await fireEvent.click(screen.getByRole('button', { name: 'Load more operation history' })); + const conflict = await screen.findByRole('alert', { name: 'Operation history conflict' }); + expect(conflict.textContent).toContain('Operation history changed. Restart from the first page.'); + expect(screen.getByRole('button', { name: 'Restart operation history' })).toBeDefined(); + } finally { + controller.destroy(); + } + }); +}); diff --git a/web/src/lib/components/operations/OperationsWorkspace.svelte b/web/src/lib/components/operations/OperationsWorkspace.svelte index 459258578..c96541964 100644 --- a/web/src/lib/components/operations/OperationsWorkspace.svelte +++ b/web/src/lib/components/operations/OperationsWorkspace.svelte @@ -2,6 +2,7 @@ import { Button, DateRangePicker, + RefreshControl, SelectDropdown, resolveRange, type RangeSelection @@ -16,19 +17,23 @@ resolveOperationFocusAnchor, type OperationFocusAnchor } from '../../operations/focus'; - import type { - OperationAction, - OperationKind, - OperationRunDetail as OperationRunDetailModel, - OperationsURLState - } from '../../operations/models'; - import OperationLaneCards from './OperationLaneCards.svelte'; + import type { OperationAction, OperationKind, OperationsURLState } from '../../operations/models'; + import { + OPERATION_KIND_LABELS, + OPERATION_LANE_LABELS, + RELATED_STATUS_LABELS, + type OperationSettingsTarget, + type RelatedStatus + } from '../../operations/labels'; import OperationRelatedStatus from './OperationRelatedStatus.svelte'; import OperationRunDetail from './OperationRunDetail.svelte'; import OperationRunTable from './OperationRunTable.svelte'; + import OperationStatusList from './OperationStatusList.svelte'; + import PageHeader from '../shell/PageHeader.svelte'; - type RelatedStatus = NonNullable; - type Controller = Pick; + type Controller = Pick< + OperationsController, 'snapshot' | 'refresh' | 'refreshStatus' | 'loadMore' | 'restart' | 'runAction' + >; let { controller, @@ -37,7 +42,8 @@ onStateChange = () => undefined, onNavigate = () => undefined, onAnnounce = () => undefined, - onConfigure = () => undefined + onConfigure = () => undefined, + onSetUp = () => undefined }: { controller: Controller; client?: APIClient; @@ -45,7 +51,8 @@ onStateChange?: (patch: Partial) => void; onNavigate?: (target: RelatedStatus) => void; onAnnounce?: (message: string) => void; - onConfigure?: (target: OperationStatusAuthority) => void; + onConfigure?: () => void; + onSetUp?: (target: OperationSettingsTarget) => void; } = $props(); let root = $state(); @@ -60,23 +67,11 @@ const laneOptions = [ { value: '', label: 'All lanes' }, - { value: 'messages', label: 'Messages' }, - { value: 'person_facts', label: 'Facts' }, - { value: 'contacts', label: 'Contacts' }, - { value: 'documents', label: 'Documents' }, - { value: 'visual_attachments', label: 'Attachments' } + ...Object.entries(OPERATION_LANE_LABELS).map(([value, label]) => ({ value, label })) ]; const kindOptions = [ { value: '', label: 'All kinds' }, - { value: 'source_sync', label: 'Source sync' }, - { value: 'message_embedding', label: 'Message embedding' }, - { value: 'person_sweep', label: 'Person fact sweep' }, - { value: 'person_embedding', label: 'Person embedding' }, - { value: 'person_enrichment', label: 'Person enrichment' }, - { value: 'carddav_sync', label: 'CardDAV sync' }, - { value: 'document_extraction', label: 'Document extraction' }, - { value: 'document_embedding', label: 'Document embedding' }, - { value: 'visual_embedding', label: 'Visual embedding' } + ...Object.entries(OPERATION_KIND_LABELS).map(([value, label]) => ({ value, label })) ]; const stateOptions = [ { value: '', label: 'All states' }, @@ -87,27 +82,11 @@ { value: 'failed', label: 'Failed' }, { value: 'cancelled', label: 'Cancelled' } ]; - const kindLabels: Record = { - source_sync: 'Source sync', - message_embedding: 'Message embedding', - person_sweep: 'Person fact sweep', - person_embedding: 'Person embedding', - person_enrichment: 'Person enrichment', - carddav_sync: 'CardDAV sync', - document_extraction: 'Document extraction', - document_embedding: 'Document embedding', - visual_embedding: 'Visual embedding' - }; const actionSuccess: Record = { carddav_sync: 'CardDAV sync request completed; current operation state was refreshed.', visual_build: 'Visual index build request completed; current operation state was refreshed.', visual_resume: 'Visual index resume request completed; current operation state was refreshed.' }; - const relatedStatusLabels: Record = { - getDocumentIndexStatus: 'Open Document index status', - getDocumentVectorStatus: 'Open Document vector status', - getVisualAttachmentStatus: 'Open Visual attachment status' - }; onMount(() => { const mediaQuery = window.matchMedia?.('(max-width: 760px)'); @@ -147,11 +126,15 @@ return undefined; } + function relatedStatusButton(target: OperationStatusAuthority): string { + return `Open ${RELATED_STATUS_LABELS[target]}`; + } + function navigateStatus(target: RelatedStatus, button: HTMLButtonElement): void { if (target === 'getDocumentIndexStatus' || target === 'getDocumentVectorStatus' || target === 'getVisualAttachmentStatus') { const buttons = Array.from(root?.querySelectorAll('button') ?? []); - const targetButtons = buttons.filter((candidate) => candidate.ariaLabel === relatedStatusLabels[target]); + const targetButtons = buttons.filter((candidate) => candidate.ariaLabel === relatedStatusButton(target)); statusFocus = { target, ordinal: Math.max(0, targetButtons.indexOf(button)), @@ -167,7 +150,7 @@ await tick(); await new Promise((resolve) => setTimeout(resolve, 0)); const buttons = Array.from(root?.querySelectorAll('button') ?? []); - const targetButtons = buttons.filter((button) => button.ariaLabel === relatedStatusLabels[focus.target]); + const targetButtons = buttons.filter((button) => button.ariaLabel === relatedStatusButton(focus.target)); (targetButtons[focus.ordinal] ?? buttons[focus.slot])?.focus(); statusFocus = undefined; } @@ -206,6 +189,19 @@ }); } + async function refreshStatus(): Promise { + const control = root?.querySelector('.kit-refresh-control'); + const hadFocus = Boolean(control?.contains(document.activeElement)); + await controller.refreshStatus(); + // Kit disables the button while busy, and a disabled button drops focus. + // Restore it only if the person has not moved focus elsewhere meanwhile. + if (!hadFocus) return; + await tick(); + const active = document.activeElement; + if (active && active !== document.body && !control?.contains(active)) return; + control?.querySelector('button')?.focus(); + } + async function runAction(action: OperationAction): Promise { const outcome = await controller.runAction(action); if (outcome === 'discarded') return; @@ -250,7 +246,7 @@ {#if current.unavailableKinds.length > 0}
{#each current.unavailableKinds as unavailable (unavailable.kind)} - {kindLabels[unavailable.kind]} history is unavailable. + {OPERATION_KIND_LABELS[unavailable.kind]} history is unavailable. {/each}
{/if} @@ -280,10 +276,11 @@ {/key} {:else if narrow && urlState.operationRunID !== null}
-
-

Operation detail

-
+ + {#snippet actions()} +
{:else} -
-

Archive operations

Operations

-
+ + {#snippet actions()} + void refreshStatus()} + /> + {/snippet} + {#if current.statusReadable} - void runAction(action)} + {onSetUp} /> {/if} @@ -326,6 +330,7 @@ {#if urlState.operationStartedFrom || urlState.operationStartedBefore}
diff --git a/web/src/lib/components/operations/OperationsWorkspace.test.ts b/web/src/lib/components/operations/OperationsWorkspace.test.ts index 4d2d35b96..55739155e 100644 --- a/web/src/lib/components/operations/OperationsWorkspace.test.ts +++ b/web/src/lib/components/operations/OperationsWorkspace.test.ts @@ -2,7 +2,13 @@ import { fireEvent, render, screen, waitFor, within } from '@testing-library/sve import { afterEach, describe, expect, it, vi } from 'vitest'; import { createAPIClient } from '../../api/client'; -import type { OperationsSnapshot, OperationsURLState } from '../../operations/models'; +import { OPERATION_KIND_LABELS } from '../../operations/labels'; +import type { + OperationKind, + OperationLane, + OperationsSnapshot, + OperationsURLState +} from '../../operations/models'; import { chooseSelectOption } from '../../../test/kit-ui'; import OperationsWorkspace from './OperationsWorkspace.svelte'; @@ -99,6 +105,8 @@ function snapshot(overrides: Partial = {}): OperationsSnapsh nextCursor: null, statusReadable: true, historyReadable: true, + statusUpdatedAt: null, + statusRefreshing: false, initialLoading: false, backgroundLoading: false, paging: false, @@ -119,61 +127,154 @@ function controller(current: OperationsSnapshot = snapshot()) { return { snapshot: current, refresh: vi.fn(async () => undefined), + refreshStatus: vi.fn(async () => true), loadMore: vi.fn(async () => undefined), restart: vi.fn(async () => undefined), runAction: vi.fn(async () => 'succeeded' as const) }; } +const off = (kind: OperationKind, lane: OperationLane, related?: string) => ({ + lane, kind, configured: false, history_availability: 'available' as const, supported_actions: [], + ...(related ? { related_status: related } : {}) +}); + +function renderKinds(kinds: Array>, props: Record = {}) { + const lanes = ['messages', 'person_facts', 'contacts', 'documents', 'visual_attachments'] as const; + const statusLanes = lanes.map((lane) => ({ lane, kinds: kinds.filter((kind) => kind.lane === lane) })); + return render(OperationsWorkspace, { + controller: controller(snapshot({ statusLanes: statusLanes as never })) as never, + state: urlState(), + ...props + }); +} + afterEach(() => vi.unstubAllGlobals()); describe('OperationsWorkspace', () => { - it('renders exact public lanes, explicit per-kind availability and semantic run summaries', () => { - const rendered = render(OperationsWorkspace, { - controller: controller() as never, - state: urlState() - }); + it('lists every lane with one row per kind and no History available text', () => { + render(OperationsWorkspace, { controller: controller() as never, state: urlState() }); + const region = screen.getByRole('region', { name: 'Operation lanes' }); + expect(within(region).getAllByRole('heading', { level: 2 }).map((heading) => heading.textContent)) + .toEqual(['Messages', 'Facts', 'Contacts', 'Documents', 'Attachments']); + expect(within(screen.getByRole('list', { name: 'Messages' })).getAllByRole('listitem')).toHaveLength(2); + const embedding = screen.getByRole('listitem', { name: 'Message embedding' }); + expect(within(embedding).getByText('Running')).toBeDefined(); + expect(within(embedding).getByText('History unavailable')).toBeDefined(); + expect(region.textContent).not.toContain('History available'); + expect(within(screen.getByRole('listitem', { name: 'Person fact sweep' })).getByText('Off')).toBeDefined(); + expect(within(screen.getByRole('listitem', { name: 'Person embedding' })).getByText('No runs yet')).toBeDefined(); + }); - const cards = screen.getByRole('region', { name: 'Operation lanes' }); - expect(within(cards).getAllByRole('heading', { level: 2 }).map((heading) => heading.textContent)).toEqual([ - 'Messages', 'Facts', 'Contacts', 'Documents', 'Attachments' - ]); - expect(rendered.container.textContent).not.toContain('Person facts'); - expect(rendered.container.textContent).not.toContain('Visual attachments'); - expect(within(cards).getByText('Source sync')).toBeDefined(); - expect(within(cards).getByText('Message embedding')).toBeDefined(); - expect(within(cards).getByText('Person fact sweep')).toBeDefined(); - expect(within(cards).getByText('Not configured')).toBeDefined(); - expect(within(cards).getByText('History unavailable')).toBeDefined(); - expect(within(cards).getAllByText('Active').length).toBeGreaterThan(0); - expect(within(cards).getAllByText('Latest').length).toBeGreaterThan(0); - expect(within(cards).getAllByText('Last successful').length).toBeGreaterThan(0); - expect(within(cards).getAllByText(/Running|Succeeded/).length).toBeGreaterThan(0); + it('leaves focus where the person moved it while a status refresh was pending', async () => { + let finish!: (value: boolean) => void; + const stub = controller(); + stub.refreshStatus.mockImplementation(() => new Promise((resolve) => { finish = resolve; })); + render(OperationsWorkspace, { controller: stub as never, state: urlState() }); + const refresh = screen.getByRole('button', { name: 'Refresh operation status' }); + refresh.focus(); + await fireEvent.click(refresh); + expect(stub.refreshStatus).toHaveBeenCalledOnce(); + + const reload = screen.getByRole('button', { name: 'Reload run history' }); + reload.focus(); + finish(true); + + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(document.activeElement).toBe(reload); }); - it('claims no recorded runs only when history is available', () => { - const current = snapshot(); - const statusLanes = current.statusLanes.map((lane) => ({ - ...lane, - kinds: lane.kinds.map((kind) => kind.kind === 'message_embedding' - ? { ...kind, active: undefined, latest: undefined, latest_successful: undefined } - : kind) - })); - render(OperationsWorkspace, { - controller: controller(snapshot({ statusLanes })) as never, - state: urlState() - }); + it('shows Status unavailable for a lane without kinds', () => { + renderKinds([]); + expect(screen.getAllByText('Status unavailable')).toHaveLength(5); + }); + + it.each([ + ['message_embedding', 'messages', { settingsCategory: 'search', settingsAuthority: 'semantic_search' }], + ['person_embedding', 'person_facts', { settingsCategory: 'search', settingsAuthority: 'person_embeddings' }], + ['visual_embedding', 'visual_attachments', { settingsCategory: 'search', settingsAuthority: 'visual_attachments' }], + ['person_enrichment', 'person_facts', { settingsCategory: 'enrichment', settingsAuthority: '' }], + ['person_sweep', 'person_facts', { settingsCategory: 'people', settingsAuthority: '' }] + ] as const)('sends an Off %s row to Settings', async (kind, lane, target) => { + const onSetUp = vi.fn(); + renderKinds([off(kind, lane)], { onSetUp }); + const row = screen.getByRole('listitem', { name: OPERATION_KIND_LABELS[kind] }); + expect(within(row).getByText('Off')).toBeDefined(); + await fireEvent.click(within(row).getByRole('button', { name: `Set up ${OPERATION_KIND_LABELS[kind]}` })); + expect(onSetUp).toHaveBeenCalledWith(target); + }); - const unavailable = screen.getByRole('region', { name: 'Message embedding' }); + it.each([ + ['document_extraction', 'Configured in config.toml on the daemon host.', 'Document indexing setup', + 'https://msgvault.io/docs/usage/document-indexing/#configure-the-policy'], + ['document_embedding', 'Configured in config.toml on the daemon host. Also needs semantic search.', + 'Document search setup', + 'https://msgvault.io/docs/usage/document-indexing/#semantic-and-hybrid-document-search'] + ] as const)('explains host configuration for an Off %s row', (kind, text, linkName, href) => { + renderKinds([off(kind, 'documents')]); + const row = screen.getByRole('listitem', { name: OPERATION_KIND_LABELS[kind] }); + expect(within(row).queryByRole('button', { name: /^Set up/ })).toBeNull(); + expect(within(row).getByText(text, { exact: false })).toBeDefined(); + const link = within(row).getByRole('link', { name: linkName }); + expect(link.getAttribute('href')).toBe(href); + expect(link.getAttribute('target')).toBe('_blank'); + expect(link.getAttribute('rel')).toBe('noreferrer'); + }); + + it('keeps the related-status button and no Set up for Off CardDAV and source rows', () => { + renderKinds([ + off('carddav_sync', 'contacts', 'getCardDAVStatus'), + off('source_sync', 'messages', 'listSourceStatus') + ]); + expect(screen.getByRole('button', { name: 'Open CardDAV settings' })).toBeDefined(); + expect(screen.getByRole('button', { name: 'Open Sources status' })).toBeDefined(); + expect(screen.queryByRole('button', { name: /^Set up/ })).toBeNull(); + }); + + it('claims no runs only when history is available', () => { + renderKinds([ + { ...off('message_embedding', 'messages'), configured: true, history_availability: 'unavailable' } as never, + { ...off('carddav_sync', 'contacts'), configured: true } + ]); + const unavailable = screen.getByRole('listitem', { name: 'Message embedding' }); expect(within(unavailable).getByText('History unavailable')).toBeDefined(); - expect(within(unavailable).queryByText('No recorded runs')).toBeNull(); + expect(within(unavailable).queryByText('No runs yet')).toBeNull(); + expect(within(screen.getByRole('listitem', { name: 'CardDAV sync' })).getByText('No runs yet')).toBeDefined(); + }); + + it('shows an active run instead of Off for a kind that is not configured', () => { + const active = { + id: 'x', kind: 'message_embedding' as const, lane: 'messages' as const, state: 'queued' as const, + started_at: '2026-08-30T12:00:00Z', counters: [] + }; + renderKinds([{ ...off('message_embedding', 'messages'), active } as never]); + const row = screen.getByRole('listitem', { name: 'Message embedding' }); + expect(within(row).getByText('Queued')).toBeDefined(); + expect(within(row).queryByText('Off')).toBeNull(); + }); + + it('shows when the last success differs from the latest run', () => { + const partial = { + id: 'x', kind: 'source_sync' as const, lane: 'messages' as const, state: 'partial' as const, + started_at: '2026-08-30T12:00:00Z', counters: [] + }; + const success = { ...partial, id: 'y', state: 'succeeded' as const, started_at: '2026-08-29T12:00:00Z' }; + renderKinds([{ + ...off('source_sync', 'messages'), configured: true, latest: partial, latest_successful: success + } as never]); + const row = screen.getByRole('listitem', { name: 'Source sync' }); + expect(within(row).getByText('Partial')).toBeDefined(); + expect(within(row).getByText(/Last succeeded/)).toBeDefined(); + }); - const availableEmpty = screen.getByRole('region', { name: 'CardDAV sync' }); - expect(within(availableEmpty).getByText('History available')).toBeDefined(); - expect(within(availableEmpty).getByText('No recorded runs')).toBeDefined(); + it('omits Last succeeded when the latest run succeeded', () => { + render(OperationsWorkspace, { controller: controller() as never, state: urlState() }); + const row = screen.getByRole('listitem', { name: 'Source sync' }); + expect(within(row).getByText('Succeeded')).toBeDefined(); + expect(within(row).queryByText(/Last succeeded/)).toBeNull(); }); - it('keeps advertised authority links and actions on lane status cards', async () => { + it('keeps advertised authority links and actions on lane status rows', async () => { const onNavigate = vi.fn(); const actions = controller(); render(OperationsWorkspace, { @@ -182,17 +283,16 @@ describe('OperationsWorkspace', () => { onNavigate }); - const cards = screen.getByRole('region', { name: 'Operation lanes' }); - await fireEvent.click(within(cards).getByRole('button', { name: 'Open Sources status' })); + const lanes = screen.getByRole('region', { name: 'Operation lanes' }); + await fireEvent.click(within(lanes).getByRole('button', { name: 'Open Sources status' })); expect(onNavigate).toHaveBeenCalledWith('listSourceStatus'); - await fireEvent.click(within(cards).getByRole('button', { name: 'Start CardDAV sync' })); + await fireEvent.click(within(lanes).getByRole('button', { name: 'Start CardDAV sync' })); expect(actions.runAction).toHaveBeenCalledWith('carddav_sync'); - expect(within(cards).getByRole('button', { name: 'Resume visual index' })).toBeDefined(); - expect(within(cards).queryByRole('button', { name: 'Build visual index' })).toBeNull(); + expect(within(lanes).getByRole('button', { name: 'Resume visual index' })).toBeDefined(); + expect(within(lanes).queryByRole('button', { name: 'Build visual index' })).toBeNull(); }); - it('passes the authoritative unconfigured document state to its related status panel', async () => { - const onConfigure = vi.fn(); + it('explains host configuration in the unconfigured document panel without a request', async () => { const fetchFn = vi.fn(); const statusLanes = snapshot().statusLanes.map((lane) => ({ ...lane, @@ -203,12 +303,16 @@ describe('OperationsWorkspace', () => { render(OperationsWorkspace, { controller: controller(snapshot({ statusLanes })) as never, client: createAPIClient(fetchFn), - state: urlState({ operationStatus: 'getDocumentIndexStatus' }), - onConfigure + state: urlState({ operationStatus: 'getDocumentIndexStatus' }) }); - await fireEvent.click(await screen.findByRole('button', { name: 'Open document index settings' })); - expect(onConfigure).toHaveBeenCalledWith('getDocumentIndexStatus'); + const panel = await screen.findByRole('region', { name: 'Document index status' }); + expect(within(panel).getByText('Off')).toBeDefined(); + expect(within(panel).getByText('Configured in config.toml on the daemon host.', { exact: false })).toBeDefined(); + const link = within(panel).getByRole('link', { name: 'Document indexing setup' }); + expect(link.getAttribute('target')).toBe('_blank'); + expect(link.getAttribute('rel')).toBe('noreferrer'); + expect(within(panel).queryByRole('button', { name: 'Open document index settings' })).toBeNull(); expect(fetchFn).not.toHaveBeenCalled(); }); @@ -255,7 +359,7 @@ describe('OperationsWorkspace', () => { const rows = within(table).getAllByRole('row').slice(1); expect(rows[0]!.textContent).toContain('Document extraction'); expect(rows[0]!.textContent).toContain('2 seconds'); - expect(rows[0]!.textContent).toContain('2 failed writes'); + expect(rows[0]!.textContent).toContain('2 writes failed'); expect(rows[1]!.textContent).toContain('Source sync'); expect(rows[1]!.textContent).toContain('1 minute 5 seconds'); @@ -263,6 +367,39 @@ describe('OperationsWorkspace', () => { expect(onStateChange).toHaveBeenCalledWith({ operationRunID: RUN_TWO }); }); + it('shows triggers, counters, and failure sentences in the runs table', () => { + const failed = run({ id: RUN_TWO, state: 'failed', trigger: undefined, + counters: [ + { name: 'processed', unit: 'messages', value: 20 }, { name: 'added', unit: 'messages', value: 20 }, + { name: 'updated', unit: 'messages', value: 0 }, { name: 'item_errors', unit: 'messages', value: 0 } + ], + error: { code: 'source_sync_failed', message: 'Source sync failed.' } }); + render(OperationsWorkspace, { + controller: controller(snapshot({ rows: [failed], unavailableKinds: [] })) as never, + state: urlState() + }); + const row = within(screen.getByRole('table', { name: 'Operation history' })).getAllByRole('row')[1]!; + expect(row.textContent).toContain('—'); + expect(row.textContent).toContain('20 messages processed · 20 added'); + expect(row.textContent).not.toContain('item errors'); + expect(row.textContent).toContain('Failed'); + expect(row.textContent).toContain('Source sync failed.'); + }); + + it('leads the run detail error with the server sentence and keeps every counter', () => { + const detail = { ...run({ state: 'failed', counters: [{ name: 'item_errors', unit: 'messages', value: 0 }], + error: { code: 'source_sync_failed', message: 'Source sync failed.' } }), supported_actions: [] }; + render(OperationsWorkspace, { + controller: controller(snapshot({ detail })) as never, + state: urlState({ operationRunID: RUN_ONE }) + }); + const error = screen.getByRole('alert', { name: 'Operation error' }); + expect(error.firstElementChild?.textContent).toBe('Source sync failed.'); + expect(within(error).getByText('Code: source_sync_failed').tagName).toBe('CODE'); + expect(screen.getByText('Item errors')).toBeDefined(); + expect(screen.getByText('0 messages')).toBeDefined(); + }); + it('renders only allowlisted detail, fixed error, related authority and advertised actions', async () => { const onNavigate = vi.fn(); const current = snapshot({ @@ -282,7 +419,7 @@ describe('OperationsWorkspace', () => { }); const detail = screen.getByRole('region', { name: 'Operation run detail' }); - expect(within(detail).getByText('timeout')).toBeDefined(); + expect(within(detail).getByText('Code: timeout')).toBeDefined(); expect(within(detail).getByText('The operation timed out.')).toBeDefined(); expect(within(detail).getByRole('button', { name: 'Open Sources status' })).toBeDefined(); expect(within(detail).getByRole('button', { name: 'Start CardDAV sync' })).toBeDefined(); @@ -440,6 +577,22 @@ describe('OperationsWorkspace', () => { expect(onStateChange).toHaveBeenCalledWith({ operationRunID: null }); }); + it('shows one visible level-one heading in the narrow detail view', () => { + vi.stubGlobal('matchMedia', () => ({ + matches: true, + addEventListener: vi.fn(), + removeEventListener: vi.fn() + })); + render(OperationsWorkspace, { + controller: controller(snapshot({ detail: null, detailLoading: true })) as never, + state: urlState({ operationRunID: RUN_ONE }) + }); + + const headings = screen.getAllByRole('heading', { level: 1 }); + expect(headings.map((heading) => heading.textContent?.trim())).toEqual(['Operation detail']); + expect(headings[0]!.closest('.kit-sr-only')).toBeNull(); + }); + it.each([ ['detail loading', { detailLoading: true }, 'status', 'Operation detail loading'], ['detail failure', { detailError: 'Unable to load operation detail.' }, 'alert', 'Operation detail failure'], diff --git a/web/src/lib/components/people/PersonTimeline.svelte b/web/src/lib/components/people/PersonTimeline.svelte index 189379cc2..1ba6bdf78 100644 --- a/web/src/lib/components/people/PersonTimeline.svelte +++ b/web/src/lib/components/people/PersonTimeline.svelte @@ -39,7 +39,7 @@
-

Canonical context

Activity

+

Activity

{totalCount === undefined ? 'Bounded timeline' : `${totalCount.toLocaleString()} items`}
.timeline { display: flex; min-height: 320px; flex-direction: column; gap: var(--space-3); } header { display: flex; align-items: end; justify-content: space-between; } - header p, header h2 { margin: 0; } - header p { color: var(--accent-amber); font-size: var(--font-size-2xs); font-weight: 800; letter-spacing: .08em; text-transform: uppercase; } + header h2 { margin: 0; } header h2 { font-size: var(--font-size-lg); } header span { color: var(--text-muted); font-size: var(--font-size-xs); } diff --git a/web/src/lib/components/reader/MessagePage.svelte b/web/src/lib/components/reader/MessagePage.svelte index 2b81d0048..752098521 100644 --- a/web/src/lib/components/reader/MessagePage.svelte +++ b/web/src/lib/components/reader/MessagePage.svelte @@ -47,6 +47,6 @@ diff --git a/web/src/lib/components/reader/ReadingPane.svelte b/web/src/lib/components/reader/ReadingPane.svelte index 2f6404b53..d6ab7b04c 100644 --- a/web/src/lib/components/reader/ReadingPane.svelte +++ b/web/src/lib/components/reader/ReadingPane.svelte @@ -1,4 +1,5 @@
- {:else if selection.kind === 'entry'} + {:else if selection.kind === 'entry' || selection.kind === 'archive'}
-

{selection.row.preview || 'No preview is available.'}

+

{(selection.kind === 'entry' ? selection.row.preview : selection.message.snippet) || 'No preview is available.'}

{:else}
{#if showFiles} + {#if selection.meetingScope} + + {:else} +

Reload this group to load meeting activity with its current search scope.

+ {/if} {:else}
@@ -327,21 +378,25 @@ min-height: 40px; flex: none; align-items: center; + flex-wrap: wrap; justify-content: space-between; - gap: var(--space-4); + gap: var(--space-2) var(--space-4); padding: var(--space-2) var(--space-4); border-bottom: 1px solid var(--border-muted); box-shadow: 0 1px 0 var(--hairline-sheen); } + /* A narrow pane wraps the actions below the heading instead of squeezing the title away. */ .pane-heading { display: flex; min-width: 0; + flex: 1 1 12rem; align-items: baseline; gap: var(--space-4); } .pane-title { + min-width: min(8rem, 100%); overflow: hidden; color: var(--text-primary); font-size: var(--font-size-sm); @@ -350,7 +405,8 @@ } .pane-meta { - flex: none; + min-width: 0; + flex: 0 1 auto; overflow: hidden; color: var(--text-muted); font-size: var(--font-size-2xs); @@ -365,28 +421,6 @@ gap: var(--space-2); } - .tasks-disclosure summary { - display: inline-flex; - align-items: center; - padding: 3px 10px; - border: 1px solid var(--control-border); - border-radius: var(--radius-md); - color: var(--text-secondary); - cursor: pointer; - font-size: var(--font-size-xs); - list-style: none; - } - - .tasks-disclosure summary::-webkit-details-marker { - display: none; - } - - .tasks-disclosure[open] summary, - .tasks-disclosure summary:hover { - background: var(--bg-surface-hover); - color: var(--text-primary); - } - .tasks-sheet { flex: none; max-height: 40%; @@ -395,6 +429,19 @@ border-bottom: 1px solid var(--border-muted); } + .meeting-actions-sheet { + flex: none; + min-height: 0; + max-height: 50%; + overflow: auto; + } + + .meeting-context-sheet { + flex: none; + padding: var(--space-3) var(--space-4); + border-bottom: 1px solid var(--border-muted); + } + .pane-body { display: flex; min-height: 0; diff --git a/web/src/lib/components/reader/ReadingPane.test.ts b/web/src/lib/components/reader/ReadingPane.test.ts index 60649ac98..e5b2c5710 100644 --- a/web/src/lib/components/reader/ReadingPane.test.ts +++ b/web/src/lib/components/reader/ReadingPane.test.ts @@ -1,4 +1,4 @@ -import { render, screen, waitFor } from '@testing-library/svelte'; +import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; import { describe, expect, it, vi } from 'vitest'; import { createAPIClient } from '../../api/client'; @@ -62,6 +62,155 @@ describe('ReadingPane task gating', () => { }); }); +describe('ReadingPane header', () => { + function taskFetch(tasks: unknown[]) { + return vi.fn(async (input) => { + const request = input instanceof Request ? input : new Request(input); + if (request.url.endsWith('/integrations/tasks/status')) { + return Response.json({ state: 'ready', project: 'project', message: 'Ready' }); + } + return Response.json({ state: 'ready', complete: true, last_scan: '2026-07-19T01:00:00Z', tasks }); + }); + } + + it('opens the linked tasks from a Tasks button without a count', async () => { + const fetchFn = taskFetch([ + { id: 'task-1', title: 'Follow up', revision: 'r1' }, + { id: 'task-2', title: 'Reply', revision: 'r2' } + ]); + render(ReadingPane, { + props: { + client: createAPIClient(fetchFn), + selection: { kind: 'entry', row: entryRow() }, + predicate: {} satisfies ExplorePredicate + } + }); + const button = screen.getByRole('button', { name: 'Tasks for this message' }); + expect(button.getAttribute('aria-expanded')).toBe('false'); + expect(button.textContent?.trim()).toBe('Tasks'); + + await fireEvent.click(button); + expect(button.getAttribute('aria-expanded')).toBe('true'); + expect(await screen.findByRole('region', { name: 'Linked tasks' })).toBeDefined(); + await screen.findByText('Reply'); + expect(button.textContent?.trim()).toBe('Tasks'); + expect(screen.getByRole('button', { name: 'Tasks for this message' })).toBe(button); + }); + + it('closes the sheet when the selection changes', async () => { + const fetchFn = taskFetch([{ id: 'task-1', title: 'Follow up', revision: 'r1' }]); + const view = render(ReadingPane, { + props: { + client: createAPIClient(fetchFn), + selection: { kind: 'entry', row: entryRow() }, + predicate: {} satisfies ExplorePredicate + } + }); + await fireEvent.click(screen.getByRole('button', { name: 'Tasks for this message' })); + await screen.findByText('Follow up'); + + await view.rerender({ selection: { kind: 'entry', row: entryRow({ key: 'entry-2', anchor_message_id: 43 }) } }); + const button = screen.getByRole('button', { name: 'Tasks for this message' }); + expect(button.getAttribute('aria-expanded')).toBe('false'); + expect(screen.queryByRole('region', { name: 'Linked tasks' })).toBeNull(); + }); + + it('closes from an icon-only button', async () => { + const onClose = vi.fn(); + render(ReadingPane, { + props: { + client: createAPIClient(vi.fn()), + selection: { kind: 'entry', row: entryRow() }, + predicate: {} satisfies ExplorePredicate, + onClose + } + }); + const close = screen.getByRole('button', { name: 'Close reading pane' }); + expect(close.textContent?.trim()).toBe(''); + await fireEvent.click(close); + expect(onClose).toHaveBeenCalledOnce(); + }); + + it.each([ + ['email', 'message', 'Email'], + ['imessage', 'conversation', 'Conversation'], + ['calendar', 'event', 'Calendar event'], + ['meeting', 'meeting', 'Meeting'] + ])('labels %s (%s) entries %s in the meta strip', (messageType, kind, name) => { + renderPane(entryRow({ message_type: messageType, kind, anchor_message_id: undefined })); + const meta = document.querySelector('.pane-meta')?.textContent ?? ''; + expect(meta.startsWith(`${name} · `)).toBe(true); + expect(meta).not.toContain(messageType === 'email' ? 'email ·' : messageType); + }); +}); + +describe('ReadingPane meeting evidence', () => { + it('uses only an exact meeting transcript anchor for context and archived actions', async () => { + const requests: Request[] = []; + const createObjectURL = vi.spyOn(URL, 'createObjectURL').mockReturnValue('blob:reader-meeting-context'); + vi.spyOn(URL, 'revokeObjectURL').mockImplementation(() => undefined); + vi.spyOn(HTMLAnchorElement.prototype, 'click').mockImplementation(() => undefined); + const fetchFn = vi.fn(async (input) => { + const request = input instanceof Request ? input : new Request(input); + requests.push(request); + const path = new URL(request.url).pathname; + if (path.endsWith('/actions')) { + return Response.json({ + schema_version: 1, + archive_uid: 'archive-test', + rows: [], + total_count: 0, + coverage: { meeting_count: 1, available: 1, partial: 0, unsupported: 0, unavailable: 0 }, + scope: { kind: 'direct' }, + }); + } + return Response.json({ + schema_version: 1, + format: 'json', + content: '{"meeting":42}', + content_bytes: 14, + truncated: false, + omitted_message_ids: [], + }); + }); + render(ReadingPane, { + client: createAPIClient(fetchFn), + selection: { + kind: 'entry', + row: entryRow({ + message_type: 'meeting_transcript', + anchor_message_id: 42, + conversation_id: undefined, + }), + }, + predicate: {}, + }); + + expect(await screen.findByText('No recorded action items')).toBeDefined(); + await fireEvent.click(screen.getByRole('button', { name: 'Export meeting context' })); + await waitFor(() => expect(createObjectURL).toHaveBeenCalledOnce()); + const bodies = await Promise.all(requests.map((request) => request.clone().json())); + expect(bodies).toContainEqual({ scope: { message_ids: [42] }, limit: 200 }); + expect(bodies).toContainEqual({ message_ids: [42], format: 'json', include_transcript: false }); + }); + + it.each(['meeting_notes', 'email'])('does not treat %s as a meeting transcript', (messageType) => { + const fetchFn = vi.fn(); + render(ReadingPane, { + client: createAPIClient(fetchFn), + selection: { + kind: 'entry', + row: entryRow({ message_type: messageType, anchor_message_id: 42, conversation_id: undefined }), + }, + predicate: {}, + }); + + expect(screen.queryByRole('region', { name: 'Meeting context export' })).toBeNull(); + expect(screen.queryByText('Archived action items')).toBeNull(); + expect(fetchFn).not.toHaveBeenCalled(); + }); +}); + describe('ReadingPane identity matches', () => { it('shows via badges for email entries without replacing existing message metadata', () => { renderPane(entryRow({ diff --git a/web/src/lib/components/relationships/RelationshipCalendar.svelte b/web/src/lib/components/relationships/RelationshipCalendar.svelte index 3313dfd84..a4cea5201 100644 --- a/web/src/lib/components/relationships/RelationshipCalendar.svelte +++ b/web/src/lib/components/relationships/RelationshipCalendar.svelte @@ -7,13 +7,14 @@ RelationshipCalendar as RelationshipCalendarModel, RelationshipCalendarDay } from '../../relationships/models'; + import { dayTooltipText } from '../../relationships/calendar-tooltip'; interface Props { calendar: RelationshipCalendarModel | null; loading: boolean; error: string | null; year?: number; - firstYear: number; + firstYear: number | null; currentYear: number; onYearChange: (year: number) => void; } @@ -47,6 +48,16 @@ buildCalendarPanel(calendar, 6, 11, 'second-half') ] : []); const hasActivity = $derived(Boolean(calendar?.days?.some((day) => day.total > 0))); + const multiYear = $derived(firstYear === null || firstYear < currentYear); + let root = $state(); + const tooltipID = $props.id(); + let tooltip = $state(null); + let tooltipNode = $state(); + let activeCell: HTMLElement | null = null; + let pointer: { x: number; y: number } | null = null; + // Survives the pointer being cleared (focus moves, hide) so scroll + // dismissal still knows what kind of pointer was last seen. + let lastPointerType: string | null = null; const weekdays = ['Sun', '', 'Tue', '', 'Thu', '', 'Sat']; const levels = ['none', 'first-quartile', 'second-quartile', 'third-quartile', 'fourth-quartile']; @@ -93,9 +104,98 @@ return date.toISOString().slice(0, 10); } - function dayLabel(day: RelationshipCalendarDay): string { - return `${day.date}: ${day.total} interactions; ${day.sent} sent, ${day.received} received, ` + - `${day.email} email, ${day.chat} chat, ${day.meetings} meetings`; + $effect(() => { + calendar; + hideTooltip(); + }); + + $effect(() => { + const cell = tooltip; + if (!cell || !tooltipNode) return; + positionTooltip(cell); + cell.setAttribute('aria-describedby', tooltipID); + document.addEventListener('keydown', dismissTooltip, true); + return () => { + cell.removeAttribute('aria-describedby'); + document.removeEventListener('keydown', dismissTooltip, true); + }; + }); + + function positionTooltip(target: HTMLElement): void { + if (!root || !tooltipNode) return; + const cell = target.getBoundingClientRect(); + const box = root.getBoundingClientRect(); + const halfWidth = tooltipNode.getBoundingClientRect().width / 2; + const center = cell.left - box.left + cell.width / 2; + tooltipNode.style.left = `${Math.max(halfWidth + 4, Math.min(center, box.width - halfWidth - 4))}px`; + tooltipNode.style.top = `${cell.top - box.top}px`; + } + + function activateTooltip(target: HTMLElement): void { + if (target === activeCell) return; + activeCell = target; + tooltip = target; + } + + function dismissTooltip(event: KeyboardEvent): void { + if (event.key !== 'Escape') return; + event.preventDefault(); + event.stopPropagation(); + tooltip = null; + pointer = null; + // Keep activeCell until the pointer leaves so movement within a dismissed + // day does not reopen it. A different day can open immediately. + } + + function showTooltip(event: Event): void { + if (event.type.startsWith('pointer')) { + const movement = event as PointerEvent; + pointer = { x: movement.clientX, y: movement.clientY }; + lastPointerType = movement.pointerType; + } else if (event.type === 'focusin') { + pointer = null; + } + const target = (event.target as HTMLElement | null)?.closest('button.heat-cell'); + if (target) activateTooltip(target); + else hideTooltip(); + } + + function hideTooltip(): void { + activeCell = null; + tooltip = null; + // Drop captured coordinates so a later scroll cannot resurrect the tip + // from wherever the pointer last was. + pointer = null; + } + + function tooltipPointerLeave(event: PointerEvent): void { + // Touch pointers fire pointerleave right after pointerup; keep a tapped + // day's tooltip up until the next tap or scroll elsewhere. + if (event.pointerType === 'touch') return; + hideTooltip(); + } + + function scrollTooltip(): void { + if (!root || !tooltip) return; + // A touch drag leaves the tap behind (its coordinates may already be + // cleared by focus); scrolling must dismiss the tooltip rather than + // re-pin it to an unrelated cell. + if (lastPointerType === 'touch') { + hideTooltip(); + return; + } + const pointedCell = pointer + ? document.elementFromPoint(pointer.x, pointer.y)?.closest('button.heat-cell') + : activeCell; + if (!pointedCell || !root.contains(pointedCell)) { + hideTooltip(); + return; + } + if (pointedCell !== activeCell) { + activateTooltip(pointedCell); + return; + } + positionTooltip(pointedCell); } function levelClass(day: RelationshipCalendarDay | undefined): string { @@ -104,25 +204,24 @@ {#snippet panel(panel: CalendarPanel, variant: 'full' | 'half')} -
-